def export(user, directory=None, warnings=True): """ Build a temporary directory with the visualization. Returns the local path where files have been written. Examples -------- >>> bandicoot.visualization.export(U) Successfully exported the visualization to /tmp/tmpsIyncS """ # Get dashboard directory current_file = os.path.realpath(__file__) current_path = os.path.dirname(current_file) dashboard_path = os.path.join(current_path, '../dashboard_src') # Create a temporary directory if needed and copy all files if directory: dirpath = directory else: dirpath = tempfile.mkdtemp() # Copy all files except source code copy_tree(dashboard_path + '/public', dirpath, update=1) # Export indicators data = user_data(user) bc.io.to_json(data, dirpath + '/data/bc_export.json', warnings=False) if warnings: print("Successfully exported the visualization to %s" % dirpath) return dirpath
def copyHtml(self): htmlDir = os.path.join(self.LOCALEDIR, "html") try: copy_tree(self.HTMLDIR, htmlDir) except Exception, e: self.raiseError("Unable to copy html from '%s': %s." % (self.HTMLDIR, e))
def install(): autotools.rawInstall("INSTALL_PREFIX=%s MANDIR=/usr/share/man" % get.installDIR()) # Rename conflicting manpages pisitools.rename("/usr/share/man/man1/passwd.1", "ssl-passwd.1") pisitools.rename("/usr/share/man/man3/rand.3", "ssl-rand.3") pisitools.rename("/usr/share/man/man3/err.3", "ssl-err.3") if get.buildTYPE() == "_emul32": from distutils.dir_util import copy_tree copy_tree("%s/_emul32/lib32/" % get.installDIR(), "%s/usr/lib32" % get.installDIR()) pisitools.removeDir("/_emul32") pisitools.remove("/usr/lib32/*.a") path = "%s/usr/lib32/pkgconfig" % get.installDIR() for f in shelltools.ls(path): pisitools.dosed("%s/%s" % (path, f), "^(prefix=\/)_emul32", r"\1usr") return # Move engines to /usr/lib/openssl/engines pisitools.dodir("/usr/lib/openssl") pisitools.domove("/usr/lib/engines", "/usr/lib/openssl") # Certificate stuff pisitools.dobin("tools/c_rehash") pisitools.dosym("/etc/pki/tls/certs/ca-bundle.crt","/etc/pki/tls/cert.pem") # Create CA dirs for cadir in ["CA", "CA/private", "CA/certs", "CA/crl", "CA/newcerts"]: pisitools.dodir("/etc/pki/%s" % cadir) # No static libs pisitools.remove("/usr/lib/*.a") pisitools.dohtml("doc/*") pisitools.dodoc("CHANGES*", "FAQ", "LICENSE", "NEWS", "README", "doc/*.txt")
def _copy_sdk_template(self, gen_api_root, package_type): """ Copy sdk template to gen_api_root directory. Args: gen_api_root (str): Root directory for generated APIs. package_type (str): Sdk template to copied from. Valid options are: core, packages. """ target_dir = os.path.join(self.ydk_root, 'sdk', self.language) if self.language == 'python': if package_type == 'service': service_name = gen_api_root.split('-')[-1] target_dir = os.path.join(target_dir, service_name) else: target_dir = os.path.join(target_dir, package_type) elif self.language == 'cpp': if package_type == 'packages': target_dir = os.path.join(target_dir, package_type) elif package_type == 'core': target_dir = os.path.join(target_dir, 'core') elif package_type == 'service': service_name = gen_api_root.split('-')[-1] target_dir = os.path.join(target_dir, service_name) elif self.language == 'go': if package_type == 'service': service_name = gen_api_root.split('-')[-1] target_dir = os.path.join(target_dir, service_name) else: target_dir = os.path.join(target_dir, package_type) shutil.rmtree(gen_api_root) logger.debug('Copying %s to %s' % (target_dir, gen_api_root)) dir_util.copy_tree(target_dir, gen_api_root)
def install(): pisitools.dodir("/usr/share") wanteddirs = [] for file_ in shelltools.ls(get.workDIR()): if shelltools.isDirectory(file_) and not "texmf" in file_: wanteddirs.append(file_) for folder in wanteddirs: pisitools.insinto("/usr/share/texmf-dist", folder) if shelltools.can_access_directory("texmf-dist"): # Recursively copy on directory on top of another, overwrite duplicate files too copy_tree("texmf-dist", "%s/usr/share/texmf-dist" % get.installDIR()) ## chmod of script files script_dir = get.installDIR() + "/usr/share/texmf-dist/scripts" if shelltools.can_access_directory(script_dir): for root, dirs, files in os.walk(script_dir): for name in files: shelltools.chmod(os.path.join(root, name), 0755) # copy config file to texmf-config pisitools.dodir("/etc/texmf/tex/context/config") shelltools.copy("%s/usr/share/texmf-dist/tex/context/config/cont-usr.tex" % get.installDIR(), \ "%s/etc/texmf/tex/context/config/cont-usr.tex" % get.installDIR()) # old packages, we will not provide them pisitools.remove("/usr/share/texmf-dist/tex/plain/config/omega.ini") pisitools.remove("/usr/share/texmf-dist/tex/plain/config/aleph.ini") pisitools.removeDir("/usr/share/texmf-dist/scripts/context/stubs/mswin/")
def fetch(in_base, in_id, out_base='/var/tmp/repositorg/', in_id_is_file=False, in_path='', ): """Fetch data and reposit verbatim inside repositorg temporal directory Parameters ---------- in_base : str Base input directory from which to copy files. in_id : str UUID of the source device. This string will be added as a subidrectory under the `out_base` directory. out_base : str, optional Output base directory. A subdirectory named according to the value of `in_id` will be created under this directory to contain the output files. in_id_is_file : bool, optional Whether the value passed to `in_id` is a file name or path. If `True`, the `in_id` will be stripped of the extension and its basename will be extracted. in_path : str, optional An additional subpath to be added under the `in_base` directory. """ if in_id_is_file: in_id = os.path.basename(in_id) in_id = os.path.splitext(in_id)[0] in_path = os.path.join(in_base,in_path) out_path = os.path.join(out_base,in_id) if not os.path.isdir(in_path): return False dir_util.copy_tree(in_path, out_path, preserve_mode=0 )
def constructPreBuiltHTMITFakeroot(self): """ Construct fakeroot from prebuilt htm-it :returns: SHA of the products repo in the fakeroot :rtype: tuple """ config = self.config logger = self.logger productsDirectory = self.productsDirectory logger.debug("Creating %s", productsDirectory) mkpath(productsDirectory) copy_tree(config.productsDir, productsDirectory) iteration = git.getCommitCount(productsDirectory, logger=logger) with changeToWorkingDir(productsDirectory): actualSHA = git.getCurrentSha(logger=logger) # Set extra python path self.setPythonPath() # Clean HTM-IT Scripts self.cleanScripts() # Purge anything not whitelisted self.purgeBlacklistedStuff() return (iteration, actualSHA)
def openTestProject(name): orgPath = os.path.join(os.path.dirname(__file__), "data", "projects", name) destPath = tempFolderInTempFolder() copy_tree(orgPath, destPath) projectFile = os.path.join(destPath, name + ".qgs") if projectFile != QgsProject.instance().fileName(): iface.addProject(projectFile)
def copy_folder_from(self, source, incremental=False): """ Copies the given source directory to this directory. If incremental is True only newer objects are overwritten. """ # There is a bug in dir_util that makes copy_tree crash if a folder in the # tree has been deleted before and readded now. To workaround the bug, we first # walk the tree and create directories that are needed. # # pylint: disable-msg=C0111,W0232 target_root = self class _DirCreator: @staticmethod def visit_folder(folder): target = folder.get_mirror_folder( source.parent, target_root, ignore_root=True) target.make() source.walk(_DirCreator) dir_util.copy_tree(str(source), self.child(source.name), preserve_symlinks=True, update=incremental)
def updateIWSPlugin(self): self.logger.info(u"Performing update of DomoPad" "s IWS plugin...") # determine the IWS server directory indigoInstallPath = indigo.server.getInstallFolderPath() pluginBasePath = os.getcwd() mainPluginHome = os.path.join(pluginBasePath, "AndroidClientHelper") iwsPluginHome = os.path.join(indigoInstallPath, "IndigoWebServer/plugins/AndroidClientHelper") self.logger.info(u"Source IWS directory: " + mainPluginHome) self.logger.info(u"Target IWS directory: " + iwsPluginHome) # ensure that we have the correct source directory... if os.path.exists(mainPluginHome) == False: self.logger.error( u"ERROR: Source directory not found! AndroidClientHelper IWS plugin install could not complete." ) return # execute the directory copy now... try: copy_tree(mainPluginHome, iwsPluginHome, preserve_mode=1) self.logger.info( u"AndroidClientHelper successfully installed/updated. Restarting Indigo IWS server to complete install." ) self.restartIWS() except: self.logger.error( u"Error copying AndroidClientHelper, AndroidClientHelper IWS plugin install could not complete." )
def bookmarkCurrentPage(self, dimdimID, newResourceID): meeting = self.getMeeting(dimdimID) resourceList = meeting.get('resources') currentResourceID = meeting.get('currentResource') currentResourceCopy = self.getCurrentResource(dimdimID) currentResourceType = self.getResourceType(dimdimID, currentResourceID) currentPageNum = currentResourceCopy.get('currentPage') currentPageCopy = self.getCurrentPage(dimdimID) newResource = {'currentPage':0, 'lastPage':0, 'pages': {0:currentPageCopy}} sourcePath = os.path.join(os.path.join(os.path.join(os.path.join(osconfig.cobArchive(), dimdimID), \ currentResourceType), currentResourceID), str(currentPageNum)) destPath = os.path.join(os.path.join(os.path.join(os.path.join(osconfig.cobArchive(), dimdimID), \ currentResourceType), newResourceID), '0') try: shutil.rmtree(destPath) except: pass dir_util.copy_tree(sourcePath, destPath) typeList = resourceList.get(currentResourceType) self.stateLock.acquire() typeList.update({newResourceID : newResource}) self.exportStateMachine(dimdimID) self.stateLock.release() return
def handle(self, dump_path, **options): if dump_path == "-": arc = tarfile.open(fileobj=sys.stdin, mode="r:gz") else: arc = tarfile.open(dump_path, mode="r:gz") base_path = tempfile.mkdtemp() arc.extractall(path=base_path) path = glob(os.path.join(base_path, "*"))[0] # media files # shutil.copytree(os.path.join(path, 'media'), settings.MEDIA_ROOT) dir_util.copy_tree(os.path.join(path, "media"), settings.MEDIA_ROOT) # load db fields old_stdout = sys.stdout sys.stdout = open(os.path.join(path, "backup_db_dump.json"), "w") call_command("dumpdata", indent=4) sys.stdout.close() sys.stdout = old_stdout call_command("flush", noinput=True, interactive=False) call_command("reset", "contenttypes", "auth", noinput=True, interactive=False) call_command("loaddata", os.path.join(path, "db_dump.json")) # rebase FilepathFields call_command("rebase_filepathfields", os.path.join(path, "fpf_bases_dump.json"))
def install(): pisitools.dodir("/usr/share") wanteddirs = [] for file_ in shelltools.ls('.'): if shelltools.isDirectory(file_) and not "texmf" in file_: wanteddirs.append(file_) for folder in wanteddirs: pisitools.insinto("/usr/share/texmf-dist", folder) if shelltools.can_access_directory("texmf-dist"): # Recursively copy on directory on top of another, overwrite duplicate files too copy_tree("texmf-dist", "%s/usr/share/texmf-dist" % get.installDIR()) ## chmod of script files script_dir = get.installDIR() + "/usr/share/texmf-dist/scripts" if shelltools.can_access_directory(script_dir): for root, dirs, files in os.walk(script_dir): for name in files: shelltools.chmod(os.path.join(root, name), 0755) pisitools.remove("/usr/share/texmf-dist/scripts/m-tx/m-tx.lua") pisitools.remove("/usr/share/texmf-dist/scripts/musixtex/musixtex.lua") pisitools.remove("/usr/share/texmf-dist/scripts/musixtex/musixflx.lua") pisitools.remove("/usr/share/texmf-dist/scripts/pmx/pmx2pdf.lua")
def extract_rpclib(args): """ THE extraction function """ workdir = tempfile.mkdtemp() rpclib_root = args.rpclib_root new_rpclib_root = tempfile.mkdtemp() copy_tree(rpclib_root, new_rpclib_root) patch_rpclib(new_rpclib_root) copy_important_files(workdir, new_rpclib_root) version = extract_rpclib_version(new_rpclib_root) shutil.rmtree(new_rpclib_root) cmake_template = Template( filename=os.path.join(os.path.dirname(os.path.realpath(__file__)), "rpc_CMakeLists.txt")) real_cmake_file = cmake_template.render( rpclib_major_version=version[0], rpclib_minor_version=version[1], rpclib_patch_version=version[2] ) with open(os.path.join(workdir, "CMakeLists.txt"), "w") as cmake_file: cmake_file.write(real_cmake_file) cmake_file.truncate() fix_naming(workdir) copy_tree(workdir, args.rpclib_target, update=True) shutil.rmtree(workdir)
def run(self): if sys.platform not in ("linux2", "win32"): msg = "**Error: Can't install on this platform: %s" % sys.platform print msg sys.exit(1) if not self.single_version_externally_managed: self.do_egg_install() else: _install.run(self) # check if someone else is copying the files to the destination # if self.single_version_externally_managed:# and not self.root: # return # install bauble.desktop and icons if sys.platform == "linux2": # install everything in share dir_util.copy_tree(os.path.join(self.build_base, "share"), os.path.join(self.install_data, "share")) elif sys.platform == "win32": # install only i18n files locales = os.path.dirname(locale_path) install_cmd = self.get_finalized_command("install") build_base = install_cmd.build_base src = os.path.join(build_base, locales) dir_util.copy_tree(src, os.path.join(self.install_data, locales))
def windows_package(args): pkgfile = 'windows_package.7z' pkgdir = os.path.abspath('windows_package') logging.info("Packaging libraries and headers in package: %s", pkgfile) j = os.path.join pkgdir_lib = os.path.abspath(j(pkgdir, 'lib')) with remember_cwd(): os.chdir(args.output) logging.info("Looking for static libraries and dlls in: \"%s", os.getcwd()) libs = list(glob.iglob('**/*.lib', recursive=True)) dlls = list(glob.iglob('**/*.dll', recursive=True)) os.makedirs(pkgdir_lib, exist_ok=True) for lib in libs: logging.info("packing lib: %s", lib) shutil.copy(lib, pkgdir_lib) for dll in dlls: logging.info("packing dll: %s", dll) shutil.copy(dll, pkgdir_lib) os.chdir(get_mxnet_root()) logging.info('packing python bindings') copy_tree('python', j(pkgdir, 'python')) logging.info('packing headers') copy_tree('include', j(pkgdir, 'include')) logging.info("Compressing package: %s", pkgfile) check_call(['7z', 'a', pkgfile, pkgdir])
def add_local_binaries_to_stage(self): progress("Adding local binaries to staging directory") safe_make_directory("appleseed/bin") dir_util.copy_tree(os.path.join(self.settings.appleseed_path, "sandbox/bin", self.settings.configuration), "appleseed/bin/") shutil.copy(os.path.join(self.settings.appleseed_path, "sandbox/bin", exe("maketx")), "appleseed/bin/") shutil.copy(os.path.join(self.settings.appleseed_path, "sandbox/bin", exe("oslc")), "appleseed/bin/") shutil.copy(os.path.join(self.settings.appleseed_path, "sandbox/bin", exe("oslinfo")), "appleseed/bin/")
def run(self): # TODO: make sure we have everything installed that we need to # bundle e.g. sqlite, psycopg2, others... _py2exe_cmd.run(self) # install locale files locales = os.path.dirname(locale_path) build_base = self.get_finalized_command("build").build_base # print build_base src = os.path.join(build_base, locales) dir_util.copy_tree(src, os.path.join(self.dist_dir, locales)) # copy GTK to the dist directory, assuming PyGTK # all-in-one installer gtk_root = "c:\\python27\\lib\\site-packages\\gtk-2.0\\runtime" dist_gtk = os.path.join(self.dist_dir, "gtk") import shutil if not os.path.exists(dist_gtk): ignore = shutil.ignore_patterns("src", "gtk-doc", "icons", "man", "demo", "aclocal", "doc", "include") shutil.copytree(gtk_root, dist_gtk, ignore=ignore) # register the pixbuf loaders exe = "%s\\bin\\gdk-pixbuf-query-loaders.exe" % dist_gtk dest = "%s\\etc\\gtk-2.0\\gdk-pixbuf.loaders" % dist_gtk cmd = 'call "%s" > "%s"' % (exe, dest) print cmd os.system(cmd) # copy the the MS-Windows gtkrc to make it the default theme rc = "%s\\share\\themes\\MS-Windows\\gtk-2.0\\gtkrc" % dist_gtk dest = "%s\\etc\\gtk-2.0" % dist_gtk file_util.copy_file(rc, dest)
def copy_sources(): """Copy the C sources into the source directory. This rearranges the source files under the python distribution directory. """ src = [] try: dir_util.remove_tree("src/") except (IOError, OSError): pass dir_util.copy_tree("../../arch", "src/arch/") dir_util.copy_tree("../../include", "src/include/") src.extend(glob.glob("../../*.[ch]")) src.extend(glob.glob("../../*.mk")) src.extend(glob.glob("../../Makefile")) src.extend(glob.glob("../../LICENSE*")) src.extend(glob.glob("../../README")) src.extend(glob.glob("../../*.TXT")) src.extend(glob.glob("../../RELEASE_NOTES")) src.extend(glob.glob("../../make.sh")) src.extend(glob.glob("../../CMakeLists.txt")) for filename in src: outpath = os.path.join("./src/", os.path.basename(filename)) log.info("%s -> %s" % (filename, outpath)) shutil.copy(filename, outpath)
def install(): """Install MLTSP config file and create data folders. Copies mltsp.yaml.example to ~/.config/mltsp/mltsp.yaml and creates data directories as described in `mltsp.config.cfg['paths']` """ import os import shutil from distutils.dir_util import copy_tree data_src = os.path.join(os.path.dirname(__file__), "data") data_dst = os.path.expanduser('~/.local/mltsp/') copy_tree(data_src, data_dst, update=1) print("Created data directory at {} and copied sample data.".format( os.path.expanduser('~/.local/mltsp/'))) cfg = os.path.expanduser('~/.config/mltsp/mltsp.yaml') cfg_dir = os.path.dirname(cfg) if os.path.exists(cfg): print('Existing configuration at {} -- not overwriting.'.format(cfg)) return if not os.path.exists(cfg_dir): os.makedirs(cfg_dir) shutil.copyfile(os.path.join(os.path.dirname(__file__), 'mltsp.yaml.example'), cfg) print('Installed {}'.format(cfg)) print('Please customize this file with authentication tokens, etc.')
def copy_static(args): if '../' in args['html_folder']: dir_util.copy_tree(os.path.join(os.path.dirname(__file__), 'static'), os.path.join(os.path.abspath(args['html_folder']), 'static')) else: dir_util.copy_tree(os.path.join(os.path.dirname(__file__), 'static'), os.path.join(os.path.abspath(args['html_folder']), 'static'))
def __deploy_app(self): """ Deploy the local SO bundle assumption here - a git repo is returned - the bundle is not managed by git """ # create temp dir...and clone the remote repo provided by OpS tmp_dir = tempfile.mkdtemp() LOG.debug('Cloning git repository: ' + self.repo_uri + ' to: ' + tmp_dir) cmd = ' '.join(['git', 'clone', self.repo_uri, tmp_dir]) os.system(cmd) # Get the SO bundle bundle_loc = CONFIG.get('service_manager', 'bundle_location', '') if bundle_loc == '': raise Exception('No bundle_location parameter supplied in sm.cfg') LOG.debug('Bundle to add to repo: ' + bundle_loc) dir_util.copy_tree(bundle_loc, tmp_dir) self.__add_openshift_files(bundle_loc, tmp_dir) # add & push to OpenShift os.system(' '.join(['cd', tmp_dir, '&&', 'git', 'add', '-A'])) os.system(' '.join(['cd', tmp_dir, '&&', 'git', 'commit', '-m', '"deployment of SO for tenant ' + self.extras['tenant_name'] + '"', '-a'])) LOG.debug('Pushing new code to remote repository...') os.system(' '.join(['cd', tmp_dir, '&&', 'git', 'push'])) shutil.rmtree(tmp_dir)
def _copy_resources(template, instance, resource, dir=False): if dir: src = path.join(template, resource) dst = path.join(instance, resource) copy_tree(src, dst) else: raise NotImplementedError()
def mergeDir(self, path, dest): """ We don't want to delete the old dir, since it might contain third party plugin content from previous installations; we simply want to merge the existing directory with the new one. """ copy_tree(path, os.path.join(dest, path), preserve_symlinks=True)
def copy_base_course(self): """ Create all the base files needed to restore the course. """ print ' > Creating course base files... ', if not os.path.exists(self.course_dir): os.makedirs(self.course_dir) copy_tree(settings.BASE_COURSE, self.course_dir) print 'Done.'
def pyinstall(source_folder): pyinstaller_path = os.path.join(os.path.curdir, 'pyinstaller') _install_pyintaller(pyinstaller_path) try: shutil.rmtree(os.path.join(pyinstaller_path, 'conan')) except Exception as e: print("Unable to remove old folder", e) try: shutil.rmtree(os.path.join(pyinstaller_path, 'conan_server')) except Exception as e: print("Unable to remove old server folder", e) conan_path = os.path.join(source_folder, 'conans', 'conan.py') conan_server_path = os.path.join(source_folder, 'conans', 'conan_server.py') subprocess.call('python pyinstaller.py -y -p %s --console %s' % (source_folder, conan_path), cwd=pyinstaller_path, shell=True) _run_bin(pyinstaller_path) subprocess.call('python pyinstaller.py -y -p %s --console %s' % (source_folder, conan_server_path), cwd=pyinstaller_path, shell=True) conan_bin = os.path.join(pyinstaller_path, 'conan', 'dist', 'conan') conan_server_folder = os.path.join(pyinstaller_path, 'conan_server', 'dist', 'conan_server') dir_util.copy_tree(conan_server_folder, conan_bin) _run_bin(pyinstaller_path) return os.path.abspath(os.path.join(pyinstaller_path, 'conan', 'dist', 'conan'))
def deploy(config, host): # add -H command to set home directory of target user new_sudo_command = "sudo -H -S -p '%(sudo_prompt)s'" print colors.green('change sudo_prefix from "%s" to "%s"' % (env['sudo_prefix'], new_sudo_command)) with settings(sudo_prefix=new_sudo_command): fixtures_dir = tempfile.mkdtemp() print colors.green('create fixtures directory: "%s"' % fixtures_dir) try: print colors.green('merge fixtures') for source_fixtures_dir in reversed(config.fixtures): dir_util.copy_tree(source_fixtures_dir, fixtures_dir) config.fixtures_path = fixtures_dir host.prepair() host.check() host.setup() except Exception, e: traceback.print_exc() print colors.red('Exception while deploing: "%s"' % e) finally:
def aggregate(outputdir=OUTPUT_DIR, buildname=DEFAULT_BUILD_NAME): """ Aggregates the captures from various campaigns to the 'default' :return: """ if not os.path.exists(outputdir): raise ValueError("outputdir does not exist at %s!" % outputdir) # Aggregate all the capture job data outputdir = str(outputdir).rstrip('/') buildpaths = glob.glob(outputdir + '/*/') aggregate_path = os.path.join(outputdir, buildname) if not os.path.exists(aggregate_path): LOGGER.debug("Creating path for aggregates at %s", aggregate_path) os.makedirs(aggregate_path) LOGGER.info("Aggregating build data to %s", aggregate_path) # Workaround bug with dir_util # See http://stackoverflow.com/questions/9160227/ dir_util._path_created = {} for buildpath in buildpaths: if str(buildpath).endswith(DEFAULT_BUILD_NAME + "/"): # Don't do this for the default build LOGGER.debug("Skipping default path: %s", buildpath) continue sourcepath = os.path.join(outputdir, buildpath) LOGGER.debug("Copying from %s to %s", buildpath, aggregate_path) dir_util.copy_tree(sourcepath, aggregate_path, update=1) return aggregate_path
def cp(source, target, force=True): if isinstance(source, list): # list # copy files to dir targets = [] for s in source: t = cp(s, target, force) targets.append(t) return targets assert_exists(source) # assert exists if not force and os.path.exists(target): return if source == target: return target if os.path.isfile(source) and os.path.isdir(target): # target is DIR target = os.path.join(target, os.path.basename(source)) if os.path.isfile(source) or os.path.islink(source): if (os.path.exists(target) or os.path.lexists(target)): if os.path.isfile(source) != os.path.isfile(target): os.unlink(target) shutil.copy(source, target) if os.path.isdir(source): # first create dirs if not os.path.exists(target): os.makedirs(target) dir_util.copy_tree(source, target) return target
def handle(self, *args, **options): app_name = args[0] module = importlib.import_module(app_name) path = os.path.dirname(module.__file__) + os.sep project_path = os.path.dirname(os.path.normpath(os.sys.modules[settings.SETTINGS_MODULE].__file__)) install_app('social_auth') copy_tree(os.path.dirname(djangobp.__file__) + os.sep + 'scaffold/socialauth/templates/accounts', path + '/templates/accounts', update=True) copy_file(os.path.dirname(djangobp.__file__) + os.sep + 'scaffold/socialauth/controllers/accounts.py', path + '/controllers', update=True) copy_file(os.path.dirname(djangobp.__file__) + os.sep + 'scaffold/scaffold/socialauthsettings.py', project_path, update=True) urls_edit = CodeEditor(path + 'urls.py') urls_edit.insert_line("url(r'', include('social_auth.urls')),", 'urlpatterns') urls_edit.commit() settings_edit = CodeEditor(project_path + os.sep + 'settings.py') settings_edit.append_line("from socialauthsettings import *") settings_edit.commit() # TODO copy controllers/accounts.py # TODO copy templates/accounts/login.html # TODO urls social auth # TODO django-social-auth settings
def copy_directory(src, dst, clear_dst=False): ensure_directory(dst) if clear_dst: clear_directory(dst) from distutils.dir_util import copy_tree copy_tree(src, dst)
def main(self): begin = time.time() source_language = "en" total_char = 0 labs = [f.path for f in os.scandir(self.src_dir) if f.is_dir()] for lab in labs: print("Translating lab manual: {}".format(lab)) start_time = time.time() for target_language_code in self.codes: start_time_lab = time.time() print( "##########################################################" ) print("Translating content from {} to {}".format( source_language, target_language_code)) with open(str(lab) + '/content.xml') as f: tree = ET.parse(f) root = tree.getroot() len_of_text = 0 for elem in root.getiterator(): text_translate = elem.text tag = elem.tag if text_translate is None or text_translate.isspace(): continue else: self.logger.info( "---------------------Start of text----------------------------------" ) self.logger.info( "-----------------------translating text-----------------------------" ) curr_len = len(text_translate.encode('utf-8')) total_char += curr_len if tag == 'name' or tag == 'dataFormat': continue if tag == 'defaultLanguageCode': elem.text = str(target_language_code) continue result = self.language_translation( text=text_translate, source_language="en", target_language=target_language_code, model=self.modelType) if self.modelType == 'custom': self.logger.info(result) elem.text = str(result) else: self.logger.info(result['translatedText']) elem.text = str(result['translatedText']) len_of_text += curr_len self.logger.info( "-----------------------translation complete-----------------------------" ) self.logger.info( "---------------------End of text----------------------------------" ) self.logger.info("") if not os.path.isdir(self.dst_dir + '/' + str(lab[7:-2]) + target_language_code): os.mkdir(self.dst_dir + '/' + str(lab[7:-2]) + target_language_code) os.mkdir(self.dst_dir + '/' + str(lab[7:-2]) + target_language_code + '/images') tree.write(self.dst_dir + '/' + str(lab[7:-2]) + target_language_code + '/content.xml', encoding="UTF-8") copy_tree( str(lab) + '/images', self.dst_dir + '/' + str(lab[7:-2]) + target_language_code + '/images') end_time = time.time() print( "Total elapsed time to translate the xml document is: {} minutes" .format((end_time - start_time_lab) / 60)) is_upload = self.upload_s3( self.dst_dir + '/' + str(lab[7:-2]) + target_language_code + '.zip', self.dst_dir, self.bucket_name, str(lab[7:-2]) + "all_translations", str(lab[7:-2]) + target_language_code) if not is_upload: print("Upload failed....Exiting!!") exit(0) final_end_time = time.time() print( "Total characters processed in this translating lab {} are {} chars" .format(lab, total_char)) print("Total Translation time {} minutes".format( (final_end_time - start_time) / 60)) end = time.time() print("Total execution time for all labs: {} hrs".format( (end - begin) / 3600))
def main(): if len(sys.argv) < 2 or sys.argv[1].startswith('-'): print usage % (aliases, ets_package_names) return arg1 = sys.argv[1] # Update the gh-pages branch if arg1 == 'update': if 2 < len(sys.argv): ets_packages = sys.argv[2:] else: ets_packages = ets_package_names.split() for ets_pkg_name in ets_packages: print "Updating documentation branch for {0}...".format( ets_pkg_name) # Find the current branch, so that we may return to it branches = subprocess.check_output(['git', 'branch'], cwd=ets_pkg_name) current_branch = [ line.split()[1] for line in branches.splitlines() if line.startswith('*') ] current_branch = current_branch[0] # Checkout the gh-pages branch try: subprocess.check_call(['git', 'checkout', 'gh-pages'], cwd=ets_pkg_name) except (OSError, subprocess.CalledProcessError), detail: print " Error running command in package %s:\n %s" % ( ets_pkg_name, detail) raw_input(" Press enter to process remaining packages.") continue # Copy the files over print "Copying files for {0}".format(ets_pkg_name) if ets_pkg_name == 'mayavi': copy_tree(ets_pkg_name + '/docs/build/tvtk/html/', ets_pkg_name + '/tvtk/') copy_tree(ets_pkg_name + '/docs/build/mayavi/html/', ets_pkg_name + '/mayavi/') else: copy_tree(ets_pkg_name + '/docs/build/html/', ets_pkg_name) # Add everything to the repository try: subprocess.check_call(['git', 'add', '.'], cwd=ets_pkg_name) except (OSError, subprocess.CalledProcessError), detail: print " Error running command in package %s:\n %s" % ( ets_pkg_name, detail) raw_input(" Press enter to process remaining packages.") continue # Commit to the repo. try: subprocess.check_call( ['git', 'commit', '-a', '-m', '"Updated docs."'], cwd=ets_pkg_name) except (OSError, subprocess.CalledProcessError), detail: print " Error running command in package %s:\n %s" % ( ets_pkg_name, detail) raw_input(" Press enter to process remaining packages.") continue
def _compare_baseline(self): with self._test_status: if int(self._case.get_value("RESUBMIT")) > 0: # This is here because the comparison is run for each submission # and we only want to compare once the whole run is finished. We # need to return a pass here to continue the submission process. self._test_status.set_status(CIME.test_status.BASELINE_PHASE, CIME.test_status.TEST_PASS_STATUS) return self._test_status.set_status(CIME.test_status.BASELINE_PHASE, CIME.test_status.TEST_FAIL_STATUS) run_dir = self._case.get_value("RUNDIR") case_name = self._case.get_value("CASE") base_dir = os.path.join( self._case.get_value("BASELINE_ROOT"), self._case.get_value("BASECMP_CASE"), ) test_name = "{}".format(case_name.split(".")[-1]) evv_config = { test_name: { "module": os.path.join(evv_lib_dir, "extensions", "ks.py"), "test-case": "Test", "test-dir": run_dir, "ref-case": "Baseline", "ref-dir": base_dir, "var-set": "default", "ninst": NINST, "critical": 13, "component": self.component, } } json_file = os.path.join(run_dir, ".".join([case_name, "json"])) with open(json_file, "w") as config_file: json.dump(evv_config, config_file, indent=4) evv_out_dir = os.path.join(run_dir, ".".join([case_name, "evv"])) evv(["-e", json_file, "-o", evv_out_dir]) with open(os.path.join(evv_out_dir, "index.json")) as evv_f: evv_status = json.load(evv_f) comments = "" for evv_elem in evv_status["Data"]["Elements"]: if (evv_elem["Type"] == "ValSummary" and evv_elem["TableTitle"] == "Kolmogorov-Smirnov test"): comments = "; ".join("{}: {}".format(key, val) for key, val in evv_elem["Data"] [test_name][""].items()) if evv_elem["Data"][test_name][""]["Test status"].lower( ) == "pass": self._test_status.set_status( CIME.test_status.BASELINE_PHASE, CIME.test_status.TEST_PASS_STATUS, ) break status = self._test_status.get_status( CIME.test_status.BASELINE_PHASE) mach_name = self._case.get_value("MACH") mach_obj = Machines(machine=mach_name) htmlroot = CIME.utils.get_htmlroot(mach_obj) urlroot = CIME.utils.get_urlroot(mach_obj) if htmlroot is not None: with CIME.utils.SharedArea(): dir_util.copy_tree( evv_out_dir, os.path.join(htmlroot, "evv", case_name), preserve_mode=False, ) if urlroot is None: urlroot = "[{}_URL]".format(mach_name.capitalize()) viewing = "{}/evv/{}/index.html".format(urlroot, case_name) else: viewing = ( "{}\n" " EVV viewing instructions can be found at: " " https://github.com/E3SM-Project/E3SM/blob/master/cime/scripts/" "climate_reproducibility/README.md#test-passfail-and-extended-output" "".format(evv_out_dir)) comments = ("{} {} for test '{}'.\n" " {}\n" " EVV results can be viewed at:\n" " {}".format( CIME.test_status.BASELINE_PHASE, status, test_name, comments, viewing, )) CIME.utils.append_testlog(comments, self._orig_caseroot)
with open(logfile, 'w') as ff: ff.write(WORK_DIR) ff.write('\n\n\n') # main directory to be used when running the knife: KNIFE_DIR = "/srv/software/knife/circularRNApipeline_Standalone" # place files in appropriate locations; fix code in future to # avoid this step anly_src = (KNIFE_DIR + "/analysis") anly_dst = (RESOURCE_DIR + "/analysis") anly_dst2 = (WORK_DIR + "/analysis") if not os.path.exists(anly_dst): copy_tree(anly_src, anly_dst) if not os.path.exists(anly_dst2): os.symlink(anly_dst, anly_dst2) comprun_src = (KNIFE_DIR + "/completeRun.sh") comprun_dst = (RESOURCE_DIR + "/completeRun.sh") comprun_dst2 = (WORK_DIR + "/completeRun.sh") if not os.path.exists(comprun_dst): copyfile(comprun_src, comprun_dst) if not os.path.exists(comprun_dst2): os.symlink(comprun_dst, comprun_dst2) findcirc_src = (KNIFE_DIR + "/findCircularRNA.sh") findcirc_dst = (RESOURCE_DIR + "/findCircularRNA.sh") findcirc_dst2 = (WORK_DIR + "/findCircularRNA.sh") if not os.path.exists(findcirc_dst):
print(line) print("Compiling with SBT...") proc = subprocess.Popen('sbt fullOptJS',shell=True) proc.wait() print("Moving files to {}...".format(args.output)) from distutils import dir_util,file_util assets_folder='assets' output_assets_folder=os.path.join(output_folder,"") makedirs_ifnot(output_assets_folder) print("Moving assets folder from '{}' to '{}'...".format(assets_folder,output_assets_folder)) dir_util.copy_tree(assets_folder,output_assets_folder) compiled_js_folder='target/scala-2.11/' output_compiled_js_folder=os.path.join(output_folder,"") makedirs_ifnot(output_compiled_js_folder) compiled_js_file='vonsim-opt.js' compiled_js_file_map=compiled_js_file+".map" compiled_js_filepath=os.path.join(compiled_js_folder,compiled_js_file) compiled_js_filepath_map=os.path.join(compiled_js_folder,compiled_js_file_map) output_compiled_js_file=os.path.join(output_folder,compiled_js_file) output_compiled_js_file_map=os.path.join(output_folder,compiled_js_file_map) print("Moving compiled js file to '{}' and source map to '{}' ...".format(output_compiled_js_file,output_compiled_js_file_map)) file_util.copy_file(compiled_js_filepath,output_compiled_js_file) file_util.copy_file(compiled_js_filepath_map,output_compiled_js_file_map)
id = 0 for r, d, file in os.walk(d): for f in file: fname, fextension = os.path.splitext(f) if fextension.lower() in args.extensions or any( a in fname.lower() for a in args.keywords): #\\?\ extends character limit? Thanks timmy #copy2 to preserve metadata shutil.copy2( os.path.join(r, f), os.path.join( "\\\?\\" + args.destination[0], os.path.join( destination, fname + "_" + str(id) + fextension))) id += 1 else: dir_util.copy_tree( d, os.path.join(args.destination[0], str(time.strftime("%Y%m%d-%H%M%S")))) if not args.quiet: print "Running..." time.sleep(1) original_drives = temp
targetmhdir = os.path.abspath(os.path.join(exportDir, 'makehuman')) for d in deleteAfterExport: f = os.path.abspath(os.path.join(targetmhdir, d)) if os.path.exists(f): if os.path.isfile(f): os.remove(f) else: shutil.rmtree(f) pluginsdir = os.path.abspath(os.path.join(targetmhdir, 'plugins')) mhx2 = os.path.abspath(os.path.join(parentdir, 'mhx2-makehuman-exchange')) if os.path.exists(mhx2): tocopy = os.path.abspath(os.path.join(mhx2, '9_export_mhx2')) todest = os.path.abspath(os.path.join(pluginsdir, '9_export_mhx2')) copy_tree(tocopy, todest) else: print("MHX2 was not found in parent directory") asset = os.path.abspath( os.path.join(parentdir, 'community-plugins-assetdownload')) if os.path.exists(asset): tocopy = os.path.abspath(os.path.join(asset, '8_asset_downloader')) todest = os.path.abspath(os.path.join(pluginsdir, '8_asset_downloader')) copy_tree(tocopy, todest) else: print("asset downloader was not found in parent directory") mhapi = os.path.abspath(os.path.join(parentdir, 'community-plugins-mhapi')) if os.path.exists(mhapi): tocopy = os.path.abspath(os.path.join(mhapi, '1_mhapi'))
def merge(self): c = self.app.conf if path.isdir(path.join(c.paths.cwd, 'initrd')): copy_tree(path.join(c.paths.cwd, 'initrd'), c.paths.initrd)
def copytree(src, dst): copy_tree(src,dst) return
def copy_web_template(self): '''Copy the base web asset files from self.directory to ./web''' copy_tree(self.template_dir, target_dir)
def bcl2fq(config): ''' takes things from /dont_touch_this/solexa_runs/XXX/Data/Intensities/BaseCalls and writes most output into config.outputDir/XXX, where XXX is the run ID. ''' lanes = config.get("Options", "lanes") if lanes != '': lanes = '_lanes{}'.format(lanes) #Make the output directories os.makedirs("%s/%s%s" % (config.get( "Paths", "outputDir"), config.get("Options", "runID"), lanes), exist_ok=True) #Make log directory os.makedirs( "%s" % (os.path.join(config.get("Paths", "logDir"), os.path.dirname(config.get("Options", "runID")))), exist_ok=True) os.makedirs(os.path.join(config.get("Paths", "outputDir"), config.get("Options", "runID"), 'InterOp'), exist_ok=True) copy_tree( os.path.join(config.get("Paths", "baseDir"), config.get("Options", "sequencer"), 'data', config.get("Options", "runID"), 'InterOp'), os.path.join(config.get("Paths", "outputDir"), config.get("Options", "runID"), 'InterOp')) old_wd = os.getcwd() os.chdir( os.path.join(config.get('Paths', 'outputDir'), config.get('Options', 'runID'))) if config.get("Options", "singleCell") == "1": #TODO: --interop-dir not supported for cellranger cmd = "{cellranger_cmd} --output-dir={output_dir} --sample-sheet={sample_sheet} --run={run_dir} {cellranger_options}".format( cellranger_cmd=config.get("cellranger", "cellranger_mkfastq"), output_dir="{}/{}".format(config.get("Paths", "outputDir"), config.get("Options", "runID")), sample_sheet=config.get("Options", "sampleSheet"), run_dir="{}/{}/data/{}".format(config.get("Paths", "baseDir"), config.get("Options", "sequencer"), config.get("Options", "runID")), cellranger_options=config.get("cellranger", "cellranger_mkfastq_options")) else: cmd = "%s %s --sample-sheet %s -o %s/%s%s -R %s/%s/data/%s --interop-dir %s/%s/InterOp" % ( config.get("bcl2fastq", "bcl2fastq"), config.get("bcl2fastq", "bcl2fastq_options"), config.get("Options", "sampleSheet"), config.get("Paths", "outputDir"), config.get("Options", "runID"), lanes, config.get("Paths", "baseDir"), config.get("Options", "sequencer"), config.get("Options", "runID"), config.get("Paths", "outputDir"), config.get("Options", "runID"), ) syslog.syslog("[bcl2fq] Running: %s\n" % cmd) logOut = open( "%s/%s%s.log" % (config.get("Paths", "logDir"), config.get("Options", "runID"), lanes), "w") subprocess.check_call(cmd, stdout=logOut, stderr=subprocess.STDOUT, shell=True) logOut.close() os.chdir(old_wd)
def main(args, train_env): logging.basicConfig(level=args.logger_level) # Set a random seed used in ChainerRL misc.set_random_seed(args.seed, gpus=(args.gpu, )) if not (args.demo and args.load): args.outdir = experiments.prepare_output_dir(args, args.outdir) temp = args.outdir.split('/')[-1] dst = args.outdir.strip(temp) def make_env(test): env = gym.make(args.env) if test: episode_length = args.eval_episode_length else: episode_length = args.episode_length env.initialize_environment( case=args.state_rep, n_historical_events=args.n_historical_events, episode_length=episode_length, n_experts=args.n_experts, n_demos_per_expert=1, n_expert_time_steps=args.length_expert_TS, seed_agent=args.seed_agent, seed_expert=args.seed_expert, adam_days=args.adam_days) # Use different random seeds for train and test envs env_seed = 2**32 - 1 - args.seed if test else args.seed env.seed(env_seed) # Cast observations to float32 because our model uses float32 env = chainerrl.wrappers.CastObservationToFloat32(env) if args.monitor: env = gym.wrappers.Monitor(env, args.outdir) if not test: # Scale rewards (and thus returns) to a reasonable range so that # training is easier env = chainerrl.wrappers.ScaleReward(env, args.reward_scale_factor) if args.render: env = chainerrl.wrappers.Render(env) return env sample_env = gym.make(args.env) sample_env.initialize_environment( case=args.state_rep, n_historical_events=args.n_historical_events, episode_length=args.episode_length, n_experts=args.n_experts, n_demos_per_expert=1, n_expert_time_steps=args.length_expert_TS, seed_agent=args.seed_agent, seed_expert=args.seed_expert, adam_days=args.adam_days) demonstrations = sample_env.generate_expert_trajectories(out_dir=dst, eval=False) timestep_limit = None #sample_env.spec.tags.get('wrapper_config.TimeLimit.max_episode_steps') # This value is None # Generate expert data for evaluation temp_env = gym.make(args.env) temp_env.initialize_environment( case=args.state_rep, n_historical_events=args.n_historical_events, episode_length= 0, # This parameter does not really matter since we create this env only for generating samples n_experts=args.n_experts, n_demos_per_expert=1, # We do not perform any clustering right now # n_demos_per_expert=args.n_demos_per_expert, # How large should the expert cluster be? n_expert_time_steps=args. eval_episode_length, # How long should each expert trajectory be? seed_expert=args.seed_expert, adam_days=args.adam_days) temp_env.generate_expert_trajectories(out_dir=dst, eval=True) obs_space = sample_env.observation_space action_space = sample_env.action_space # Normalize observations based on their empirical mean and variance if args.state_rep == 1: obs_dim = obs_space.low.size elif args.state_rep == 2 or args.state_rep == 21 or args.state_rep == 22 or args.state_rep == 24 or args.state_rep == 4 or args.state_rep == 221 or args.state_rep == 222 \ or args.state_rep == 71 or args.state_rep == 17 or args.state_rep == 81: obs_dim = obs_space.n elif args.state_rep == 3 or args.state_rep == 11 or args.state_rep == 23 or args.state_rep == 31 or args.state_rep == 7: obs_dim = obs_space.nvec.size else: raise NotImplementedError if args.normalize_obs: obs_normalizer = chainerrl.links.EmpiricalNormalization( obs_dim, clip_threshold=5) # shape: Shape of input values except batch axis else: obs_normalizer = None # Switch policy types accordingly to action space types if args.arch == 'FFSoftmax': model = A3CFFSoftmax(obs_dim, action_space.n, hidden_sizes=args.G_layers) elif args.arch == 'FFMellowmax': model = A3CFFMellowmax(obs_space.low.size, action_space.n) elif args.arch == 'FFGaussian': model = A3CFFGaussian(obs_space.low.size, action_space, bound_mean=args.bound_mean) opt = chainer.optimizers.Adam(alpha=args.lr, eps=10e-1) opt.setup(model) if args.show_D_dummy: # Let discriminator see dummy input_dim_D = obs_dim + 1 elif not args.show_D_dummy: # Do not let discriminator see dummy if args.state_rep == 21 or args.state_rep == 17: input_dim_D = obs_dim + 1 else: input_dim_D = obs_dim + 1 - args.n_experts if args.weight_decay > 0: opt.add_hook(NonbiasWeightDecay(args.weight_decay)) if args.algo == 'ppo': agent = PPO( model, opt, obs_normalizer=obs_normalizer, gpu=args.gpu, update_interval=args.update_interval, minibatch_size=args.batchsize, epochs=args.epochs, clip_eps_vf=None, entropy_coef=args.entropy_coef, standardize_advantages=args.standardize_advantages, ) elif args.algo == 'gail': from customer_behaviour.algorithms.irl.gail import GAIL as G from customer_behaviour.algorithms.irl.gail import Discriminator as D demonstrations = np.load(dst + '/expert_trajectories.npz') D = D(gpu=args.gpu, input_dim=input_dim_D, hidden_sizes=args.D_layers, loss_type=args.loss_type) agent = G(env=sample_env, demonstrations=demonstrations, discriminator=D, model=model, optimizer=opt, obs_normalizer=obs_normalizer, gpu=args.gpu, update_interval=args.update_interval, minibatch_size=args.batchsize, epochs=args.epochs, clip_eps_vf=None, entropy_coef=args.entropy_coef, standardize_advantages=args.standardize_advantages, args=args) elif args.algo == 'airl': from customer_behaviour.algorithms.irl.airl import AIRL as G from customer_behaviour.algorithms.irl.airl import Discriminator as D # obs_normalizer = None demonstrations = np.load(dst + '/expert_trajectories.npz') D = D(gpu=args.gpu, input_dim=input_dim_D - 1, hidden_sizes=args.D_layers) # AIRL only inputs state to D agent = G(env=sample_env, demonstrations=demonstrations, discriminator=D, model=model, optimizer=opt, obs_normalizer=obs_normalizer, gpu=args.gpu, update_interval=args.update_interval, minibatch_size=args.batchsize, epochs=args.epochs, clip_eps_vf=None, entropy_coef=args.entropy_coef, standardize_advantages=args.standardize_advantages, noise=args.noise, n_experts=args.n_experts, episode_length=args.episode_length, adam_days=args.adam_days, dummy_D=args.show_D_dummy) elif args.algo == 'mmct-gail': from customer_behaviour.algorithms.irl.gail.mmct_gail import MMCTGAIL as G from customer_behaviour.algorithms.irl.gail import Discriminator as D demonstrations = np.load(dst + '/expert_trajectories.npz') D = D(gpu=args.gpu, input_dim=input_dim_D, hidden_sizes=args.D_layers, loss_type=args.loss_type) agent = G(env=sample_env, demonstrations=demonstrations, discriminator=D, model=model, optimizer=opt, obs_normalizer=obs_normalizer, gpu=args.gpu, update_interval=args.update_interval, minibatch_size=args.batchsize, epochs=args.epochs, clip_eps_vf=None, entropy_coef=args.entropy_coef, standardize_advantages=args.standardize_advantages, args=args) if args.load: # By default, not in here agent.load(args.load) if args.demo: # By default, not in here env = make_env(True) eval_stats = experiments.eval_performance( env=env, agent=agent, n_steps=None, n_episodes=args.eval_n_runs, max_episode_len=timestep_limit) print('n_runs: {} mean: {} median: {} stdev {}'.format( args.eval_n_runs, eval_stats['mean'], eval_stats['median'], eval_stats['stdev'])) outdir = args.load if args.load else args.outdir save_agent_demo(make_env(False), agent, outdir) else: # Linearly decay the learning rate to zero def lr_setter(env, agent, value): agent.optimizer.alpha = value lr_decay_hook = experiments.LinearInterpolationHook( args.steps, args.lr, 0, lr_setter) # Linearly decay the clipping parameter to zero def clip_eps_setter(env, agent, value): agent.clip_eps = max(value, 1e-8) clip_eps_decay_hook = experiments.LinearInterpolationHook( args.steps, 0.2, 0, clip_eps_setter) if train_env is None: experiments.train_agent_with_evaluation( agent=agent, env=make_env( False ), # Environment train the agent against (False -> scaled rewards) eval_env=make_env(True), # Environment used for evaluation outdir=args.outdir, steps=args. steps, # Total number of timesteps for training (args.n_training_episodes*args.episode_length) eval_n_steps= None, # Number of timesteps at each evaluation phase eval_n_episodes=args. eval_n_runs, # Number of episodes at each evaluation phase (default: 10) eval_interval=args. eval_interval, # Interval of evaluation (defualt: 10000 steps (?)) train_max_episode_len= timestep_limit, # Maximum episode length during training (is None) save_best_so_far_agent=False, step_hooks=[ lr_decay_hook, clip_eps_decay_hook, ], checkpoint_freq=args.eval_interval) else: experiments.train_agent_batch_with_evaluation( agent=agent, env=train_env, steps=args.steps, eval_n_steps=None, eval_n_episodes=args.eval_n_runs, eval_interval=args.eval_interval, outdir=args.outdir, max_episode_len=timestep_limit, eval_max_episode_len=None, eval_env=make_env(True), step_hooks=[ lr_decay_hook, clip_eps_decay_hook, ], save_best_so_far_agent=False, checkpoint_freq=args.eval_interval, log_interval=args.update_interval) save_agent_demo( make_env(True), agent, args.outdir, 10 * args.eval_episode_length ) # originally it was make_env(test=False) which seems strange # Move result files to correct folder and remove empty folder move_dir(args.outdir, dst) os.rmdir(args.outdir) if args.save_results: print('Saving result...') res2.save_data(dst, 10000, 50, N=1) print('Running evaluate policy...') ep.eval_policy(a_dir_path=dst) # else: # if args.n_experts <= 10: # print('Running evaluate policy...') # ep.eval_policy(a_dir_path=dst) # # print('Running evaluate training...') # # ets.eval_training(a_dir_path=dst) # print('Done') if args.save_report_material: print('Saving dataframe...') if args.state_rep == 21: if args.algo == 'gail': folder_name = 'gail' elif args.algo == 'airl': folder_name = 'airl' elif args.state_rep == 22: if args.algo == 'gail': folder_name = 'gail_dummies' elif args.algo == 'airl': folder_name = 'airl_dummies' elif args.state_rep == 81: if args.algo == 'gail': folder_name = 'gail_adams' elif args.algo == 'airl': folder_name = 'airl_adams' elif args.state_rep == 17: folder_name = 'ail' elif args.state_rep == 221: folder_name = 'ail_dummies' elif args.state_rep == 71: folder_name = 'ail_adams' report_material.save_df(dst, folder_name) if args.save_folder is not None: print('Saving result to ' + args.save_folder) os.makedirs(os.path.join(os.getcwd(), args.save_folder), exist_ok=True) from distutils.dir_util import copy_tree copy_tree( os.path.join(os.getcwd(), dst), os.path.join(os.getcwd(), args.save_folder, args.outdir.split('/')[-2]))
def train(dataset, learn_step=0.005, weight_decay=1e-4, num_epochs=500, max_patience=100, data_augmentation={}, savepath=None, loadpath=None, early_stop_class=None, batch_size=None, resume=False, train_from_0_255=False): # # Prepare load/save directories # exp_name = 'unet_' + 'data_aug' if bool(data_augmentation) else '' if savepath is None: raise ValueError('A saving directory must be specified') savepath = os.path.join(savepath, dataset, exp_name) # loadpath = os.path.join(loadpath, dataset, exp_name) print(savepath) # print loadpath if not os.path.exists(savepath): os.makedirs(savepath) else: print('\033[93m The following folder already exists {}. ' 'It will be overwritten in a few seconds...\033[0m'.format( savepath)) print('Saving directory : ' + savepath) with open(os.path.join(savepath, "config.txt"), "w") as f: for key, value in locals().items(): f.write('{} = {}\n'.format(key, value)) # # Define symbolic variables # input_var = T.tensor4('input_var') target_var = T.ivector('target_var') # # Build dataset iterator # if batch_size is not None: bs = batch_size else: bs = [10, 1, 1] train_iter = IsbiEmStacksDataset(which_set='train', batch_size=batch_size[0], seq_per_subset=0, seq_length=0, data_augm_kwargs=data_augmentation, return_one_hot=False, return_01c=False, overlap=0, use_threads=True, shuffle_at_each_epoch=True, return_list=True, return_0_255=False) val_iter = IsbiEmStacksDataset(which_set='val', batch_size=batch_size[1], seq_per_subset=0, seq_length=0, return_one_hot=False, return_01c=False, use_threads=True, shuffle_at_each_epoch=False, return_list=True, return_0_255=False) test_iter = None batch = train_iter.next() input_dim = (np.shape(batch[0])[2], np.shape(batch[0])[3] ) #(x,y) image shape n_batches_train = train_iter.nbatches n_batches_val = val_iter.nbatches n_batches_test = test_iter.nbatches if test_iter is not None else 0 n_classes = train_iter.non_void_nclasses void_labels = train_iter.void_labels nb_in_channels = train_iter.data_shape[0] print("Batch. train: %d, val %d, test %d" % (n_batches_train, n_batches_val, n_batches_test)) print("Nb of classes: %d" % (n_classes)) print("Nb. of input channels: %d" % (nb_in_channels)) # # Build network # net = build_UNet( n_input_channels=nb_in_channels, # BATCH_SIZE = batch_size, num_output_classes=n_classes, base_n_filters=64, do_dropout=False, input_dim=(None, None)) output_layer = net["output_flattened"] # # Define and compile theano functions # print("Defining and compiling training functions") prediction = lasagne.layers.get_output(output_layer, input_var) loss = crossentropy_metric(prediction, target_var, void_labels) if weight_decay > 0: weightsl2 = regularize_network_params(output_layer, lasagne.regularization.l2) loss += weight_decay * weightsl2 params = lasagne.layers.get_all_params(output_layer, trainable=True) updates = lasagne.updates.adam(loss, params, learning_rate=learn_step) train_fn = theano.function([input_var, target_var], loss, updates=updates) print("Defining and compiling test functions") test_prediction = lasagne.layers.get_output(output_layer, input_var, deterministic=True) test_loss = crossentropy_metric(test_prediction, target_var, void_labels) test_acc = accuracy_metric(test_prediction, target_var, void_labels) test_jacc = jaccard_metric(test_prediction, target_var, n_classes) val_fn = theano.function([input_var, target_var], [test_loss, test_acc, test_jacc]) # # Train # err_train = [] err_valid = [] acc_valid = [] jacc_valid = [] patience = 0 # Training main loop print("Start training") for epoch in range(num_epochs): # Single epoch training and validation start_time = time.time() cost_train_tot = 0 # Train print('Training steps ') for i in range(n_batches_train): print(i) # Get minibatch X_train_batch, L_train_batch = train_iter.next() L_train_batch = np.reshape(L_train_batch, np.prod(L_train_batch.shape)) # Training step cost_train = train_fn(X_train_batch, L_train_batch) out_str = "cost %f" % (cost_train) cost_train_tot += cost_train err_train += [cost_train_tot / n_batches_train] # Validation cost_val_tot = 0 acc_val_tot = 0 jacc_val_tot = np.zeros((2, n_classes)) print('Validation steps') for i in range(n_batches_val): print(i) # Get minibatch X_val_batch, L_val_batch = val_iter.next() L_val_batch = np.reshape(L_val_batch, np.prod(L_val_batch.shape)) # Validation step cost_val, acc_val, jacc_val = val_fn(X_val_batch, L_val_batch) acc_val_tot += acc_val cost_val_tot += cost_val jacc_val_tot += jacc_val err_valid += [cost_val_tot / n_batches_val] acc_valid += [acc_val_tot / n_batches_val] jacc_perclass_valid = jacc_val_tot[0, :] / jacc_val_tot[1, :] if early_stop_class == None: jacc_valid += [np.mean(jacc_perclass_valid)] else: jacc_valid += [jacc_perclass_valid[early_stop_class]] out_str = "EPOCH %i: Avg epoch training cost train %f, cost val %f" +\ ", acc val %f, jacc val class 0 % f, jacc val class 1 %f, jacc val %f took %f s" out_str = out_str % (epoch, err_train[epoch], err_valid[epoch], acc_valid[epoch], jacc_perclass_valid[0], jacc_perclass_valid[1], jacc_valid[epoch], time.time() - start_time) print(out_str) with open(os.path.join(savepath, "unet_output.log"), "a") as f: f.write(out_str + "\n") # Early stopping and saving stuff if epoch == 0: best_jacc_val = jacc_valid[epoch] elif epoch > 1 and jacc_valid[epoch] > best_jacc_val: best_jacc_val = jacc_valid[epoch] patience = 0 np.savez(os.path.join(savepath, 'new_unet_model_best.npz'), *lasagne.layers.get_all_param_values(output_layer)) np.savez(os.path.join(savepath, 'unet_errors_best.npz'), err_valid, err_train, acc_valid, jacc_valid) else: patience += 1 np.savez(os.path.join(savepath, 'new_unet_model_last.npz'), *lasagne.layers.get_all_param_values(output_layer)) np.savez(os.path.join(savepath, 'unet_errors_last.npz'), err_valid, err_train, acc_valid, jacc_valid) # Finish training if patience has expired or max nber of epochs # reached if patience == max_patience or epoch == num_epochs - 1: if test_iter is not None: # Load best model weights with np.load(os.path.join(savepath, 'new_unet_model_best.npz')) as f: param_values = [ f['arr_%d' % i] for i in range(len(f.files)) ] nlayers = len(lasagne.layers.get_all_params(output_layer)) lasagne.layers.set_all_param_values(output_layer, param_values[:nlayers]) # Test cost_test_tot = 0 acc_test_tot = 0 jacc_test_tot = np.zeros((2, n_classes)) for i in range(n_batches_test): # Get minibatch X_test_batch, L_test_batch = test_iter.next() L_test_batch = np.reshape(L_test_batch, np.prod(L_test_batch.shape)) # Test step cost_test, acc_test, jacc_test = val_fn( X_test_batch, L_test_batch) acc_test_tot += acc_test cost_test_tot += cost_test jacc_test_tot += jacc_test err_test = cost_test_tot / n_batches_test acc_test = acc_test_tot / n_batches_test jacc_test_perclass = jacc_test_tot[0, :] / jacc_test_tot[1, :] jacc_test = np.mean(jacc_test_perclass) out_str = "FINAL MODEL: err test % f, acc test %f, " +\ "jacc test class 0 %f, jacc test class 1 %f, jacc test %f" out_str = out_str % (err_test, acc_test, jacc_test_perclass[0], jacc_test_perclass[1], jacc_test) print(out_str) if savepath != loadpath: print('Copying model and other training files to {}'.format( loadpath)) copy_tree(savepath, loadpath) # End return
def split_train_val_test(self, val_fraction=0.222, test_fraction=0.1, balance=False, clip=False): '''split data into train/val and test sets, with test set by by patient and train/val by image :param val_factor - percent to be left out as test set on PATIENT level :param test_faction - percent to be left out as test set on PATIENT level :param: balance (bool) --> perform oversampling of the minority class ''' #first, get a list of all the patients patients = [patient for patient in os.listdir(self.tumors_by_pt)] #make test set on patient level, subtract these patients from others test_patients = [ patients[i] for i in random.sample( range(len(patients)), int(len(patients) * test_fraction)) ] train_val_patients = list(set(patients) - set(test_patients)) val_patients = [ train_val_patients[i] for i in random.sample(range(len(train_val_patients)), int(len(train_val_patients) * val_fraction)) ] train_patients = list(set(train_val_patients) - set(val_patients)) print('total of {} test patients'.format(len(test_patients))) print('total of {} val patients'.format(len(val_patients))) print('total of {} train patients'.format(len(train_patients))) #obtain test_tumors test_sample = self.find_sample(test_patients) val_sample = self.find_sample(val_patients) train_sample = self.find_sample(train_patients) train_val_dict = {'train': train_sample, 'val': val_sample} #set up basic file structure: #set up file structure if not os.path.exists(self.model_dev): os.mkdir(self.model_dev) for filetype in [ 'train', 'val', 'test', 'val_pt', 'all_train_images', 'all_val_images' ]: if not os.path.exists(os.path.join(self.model_dev, filetype)): os.mkdir(os.path.join(self.model_dev, filetype)) for filetype in ['train', 'val']: for PIRADS in ['PIRADS_2', 'PIRADS_3', 'PIRADS_4', 'PIRADS_5']: if not os.path.exists( os.path.join(self.model_dev, filetype, PIRADS)): os.mkdir(os.path.join(self.model_dev, filetype, PIRADS)) #first, copy all tumor-level information into a new file print("making test set") for tumor in test_sample: print(tumor) copy_tree(os.path.join(self.tumors, tumor), os.path.join(self.model_dev, 'test', tumor)) print('making val_pt set') for tumor in val_sample: copy_tree(os.path.join(self.tumors, tumor), os.path.join(self.model_dev, 'val_pt', tumor)) #next, copy all training images to an 'all folder' print('copying all train/val images to new folder') for dataset in train_val_dict.keys(): print(dataset) sample = train_val_dict[dataset] for tumor in sample: files = os.listdir(os.path.join(self.tumors, tumor)) if dataset == 'train': if clip == True: print( "length of files is {} for tumor {} before clipping" .format(len(files), tumor)) if len(files) > 3: del files[0] del files[-1] print( "length of files is {} for tumor {} after clipping" .format(len(files), tumor)) for file in files: shutil.copy2(os.path.join(self.tumors,tumor,file),\ os.path.join(self.model_dev,'all_'+dataset+'_images',file)) #split data by file for key in train_val_dict.keys(): print("making {} set".format(key)) for file in os.listdir( os.path.join(self.model_dev, 'all_' + key + '_images')): if file != 'Thumbs.db': print(file) if file.split('_')[8] == '2': shutil.copy2( os.path.join(self.model_dev, 'all_' + key + '_images', file), os.path.join(self.model_dev, key, 'PIRADS_2', file)) if file.split('_')[8] == '3': shutil.copy2( os.path.join(self.model_dev, 'all_' + key + '_images', file), os.path.join(self.model_dev, key, 'PIRADS_3', file)) if file.split('_')[8] == '4': shutil.copy2( os.path.join(self.model_dev, 'all_' + key + '_images', file), os.path.join(self.model_dev, key, 'PIRADS_4', file)) if file.split('_')[8] == '5': shutil.copy2( os.path.join(self.model_dev, 'all_' + key + '_images', file), os.path.join(self.model_dev, key, 'PIRADS_5', file)) if balance == True: self.balance_dataset(cat1='PIRADS_2', cat2='PIRADS_3', cat3='PIRADS_4', cat4='PIRADS_5')