def run_upgrade(self): nid = "" aptHasErrors = self.apt.aptHasErrors() if aptHasErrors is not None: MessageDialog(self.aptErrorText, aptHasErrors) elif self.upgradables: if self.apt.upgradablePackages: self.log.write("=================== upgradable pacages ====================", "UM.run_upgrade", "debug") self.log.write(self.createLogString(self.apt.upgradablePackages), "UM.run_upgrade", "debug") if self.apt.removedPackages: self.log.write("==================== removed packages =====================", "UM.run_upgrade", "debug") self.log.write(self.createLogString(self.apt.removedPackages), "UM.run_upgrade", "debug") if self.apt.newPackages: self.log.write("======================= new packages =======================", "UM.run_upgrade", "debug") self.log.write(self.createLogString(self.apt.newPackages), "UM.run_upgrade", "debug") if self.apt.heldbackPackages: self.log.write("=================== kept back packages =====================", "UM.run_upgrade", "debug") self.log.write(self.createLogString(self.apt.heldbackPackages), "UM.run_upgrade", "debug") if not self.quickUpdate: self.showOutput() contMsg = _("Continue installation?") if self.upgradableUM: cmd = "%s install updatemanager" % self.umglobal.settings['apt-get-string'] cmd += "; %s install %s" % (self.umglobal.settings['apt-get-string'], " ".join(self.apt.getPackageDependencies('updatemanager'))) nid = 'uminstallum' self.prepForCommand(nid) if self.quickUpdate: self.ec.run(cmd) else: self.terminal.executeCommand(cmd, nid) self.log.write("Execute command: %s (%s)" % (cmd, nid), "UM.on_btnInstall_clicked", "debug") else: msg = self.getDistUpgradeInfo() answer = True if msg != "": answer = self.showConfirmationDlg(contMsg, msg) if answer: cmd = "%s dist-upgrade" % self.umglobal.settings['apt-get-string'] #if self.umglobal.newUpd: pre = join(self.umglobal.filesDir, self.umglobal.settings['pre-upd'].replace("[VERSION]", self.umglobal.serverUpdVersion)) post = join(self.umglobal.filesDir, self.umglobal.settings['post-upd'].replace("[VERSION]", self.umglobal.serverUpdVersion)) if exists(pre): cmd = "/bin/bash %(pre)s; %(cmd)s" % { "pre": pre, "cmd": cmd } if exists(post): cmd = "%(cmd)s; /bin/bash %(post)s" % { "cmd": cmd, "post": post } nid = 'umupd' self.prepForCommand(nid) if self.quickUpdate: self.ec.run(cmd) else: self.terminal.executeCommand(cmd, nid) self.log.write("Execute command: %s (%s)" % (cmd, nid), "UM.on_btnInstall_clicked", "debug") else: if not self.quickUpdate: MessageDialog(self.btnInstall.get_label(), self.uptodateText) return nid
def results_table(path_dict): """ Return precalculated results images for subject info in `path_dict` Parameters ---------- path_dict : dict containing key 'rootdir' Returns ------- rtab : dict dict with keys given by run directories for this subject, values being a list with filenames of effect and sd images. """ # Which runs correspond to this design type? rootdir = path_dict['rootdir'] runs = filter(lambda f: isdir(pjoin(rootdir, f)), ['results_%02d' % i for i in range(1,5)] ) # Find out which contrasts have t-statistics, # storing the filenames for reading below results = {} for rundir in runs: rundir = pjoin(rootdir, rundir) for condir in listdir(rundir): for stat in ['sd', 'effect']: fname_effect = abspath(pjoin(rundir, condir, 'effect.nii')) fname_sd = abspath(pjoin(rundir, condir, 'sd.nii')) if exists(fname_effect) and exists(fname_sd): results.setdefault(condir, []).append([fname_effect, fname_sd]) return results
def valid_dir(d): # type: (Dict) -> bool dir = d['path'] if not path.exists(dir): return True if not path.isdir(dir): return False if set(['Makefile', 'make.bat']) & set(os.listdir(dir)): # type: ignore return False if d['sep']: dir = os.path.join('source', dir) if not path.exists(dir): return True if not path.isdir(dir): return False reserved_names = [ 'conf.py', d['dot'] + 'static', d['dot'] + 'templates', d['master'] + d['suffix'], ] if set(reserved_names) & set(os.listdir(dir)): # type: ignore return False return True
def setup_worker(fp, server_addr, port, counter=0, verbose=False, error_profile=None): """ This routine starts the worker. fp: fp that should be used to store intermediate data server_addr: ip address of server port: port on server to connect to counter: counts each round of denoising verbose: verbose flag error_profile: path to error profile .dat file """ if fp==None: raise ValueError, "setup_worker needs file path for worker" log_fh=None if verbose: dir = dirname(fp+".log") if not exists(dir): makedirs(dir) log_fh = open(fp+".log","a",0) #use local tmp if possible new_fp = fp if exists("/tmp"): new_fp = "/tmp/" + split(fp)[1] #set up the workers and start the loop worker = DenoiseWorker(new_fp, server_addr, port, counter=counter, log_fh=log_fh, error_profile=error_profile) #this asyncore loop will run until the server closes the connection loop()
def getdeps(self): """Download and Extract Sources""" for source in self.sources: self.log.info("") self.log.info("#####################################################") # Skip anything already extracted extractdir = abspath(join(DepSource.RootExtractDir, source.destsubdir)) if exists(extractdir): self.log.warn("Deps Subdir: " + source.destsubdir + " already exists, skipping") continue extracted = False downloaded = source.download() if downloaded == False: self.log.error("Download Failed") else: extracted = source.extract() # Remove the archive file if source.destsubdir != "atmel-asf": source.remove_archivefile() # Re-jig the directories for those that need it for source in self.sources: source.movetoparent_multiple() return # Check for ASF Sources if not exists(join(self.DepsDirectory, "atmel-asf")): self.log.warn("There was no Atmel ASF Archive file found") self.log.warn("asf is not required but you can manually download the below file for the Atmel Source") self.log.warn("http://www.atmel.com/tools/avrsoftwareframework.aspx?tab=overview") self.log.warn("So far this is only used for porting mbed to sam based mcu's") return
def calc_ppmm(fnamep='', styr=0, enyr=0, model=''): if not path.exists(RootDir + "/" + model + "/junk"): system("mkdir -p " + RootDir + "/" + model + "/junk") fnp_nodir = split(fnamep, "/")[-1] nyrs = enyr - styr + 1 for i in range(nyrs): y = styr + i fnp = OUTTEMP + "/" + model + "/junk/" + fnp_nodir + str(y) + ".nc" fnpr = RootDir + "/" + model + "/junk/" + fnp_nodir + str(y) + ".nc" fnpr = fnpr.replace('pr', 'prmm') if not path.exists(fnp): if y == enyr: print 'infile not found: ', fnp, ' ...skipping last year' break else: raise Exception('infile not found: ' + fnp) # calc pp in mm if not extist if not path.exists(fnpr): print "\n... calculating daily pp in mm for %s" % (path.basename(fnp)) txt1 = "cdo -m 1e+20 -mulc,86400 %s tmpfile" % fnp print txt1 system(txt1) txt2 = "ncatted -a units,pr,o,c,'mm/day' tmpfile" print txt2 system(txt2) txt3 = "mv tmpfile " + fnpr print txt3 system(txt3)
def _download_reference_files(conn): print('Downloading reference files') if not exists(reference_base_dir): mkdir(reference_base_dir) files = {'tree': (get_reference_fp('gg_13_8-97_otus.tree'), 'ftp://ftp.microbio.me/greengenes_release/' 'gg_13_8_otus/trees/97_otus.tree'), 'taxonomy': (get_reference_fp('gg_13_8-97_otu_taxonomy.txt'), 'ftp://ftp.microbio.me/greengenes_release/' 'gg_13_8_otus/taxonomy/97_otu_taxonomy.txt'), 'sequence': (get_reference_fp('gg_13_8-97_otus.fasta'), 'ftp://ftp.microbio.me/greengenes_release/' 'gg_13_8_otus/rep_set/97_otus.fasta')} for file_type, (local_fp, url) in viewitems(files): # Do not download the file if it exists already if exists(local_fp): print("SKIPPING %s: file already exists at %s. To " "download the file again, erase the existing file first" % (file_type, local_fp)) else: try: urlretrieve(url, local_fp) except: raise IOError("Error: Could not fetch %s file from %s" % (file_type, url)) ref = Reference.create('Greengenes', '13_8', files['sequence'][0], files['taxonomy'][0], files['tree'][0]) _insert_processed_params(conn, ref)
def snapshot(source, destination, name=None): """Snapshot one directory to another. Specify names to snapshot small, named differences.""" source = source + sep destination = destination + sep if not path.isdir(source): raise RuntimeError("source is not a directory") if path.exists(destination): if not path.isdir(destination): raise RuntimeError("destination is not a directory") if name is None: raise RuntimeError("can't snapshot base snapshot if destination exists") snapdir = path.join(destination, ".snapdir") if path.exists(path.join(source, ".snapdir")): raise RuntimeError("snapdir exists in source directory") if name is None: check_call(["rsync", "--del", "-av", source, destination]) makedirs(snapdir) else: if not path.exists(snapdir): raise RuntimeError("No snapdir in destination directory") check_call(["rsync", "--del", "-av", "--only-write-batch={}".format(path.join(snapdir, name)), source, destination])
def checkForSameFiles(self, starting=False): """ checks if same file was/is downloaded within same package :param starting: indicates that the current download is going to start :raises SkipDownload: """ pack = self.pyfile.package() for pyfile in self.core.files.cache.values(): if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder: if pyfile.status in (0, 12): #finished or downloading raise SkipDownload(pyfile.pluginname) elif pyfile.status in ( 5, 7) and starting: #a download is waiting/starting and was appenrently started before raise SkipDownload(pyfile.pluginname) download_folder = self.config['general']['download_folder'] location = save_join(download_folder, pack.folder, self.pyfile.name) if starting and self.core.config['download']['skip_existing'] and exists(location): size = os.stat(location).st_size if size >= self.pyfile.size: raise SkipDownload("File exists.") pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) if pyfile: if exists(location): raise SkipDownload(pyfile[0]) self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name)
def test_arguments(self): # only pass one argument. spider script shouldn't be created self.assertEqual(2, self.call('genspider', 'test_name')) assert not exists(join(self.proj_mod_path, 'spiders', 'test_name.py')) # pass two arguments <name> <domain>. spider script should be created self.assertEqual(0, self.call('genspider', 'test_name', 'test.com')) assert exists(join(self.proj_mod_path, 'spiders', 'test_name.py'))
def find_module(mod): '''find the .ko file for kernel module named mod. Searches the $RTE_SDK/$RTE_TARGET directory, the kernel modules directory and finally under the parent directory of the script ''' # check $RTE_SDK/$RTE_TARGET directory if 'RTE_SDK' in os.environ and 'RTE_TARGET' in os.environ: path = "%s/%s/kmod/%s.ko" % (os.environ['RTE_SDK'],\ os.environ['RTE_TARGET'], mod) if exists(path): return path # check using depmod try: depmod_out = check_output(["modinfo", "-n", mod], \ stderr=subprocess.STDOUT).lower() if "error" not in depmod_out: path = depmod_out.strip() if exists(path): return path except: # if modinfo can't find module, it fails, so continue pass # check for a copy based off current path tools_dir = dirname(abspath(sys.argv[0])) if (tools_dir.endswith("tools")): base_dir = dirname(tools_dir) find_out = check_output(["find", base_dir, "-name", mod + ".ko"]) if len(find_out) > 0: #something matched path = find_out.splitlines()[0] if exists(path): return path
def start(): """Start the CherryPy application server.""" setupdir = dirname(dirname(__file__)) curdir = os.getcwd() # First look on the command line for a desired config file, # if it's not on the command line, then look for 'setup.py' # in the current directory. If there, load configuration # from a file called 'dev.cfg'. If it's not there, the project # is probably installed and we'll look first for a file called # 'prod.cfg' in the current directory and then for a default # config file called 'default.cfg' packaged in the egg. if len(sys.argv) > 1: configfile = sys.argv[1] elif exists(join(setupdir, "setup.py")): configfile = join(setupdir, "dev.cfg") elif exists(join(curdir, "prod.cfg")): configfile = join(curdir, "prod.cfg") else: try: configfile = pkg_resources.resource_filename( pkg_resources.Requirement.parse("gordonweb"), "config/default.cfg") except pkg_resources.DistributionNotFound: raise ConfigurationError("Could not find default configuration.") turbogears.update_config(configfile=configfile, modulename="gordonweb.config") from gordonweb.controllers import Root turbogears.start_server(Root())
def checkifMCNPproject(directory,r): if not path.exists(path.join(directory, "cards")): print "\n\033[1;34mMCNPmanager error:\033[1;32m %s contains no MCNPmanager project\033[0m\n" % (directory) return r elif not path.exists(path.join(directory, "geom")): print "\n\033[1;34mMCNPmanager error:\033[1;32m %s contains no MCNPmanager project\033[0m\n" % (directory) return r
def find_pylintrc(): """search the pylint rc file and return its path if it find it, else None """ # is there a pylint rc file in the current directory ? if exists("pylintrc"): return abspath("pylintrc") if isfile("__init__.py"): curdir = abspath(os.getcwd()) while isfile(join(curdir, "__init__.py")): curdir = abspath(join(curdir, "..")) if isfile(join(curdir, "pylintrc")): return join(curdir, "pylintrc") if "PYLINTRC" in os.environ and exists(os.environ["PYLINTRC"]): pylintrc = os.environ["PYLINTRC"] else: user_home = expanduser("~") if user_home == "~" or user_home == "/root": pylintrc = ".pylintrc" else: pylintrc = join(user_home, ".pylintrc") if not isfile(pylintrc): pylintrc = join(user_home, ".config", "pylintrc") if not isfile(pylintrc): if isfile("/etc/pylintrc"): pylintrc = "/etc/pylintrc" else: pylintrc = None return pylintrc
def git_checkout(repo_path, checkout_path, ref): ''' Check out a git repository to a given reference and path. This function is assumed to be run in a lock. ''' jlogger.info('Checking out to ' + checkout_path) if not exists(checkout_path): mkdir(checkout_path) hash_file = checkout_path + '.commit-hash' commit_hash = get_ref_sha(repo_path, ref) do_checkout = True if exists(hash_file): previous_hash = open(hash_file).read().strip() if previous_hash == commit_hash: jlogger.debug('Skipping checkout to '+checkout_path) do_checkout = False if do_checkout: run_cmd(('git', '--work-tree='+checkout_path, 'checkout', ref, '--', '.'), repo_path) touch(checkout_path) with open(hash_file, 'w') as file: print >> file, commit_hash
def copy_packages(packages_names, dest, create_links=False, extra_ignores=None): """Copy python packages ``packages_names`` to ``dest``, spurious data. Copy will happen without tests, testdata, mercurial data or C extension module source with it. ``py2app`` include and exclude rules are **quite** funky, and doing this is the only reliable way to make sure we don't end up with useless stuff in our app. """ if ISWINDOWS: create_links = False if not extra_ignores: extra_ignores = [] ignore = shutil.ignore_patterns('.hg*', 'tests', 'testdata', 'modules', 'docs', 'locale', *extra_ignores) for package_name in packages_names: if op.exists(package_name): source_path = package_name else: mod = __import__(package_name) source_path = mod.__file__ if mod.__file__.endswith('__init__.py'): source_path = op.dirname(source_path) dest_name = op.basename(source_path) dest_path = op.join(dest, dest_name) if op.exists(dest_path): if op.islink(dest_path): os.unlink(dest_path) else: shutil.rmtree(dest_path) print("Copying package at {0} to {1}".format(source_path, dest_path)) if create_links: os.symlink(op.abspath(source_path), dest_path) else: if op.isdir(source_path): shutil.copytree(source_path, dest_path, ignore=ignore) else: shutil.copy(source_path, dest_path)
def test_write_file(self): sys.argv = [sys.argv[0], test_md] target_file = path.join(curdir, 'test.rst') os.remove(target_file) self.assertFalse(path.exists(target_file)) main() self.assertTrue(path.exists(target_file))
def go(self): sel = self["list"].getCurrent() if sel is None: return if len(sel[0]) <= 1: return selectedhost = sel[0][2] selectedhostname = sel[0][1] self.hostcache_file = '/etc/enigma2/' + selectedhostname.strip() + '.cache' #Path to cache directory if os_path.exists(self.hostcache_file): try: self.hostdata = load_cache(self.hostcache_file) except: print 'load cache failed' pass if sel[0][0] == 'host': # host entry selected if selectedhost in self.expanded: self.expanded.remove(selectedhost) else: self.expanded.append(selectedhost) self.updateNetworkList() elif sel[0][0] == 'nfsShare': # share entry selected self.openMountEdit(sel[0]) elif sel[0][0] == 'smbShare': # share entry selected if os_path.exists(self.hostcache_file): self.openMountEdit(sel[0]) else: self.session.openWithCallback(self.passwordQuestion, MessageBox, (_("Do you want to enter a username and password for this host?") ) )
def _get_path(path, key, name): """Helper to get a dataset path""" if path is None: # use an intelligent guess if it's not defined def_path = op.realpath(op.join(op.dirname(__file__), '..', '..', 'examples')) if get_config(key) is None: key = 'MNE_DATA' path = get_config(key, def_path) # use the same for all datasets if not op.exists(path) or not os.access(path, os.W_OK): try: os.mkdir(path) except OSError: try: logger.info('Checking for %s data in ' '"~/mne_data"...' % name) path = op.join(op.expanduser("~"), "mne_data") if not op.exists(path): logger.info("Trying to create " "'~/mne_data' in home directory") os.mkdir(path) except OSError: raise OSError("User does not have write permissions " "at '%s', try giving the path as an " "argument to data_path() where user has " "write permissions, for ex:data_path" "('/home/xyz/me2/')" % (path)) if not isinstance(path, string_types): raise ValueError('path must be a string or None') return path
def install(): ORIGIN=abspath(dirname(__file__)) for dname in os.listdir(ORIGIN): if dname.startswith('.'): continue if not isdir(dname): continue with open(join(ORIGIN, dname, '.target'), 'r') as f: target_dir = f.read().strip() if not exists(target_dir): os.makedirs(target_dir) for phile in os.listdir(join(ORIGIN, dname)): if phile.startswith('.'): continue path_from = join(ORIGIN, dname, phile) if phile.startswith('_'): phile = '.%s' % phile[1:] path_to = join(target_dir, phile) if islink(path_to): continue if exists(path_to): os.rename(path_to, "%s_" % path_to) os.symlink(path_from, path_to)
def getValue(self): service = self.source.service info = service and service.info() if not info: return -1 elif self.type == self.XRES: video_width = None if path.exists('/proc/stb/vmpeg/0/xres'): f = open('/proc/stb/vmpeg/0/xres', 'r') video_width = int(f.read(), 16) f.close() if not video_width: video_width = info.getInfo(iServiceInformation.sVideoWidth) return str(video_width) elif self.type == self.YRES: video_height = None if path.exists('/proc/stb/vmpeg/0/yres'): f = open('/proc/stb/vmpeg/0/yres', 'r') video_height = int(f.read(), 16) f.close() if not video_height: video_height = info.getInfo(iServiceInformation.sVideoHeight) return str(video_height) elif self.type == self.FRAMERATE: video_rate = None if path.exists('/proc/stb/vmpeg/0/framerate'): f = open('/proc/stb/vmpeg/0/framerate', 'r') video_rate = f.read() f.close() if not video_rate: video_rate = info.getInfo(iServiceInformation.sFrameRate) return str(video_rate) else: return -1
def resync_engine_file(): user_config = path.join(get_userhome(), '.config', 'ibus-anthy', 'engines.xml') system_config = path.join(config.PKGDATADIR, 'engine', 'default.xml') if not path.exists(user_config): return if not path.exists(system_config): os.unlink(user_config) return # path.getmtime depends on the build time rather than install time. def __get_engine_file_version(engine_file): version_str = '' dom = xml.dom.minidom.parse(engine_file) elements = dom.getElementsByTagName('version') nodes = [] if len(elements) > 0: nodes = elements[0].childNodes if len(nodes) > 0: version_str = nodes[0].data if version_str != '': version_str = version_str.strip() return version_str user_config_version = __get_engine_file_version(user_config) system_config_version = __get_engine_file_version(system_config) if system_config_version > user_config_version: import shutil shutil.copyfile(system_config, user_config)
def ensure_bootstrapped(self, target=None): if self.context.bootstrapped: return rust_root = self.config["tools"]["rust-root"] rustc_path = path.join( rust_root, "rustc", "bin", "rustc" + BIN_SUFFIX ) rustc_binary_exists = path.exists(rustc_path) base_target_path = path.join(rust_root, "rustc", "lib", "rustlib") target_exists = True if target is not None: target_path = path.join(base_target_path, target) target_exists = path.exists(target_path) if not (self.config['tools']['system-rust'] or (rustc_binary_exists and target_exists)): print("looking for rustc at %s" % (rustc_path)) Registrar.dispatch("bootstrap-rust", context=self.context, target=filter(None, [target])) cargo_path = path.join(self.config["tools"]["cargo-root"], "cargo", "bin", "cargo" + BIN_SUFFIX) cargo_binary_exists = path.exists(cargo_path) if not self.config["tools"]["system-cargo"] and not cargo_binary_exists: Registrar.dispatch("bootstrap-cargo", context=self.context) self.context.bootstrapped = True
def test_ampliconnoise_install(self): """ AmpliconNoise install looks sane.""" url = "http://qiime.org/install/install.html#ampliconnoise-install-notes" pyro_lookup_file = getenv('PYRO_LOOKUP_FILE') self.assertTrue(pyro_lookup_file is not None, "$PYRO_LOOKUP_FILE variable is not set. See %s for help." % url) self.assertTrue(exists(pyro_lookup_file), "$PYRO_LOOKUP_FILE variable is not set to an existing filepath.") seq_lookup_file = getenv('SEQ_LOOKUP_FILE') self.assertTrue(seq_lookup_file is not None, "$SEQ_LOOKUP_FILE variable is not set. See %s for help." % url) self.assertTrue(exists(seq_lookup_file), "$SEQ_LOOKUP_FILE variable is not set to an existing filepath.") self.assertTrue(which("SplitKeys.pl"), "Couldn't find SplitKeys.pl. " + "Perhaps AmpliconNoise Scripts directory isn't in $PATH?" + " See %s for help." % url) self.assertTrue(which("FCluster"), "Couldn't find FCluster. " + "Perhaps the AmpliconNoise bin directory isn't in $PATH?" + " See %s for help." % url) self.assertTrue(which("Perseus"), "Couldn't find Perseus. " + "Perhaps the AmpliconNoise bin directory isn't in $PATH?" + " See %s for help." % url)
def processMovieStep(self, movieId, movieFn, *args): movieFolder = self._getMovieFolder(movieId) movieName = basename(movieFn) if self._filterMovie(movieId, movieFn): makePath(movieFolder) createLink(movieFn, join(movieFolder, movieName)) toDelete = [movieName] if movieName.endswith('bz2'): # We assume that if compressed the name ends with .mrc.bz2 movieMrc = movieName.replace('.bz2', '') toDelete.append(movieMrc) if not exists(movieMrc): self.runJob('bzip2', '-d -f %s' % movieName, cwd=movieFolder) elif movieName.endswith('tbz'): # We assume that if compressed the name ends with .tbz movieMrc = movieName.replace('.tbz', '.mrc') toDelete.append(movieMrc) if not exists(movieMrc): self.runJob('tar', 'jxf %s' % movieName, cwd=movieFolder) else: movieMrc = movieName self.info("Processing movie: %s" % movieMrc) self._processMovie(movieId, movieMrc, movieFolder, *args) if self.cleanMovieData: print "erasing.....movieFolder: ", movieFolder os.system('rm -rf %s' % movieFolder) # cleanPath(movieFolder) else: self.info('Clean movie data DISABLED. Movie folder will remain in disk!!!')
def _xcassets(self, title, command): parser = argparse.ArgumentParser( description="Generate {} for your project".format(title)) parser.add_argument("filename", help="Path to your project or xcodeproj") parser.add_argument("image", help="Path to your initial {}.png".format(title.lower())) args = parser.parse_args(sys.argv[2:]) if not exists(args.image): print("ERROR: image path does not exists.") return filename = args.filename if not filename.endswith(".xcodeproj"): # try to find the xcodeproj from glob import glob xcodeproj = glob(join(filename, "*.xcodeproj")) if not xcodeproj: print("ERROR: Unable to find a xcodeproj in {}".format(filename)) sys.exit(1) filename = xcodeproj[0] project_name = filename.split("/")[-1].replace(".xcodeproj", "") images_xcassets = realpath(join(filename, "..", project_name, "Images.xcassets")) if not exists(images_xcassets): print("WARNING: Images.xcassets not found, creating it.") makedirs(images_xcassets) print("Images.xcassets located at {}".format(images_xcassets)) command(images_xcassets, args.image)
def main(sudo, pre=False, repos=None): photon = Photon( dict(sudo=sudo, repos=repos), config=None, meta='photon_selfupgrade.json' ) settings = photon.settings.get if settings['repos']: for repo in settings['repos']: if path.exists(repo) and path.exists(path.join(repo, '.git')): photon.git_handler(repo)._pull() else: photon.m('skipping non repo', more=dict(repo=repo)) upres = photon.m( 'attempting selfupgrade', cmdd=dict( cmd='%s pip3 install -U %s photon_core' % ( 'sudo' if sudo else '', '--pre' if pre else '' ) ), more=dict(sudo=sudo), critical=False ) if upres.get('returncode') == 0: photon.m('all went well') else: photon.m('I am dead! ' * 23, state=True)
def map_reads_indels(reference): """ :return: check unmapped reads for indels """ if exists(INDEL_file_path): logger.info("Loading INDELs from {}".format(INDEL_file_path)) indels = cPickle.load(open(INDEL_file_path, "rb")) return indels if exists(unmapped_read_file_path): logger.info("Loading Unmapped Reads from {}".format(unmapped_read_file_path)) unmapped_reads = cPickle.load(open(unmapped_read_file_path, "rb")) logger.info("Checking {} unmapped reads for insertions / deletions".format(len(unmapped_reads))) start = time.clock() for count, unmapped in enumerate(unmapped_reads): if count % 1000 == 0 and count != 0: logger.info("Mapping reads to reference. Completed {} %".format( str(100 * count / float(len(unmapped_reads)))[:5])) logger.info("Reads remaining : {0}".format(str(len(unmapped_reads) - count))) check_for_indels(unmapped[0], reference, unmapped[1], unmapped[2]) ins = get_insertions() DEL = get_deletions() logger.info("Dumping INDELs at {}".format(INDEL_file_path)) cPickle.dump((ins, DEL), open(INDEL_file_path, "wb")) return ins, DEL else: logger.info("Run STR Code first to get unmapped reads") exit()
def test(ctx, projects): """ Test the source distribution """ if not exists(dist_dir): ctx.invoke(sdist) with empty_pythonpath(): print("Creating virtualenv for test install...") if exists(test_venv_dir): shutil.rmtree(test_venv_dir) subprocess.check_call(["virtualenv", "-q", test_venv_dir]) pip = join(test_venv_dir, venv_bin, "pip") with chdir(dist_dir): for project in projects: print("Installing %s..." % project) dist = glob.glob("./%s*" % project)[0] subprocess.check_call([pip, "install", "-q", dist]) print("Running binaries...") for project in projects: for tool in tools[project]: tool = join(test_venv_dir, venv_bin, tool) print(tool) print(subprocess.check_output([tool, "--version"])) print("Virtualenv available for further testing:") print("source %s" % normpath(join(test_venv_dir, venv_bin, "activate")))
def __init__(self, session, parent): Screen.__init__(self, session, parent=parent) self["selected"] = StaticText("ViX:" + getImageVersion()) AboutText = _("Model: %s %s\n") % (getMachineBrand(), getMachineName()) if path.exists('/proc/stb/info/chipset'): chipset = open('/proc/stb/info/chipset', 'r').read() AboutText += _("Chipset: BCM%s") % chipset.replace('\n', '') + "\n" AboutText += _("Version: %s") % getImageVersion() + "\n" AboutText += _("Build: %s") % getImageBuild() + "\n" AboutText += _("Kernel: %s") % about.getKernelVersionString() + "\n" string = getDriverDate() year = string[0:4] month = string[4:6] day = string[6:8] driversdate = '-'.join((year, month, day)) AboutText += _("Drivers: %s") % driversdate + "\n" AboutText += _("Last update: %s") % getEnigmaVersionString() + "\n\n" tempinfo = "" if path.exists('/proc/stb/sensors/temp0/value'): tempinfo = open('/proc/stb/sensors/temp0/value', 'r').read() elif path.exists('/proc/stb/fp/temp_sensor'): tempinfo = open('/proc/stb/fp/temp_sensor', 'r').read() if tempinfo and int(tempinfo.replace('\n', '')) > 0: mark = str('\xc2\xb0') AboutText += _("System temperature: %s") % tempinfo.replace('\n', '') + mark + "C\n\n" self["AboutText"] = StaticText(AboutText)
# -*- coding: utf-8 -*- # @Time : 2019/7/19 11:03 # @Author : songxy # @Email : [email protected] # @File : 4.os模块.py from os import path p = path.join('D:/', 'svn', 'ResourceManage', '00密码管理') p = path.abspath(__file__) print(p, type(p)) print(path.exists(p)) print(path.abspath(p)) print(path.split(path.abspath(p))) print(path.dirname(path.abspath(p))) print(path.basename(path.abspath(p)))
def answer(): if request.method == 'GET': records = getrecords() selected = {} for ns, l in records.items(): # iterate over filenumbers if ns in selected or 'd' in l: # and filename suffix [l]ist continue # already did this filenumber elif not 'a' in l: selected[ns] = 'a' # question needs an answer elif not ('e' in l or 'o' in l): selected[ns] = 'r' # answer needs review elif 'o' in l and not 't' in l: selected[ns] = 't' # opposing review needs tie-breaker if len(selected) < 1: flash('No open questions remaining to answer or review.') return redirect(url_for('index')) chosen = choice(selected.keys()) # pick a question at random needs = selected[chosen] # type of response needed files = {} # files' contents in a suffix for suffix in records[chosen]: # iterate over the files available with open(recdir + chosen + suffix, 'r') as f: files[suffix] = sub(r'--REGISTRATION-ID:.*--$', '', f.read()) # read textual contents of each logdate = datetime.strftime(date.today(), '%Y-%m-%d') logfn = './logs/activity-' + logdate user = '******' # Username is Anonymous by default if 'token' in session: token = session['token'] tokenfilename = 'registered/' + token with open( tokenfilename, 'r' ) as f: # for getting username associated with the set token user = f.readline()[:-1] with open( logfn, 'a' ) as f: # logging username, IP addr, end-point, request type, question number, required answer log = user + ' ' + request.environ[ 'REMOTE_ADDR'] + ' answer' + ' GET' + ' ' + chosen + ' ' + needs + '\n' f.write(log) return render_template('answer.html', record=chosen, response=needs, files=files) # invoke the template elif request.method == 'POST': record = request.form['record'] # file number with zeroes response = request.form[ 'response'] # [submit button] 1 of: a,e,o,te,to answer = linkandescape(request.form['answer']) answer = answer + frameurl(request.form['iframeurl']) ### sanity-check size if response in ['te', 'to']: # tie breaker if path.exists(recdir + record + 't'): flash('Someone else just submitted that tiebreaker.') return redirect(url_for('index')) ###@@@ SLOW RACE: see below rename(recdir + record + 'o', recdir + record + 't') response = response[1] # 2nd character fn = recdir + record + response # filename ### sanity check? if path.exists(fn): # does file exist? flash('Someone else just submitted the requested response.') return redirect(url_for('index')) ###@@@ SLOW RACE: lock choice() responses below for some time? with open(fn, 'w') as f: f.write(answer + '\n') ### only add \n if not already at end? if 'token' in session: f.write('--REGISTRATION-ID:' + session['token'] + '--') logdate = datetime.strftime(date.today(), '%Y-%m-%d') logfn = './logs/activity-' + logdate user = '******' # Username is Anonymous by default if 'token' in session: token = session['token'] tokenfilename = 'registered/' + token with open( tokenfilename, 'r' ) as f: # for getting username associated with the set token user = f.readline()[:-1] with open( logfn, 'a' ) as f: # logging username, IP addr, end-point, request type, question number, given answer log = user + ' ' + request.environ[ 'REMOTE_ADDR'] + ' answer' + ' POST' + ' ' + record + ' ' + response + '\n' f.write(log) if 'token' in session: userfile = 'registered/' + session['token'] if path.exists(fn): with open(userfile, 'a') as f: f.write(fn[8:] + '\n') flash('Thank you for your response.') # displays in layout.html return redirect(url_for('index'))
q2_seq = token.texts_to_sequences(q2_l) word_indx = token.word_index dict_word_indx = word_indx with open(path + 'word_indx.json', 'w') as dfile: json.dump(dict_word_indx, dfile) #download GLOVE embeddings: # Citations: Stackoverflow.com,'http://nlp.stanford.edu/data/glove.840B.300d.zip' """ i/p: GloVe o/p: embeddings_index dict """ GLOVE_URL = 'http://nlp.stanford.edu/data/glove.840B.300d.zip' if not exists(expanduser('~/.keras/datasets/glove.840B.300d.zip')): zipfile = ZipFile(get_file('glove.840B.300d.zip', GLOVE_URL)) zipfile.extract('glove.840B.300d.txt', path=expanduser('~/.keras/datasets/')) print("Processing", 'glove.840B.300d.txt') embed_indx = {} with open(expanduser('~/.keras/datasets/glove.840B.300d.txt'), encoding='utf-8') as filee: for l in filee: val = l.split(' ') w = val[0] embed = np.asarray(val[1:], dtype='float32') embed_indx[w] = embed
from marker_detection.msg import pre_detection as pre_det_msg from marker_detection.msg import marker_detection_result as md_result from marker_detection.msg import bbox as bbox_msg ros_root = os.getcwd() FRCN_root = ros_root + "/../" CLASSES = ('__background__', 'marker') # Re-make a folder for saved images saved_img_path = ros_root + "/src/marker_detection/detected_img/" if osp.exists(saved_img_path): shutil.rmtree(saved_img_path) os.makedirs(saved_img_path) else: os.makedirs(saved_img_path) def vis_detections(im, class_name, dets, thresh=0.5): """Draw detected bounding boxes.""" inds = np.where(dets[:, -1] >= thresh)[0] # Case that nothing detected num_saved_img = len(os.listdir(saved_img_path)) if len(inds) == 0: cv2.imwrite(saved_img_path + str(num_saved_img) + ".png", im) # publish image message
def _proxy_exists(self, path): # TODO: decide either it should may be retrieved right away. # For now, as long as it is a symlink pointing to under .git/annex if exists(path): return True return lexists(path) and 'annex/objects' in str(realpath(path))
data_path = sample.data_path() event_id = {'Auditory/Left': 1, 'Auditory/Right': 2, 'Visual/Left': 3, 'Visual/Right': 4, 'Smiley': 5, 'Button': 32} raw_fname = op.join(data_path, 'MEG', 'sample', 'sample_audvis_raw.fif') events_data = op.join(data_path, 'MEG', 'sample', 'sample_audvis_raw-eve.fif') output_path = op.join(data_path, '..', 'MNE-sample-data-bids') # %% # To ensure the output path doesn't contain any leftover files from previous # tests and example runs, we simply delete it. # # .. warning:: Do not delete directories that may contain important data! # if op.exists(output_path): shutil.rmtree(output_path) # %% # # .. note:: # # ``mne-bids`` will try to infer as much information from the data as # possible to then save this data in BIDS-specific "sidecar" files. For # example the manufacturer information, which is inferred from the data file # extension. However, sometimes inferring is ambiguous (e.g., if your file # format is non-standard for the manufacturer). In these cases, MNE-BIDS does # *not* guess and you will have to update your BIDS fields manually. # # Based on our path definitions above, we read the raw data file, define # a new BIDS name for it, and then run the automatic BIDS conversion.
def save_html(self, filepath=None, overwrite=False, js_source='web', protocol='https', menu='all', scroll_behavior='pan', enable_editing=True, enable_keys=True, minified_js=True, never_ask_before_quit=False, static_site_index_json=None): """Save an HTML file containing the map. :param string filepath: The HTML file will be saved to this location. When js_source is 'local', then a new directory will be created with this name. :param Boolean overwrite: Overwrite existing files. :param string js_source: Can be one of the following: - *web* (Default) - Use JavaScript files from escher.github.io. - *local* - Use compiled JavaScript files in the local Escher installation. Works offline. To make the dependencies available to the downloaded file, a new directory will be made with the name specified by filepath. - *dev* - No longer necessary with source maps. This now gives the same behavior as 'local'. :param string protocol: The protocol can be 'http', 'https', or None which indicates a 'protocol relative URL', as in //escher.github.io. Ignored if source is local. :param string menu: Menu bar options include: - *none* - No menu or buttons. - *zoom* - Just zoom buttons. - *all* (Default) - Menu and button bar (requires Bootstrap). :param string scroll_behavior: Scroll behavior options: - *pan* - Pan the map. - *zoom* - Zoom the map. - *none* (Default) - No scroll events. :param Boolean enable_editing: Enable the map editing modes. :param Boolean enable_keys: Enable keyboard shortcuts. :param Boolean minified_js: If True, use the minified version of JavaScript and CSS files. :param number height: Height of the HTML container. :param Boolean never_ask_before_quit: Never display an alert asking if you want to leave the page. By default, this message is displayed if enable_editing is True. :param string static_site_index_json: The index, as a JSON string, for the static site. Use javascript to parse the URL options. Used for generating static pages (see static_site.py). """ filepath = expanduser(filepath) if js_source in ['local', 'dev']: if filepath is None: raise Exception('Must provide a filepath when js_source is not "web"') # make a directory directory = re.sub(r'\.html$', '', filepath) if exists(directory): if not overwrite: raise Exception('Directory already exists: %s' % directory) else: os.makedirs(directory) # add dependencies to the directory escher = get_url('escher_min' if minified_js else 'escher', 'local') builder_css = get_url('builder_css_min' if minified_js else 'builder_css', 'local') d3 = get_url('d3', 'local') favicon = get_url('favicon', 'local') if menu == 'all': boot_css = get_url('boot_css', 'local') boot_js = get_url('boot_js', 'local') jquery = get_url('jquery', 'local') boot_font_eot = get_url('boot_font_eot', 'local') boot_font_svg = get_url('boot_font_svg', 'local') boot_font_woff = get_url('boot_font_woff', 'local') boot_font_woff2 = get_url('boot_font_woff2', 'local') boot_font_ttf = get_url('boot_font_ttf', 'local') else: boot_css = boot_js = jquery = None boot_font_eot = boot_font_svg = None boot_font_woff = boot_font_woff2 = boot_font_ttf = None for path in [escher, builder_css, boot_css, boot_js, jquery, d3, favicon, boot_font_eot, boot_font_svg, boot_font_woff, boot_font_woff2, boot_font_ttf]: if path is None: continue src = join(root_directory, path) dest = join(directory, path) dest_dir = dirname(dest) if not exists(dest_dir): os.makedirs(dest_dir) shutil.copy(src, dest) filepath = join(directory, 'index.html') else: if not filepath.endswith('.html'): filepath += '.html' if exists(filepath) and not overwrite: raise Exception('File already exists: %s' % filepath) html = self._get_html(js_source=js_source, menu=menu, scroll_behavior=scroll_behavior, html_wrapper=True, enable_editing=enable_editing, enable_keys=enable_keys, minified_js=minified_js, fill_screen=True, height="100%", never_ask_before_quit=never_ask_before_quit, static_site_index_json=static_site_index_json, protocol=protocol) if filepath is not None: with open(filepath, 'wb') as f: f.write(html.encode('utf-8')) return filepath else: from tempfile import mkstemp from os import write, close os_file, filename = mkstemp(suffix=".html", text=False) # binary write(os_file, html.encode('utf-8')) close(os_file) return filename
help="Number of Frames per second.", default=15, ) parser.add_argument( "-o", "--output", dest="output_path", type=str, help="Output path for the resultant sildeshow.", default="output.mp4", ) args = parser.parse_args() flag = 1 if not path.exists(args.path_to_images): flag = "Invalid path: " + args.path_to_images if not path.isfile(music_file_path): flag = "Invalid path: " + args.path_to_music if args.transition_type not in allowed_transition_types: print("Invalid transition type.") print( "Following are the list of allowed transition types:", *allowed_transition_types, sep="\n\t\t" ) print("\n\nFor more clarity visit: https://trac.ffmpeg.org/wiki/Xfade\n\n") exit(2) try: if args.path_to_music[-3:] not in allowed_sound_formats: print("Music file format invalid.")
def initialize_VPN(stored_settings=0,save=0,area_input=None): ###load stored settings if needed and set input_needed variables to zero if settings are provided### additional_settings_needed = 1 additional_settings_list = list() if stored_settings == 1: instructions = saved_settings_check() additional_settings_needed = 0 input_needed = 0 elif area_input is not None: input_needed = 2 else: input_needed = 1 ###performing system check### opsys = platform.system() ##windows## if opsys == "Windows": print("\33[33mYou're using Windows.\n" "Performing system check...\n" "###########################\n\33[0m") #seek and set windows installation path# option_1_path = 'C:/Program Files/NordVPN' option_2_path = 'C:/Program Files (x86)/NordVPN' custom_path = str() if path.exists(option_1_path) == True: cwd_path = option_1_path elif path.exists(option_2_path) == True: cwd_path = option_2_path else: custom_path = input("\x1b[93mIt looks like you've installed NordVPN in an uncommon folder. Would you mind telling me which folder? (e.g. D:/customfolder/nordvpn)\x1b[0m") while path.exists(custom_path) == False: custom_path = input("\x1b[93mI'm sorry, but this folder doesn't exist. Please double-check your input.\x1b[0m") while os.path.isfile(custom_path+"/NordVPN.exe") == False: custom_path = input("\x1b[93mI'm sorry, but the NordVPN application is not located in this folder. Please double-check your input.\x1b[0m") cwd_path = custom_path print("NordVPN installation check: \33[92m\N{check mark}\33[0m") #check if nordvpn service is already running in the background check_service = "nordvpn-service.exe" in (p.name() for p in psutil.process_iter()) if check_service is False: raise Exception("NordVPN service hasn't been initialized, please start this service in [task manager] --> [services] and restart your script") print("NordVPN service check: \33[92m\N{check mark}\33[0m") # start NordVPN app and disconnect from VPN service if necessary# print("Opening NordVPN app and disconnecting if necessary...") open_nord_win = subprocess.Popen(["nordvpn", "-d"],shell=True,cwd=cwd_path,stdout=DEVNULL) while ("NordVPN.exe" in (p.name() for p in psutil.process_iter())) == False: time.sleep(3) open_nord_win.kill() print("NordVPN app launched: \33[92m\N{check mark}\33[0m") print("#####################################") ##linux## elif opsys == "Linux": print("\n\33[33mYou're using Linux.\n" "Performing system check...\n" "###########################\n\33[0m") #check if nordvpn is installed on linux# check_nord_linux = check_output(["nordvpn"]) if len(check_nord_linux) > 0: print("NordVPN installation check: \33[92m\N{check mark}\33[0m") else: raise Exception("NordVPN is not installed on your Linux machine.\n" "Follow instructions on shorturl.at/ioDQ2 to install the NordVpn app.") #check if user is logged in. If not, ask for credentials and log in or use credentials from stored settings if available.# check_nord_linux_acc = str(check_output(["nordvpn","account"])) if "not logged in" in check_nord_linux_acc: login_needed = 1 while login_needed == 1: login_message = input("\n\033[34mYou are not logged in. Please provide your credentials in the form of LOGIN/PASSWORD\n\033[0m") try: if instructions['credentials'] in locals(): credentials = stored_settings['credentials'] else: credentials = login_message except: credentials = login_message finally: try: login = credentials.split("/")[0] password = credentials.split("/")[1] except IndexError: error_login = input("\n\033[34mYou have provided your credentials in the wrong format. Press enter and please try again.\n" "Your input should look something like this: [email protected]/password\033[0m") else: login_needed = 0 try: login_nordvpn = check_output(["nordvpn","login","-u",login,"-p",password]) except subprocess.CalledProcessError: raise Exception("\nSorry,something went wrong while trying to log in\n") if "Welcome" in str(login_nordvpn): print("\n\n\033[34mLogin successful!\n\033[0m\n") pass else: raise Exception("\nSorry, NordVPN throws an unexpected message, namely:\n"+str(login_nordvpn)) else: print("NordVPN login check: \33[92m\N{check mark}\33[0m") #provide opportunity to execute additional settings.# settings_input_message = "\n\033[34mDo you want to execute additional settings?\033[0m" while additional_settings_needed == 1: additional_settings = input(settings_input_message+ "\n_________________________\n\n" "Press enter to continue\n" "Type 'help' for available options\n").strip() if additional_settings == "help": options_linux = pkg_resources.open_text(NordVPN_options, 'options_linux.txt').read().split('\n') for line in options_linux: print(line) additional_settings = input("").strip() additional_settings = str(additional_settings).split(" ") if len(additional_settings[0]) > 0: settings_input_message = additional_settings_linux(additional_settings) if any(re.findall(r'done|already been executed', settings_input_message,re.IGNORECASE)): additional_settings_list.append(additional_settings) else: additional_settings_needed = 0 #however, if provided, just skip the additional settings option and execute the stored settings.# if 'instructions' in locals(): try: if len(instructions['additional_settings'][0][0]) > 0: print("Executing stored additional settings....\n") for count,instruction in enumerate(instructions['additional_settings']): print("Executing stored setting #"+str(count+1)+": "+str(instruction)) additional_settings_linux(instruction) else: pass except Exception: pass else: raise Exception("I'm sorry, NordVPN switcher only works for Windows and Linux machines.") ###provide settings for VPN rotation### ##open available options and store these in a dict## areas_list = pkg_resources.open_text(NordVPN_options, 'countrylist.txt').read().split('\n') country_dict = {'countries':areas_list[0:60],'europe': areas_list[0:36], 'americas': areas_list[36:44], 'africa east india': areas_list[49:60],'asia pacific': areas_list[49:60], 'regions australia': areas_list[60:65],'regions canada': areas_list[65:68], 'regions germany': areas_list[68:70], 'regions india': areas_list[70:72], 'regions united states': areas_list[72:87],'special groups':areas_list[87:len(areas_list)]} ##provide input if needed## while input_needed > 0: if input_needed == 2: print("\nYou've entered a list of connection options. Checking list...\n") try: settings_servers = [area.lower() for area in area_input] settings_servers = ",".join(settings_servers) except TypeError: raise Exception("I expected a list here. Are you sure you've not entered a string or some other object?\n ") else: settings_servers = input("\n\033[34mI want to connect to...\n" "_________________________\n" "Type 'help' for available options\n\033[0m").strip().lower() #define help menu# if settings_servers.lower().strip() == 'help': #notation for specific servers differs between Windows and Linux.# if opsys == "Windows": notation_specific_server = " (e.g. Netherlands #742,Belgium #166)\n" else: notation_specific_server = " (e.g. nl742,be166)\n" settings_servers = input("\nOptions:\n" "##########\n" "* type 'quick' to choose quickconnect \n" "* Single country or local region (e.g.Germany)\n" "* Regions within country (e.g. regions united states')\n" "* World regions (europe/americas/africa east india/asia pacific)\n" "* Random multiple countries and/or local regions (e.g.France,Netherlands,Chicago)\n" "* Random n countries (e.g. random countries 10)\n" "* Random n countries within larger region (e.g. random countries europe 5)\n" "* Random n regions in country (e.g. random regions United States 6)\n"\ "* Specialty group name (e.g. Dedicated IP,Double VPN)\n" "* Specific list of servers"+notation_specific_server).strip().lower() #set base command according to running os# if opsys == "Windows": nordvpn_command = ["nordvpn", "-c"] if opsys == "Linux": nordvpn_command = ["nordvpn", "c"] #create sample of regions from input.# #1. if quick connect# if settings_servers == "quick": if input_needed == 1: quickconnect_check = input("\nYou are choosing for the quick connect option. Are you sure? (y/n)\n") if 'y' in quickconnect_check: sample_countries = [""] input_needed = 0 pass else: print("\nYou are choosing for the quick connect option.\n") #2. if provided specific servers. Notation differs for Windows and Linux machines, so two options are checked (first is Windows, second is Linux# elif "#" in settings_servers or re.compile(r'^[a-zA-Z]+[0-9]+').search(settings_servers.split(',')[0]) is not None: if opsys == "Windows": nordvpn_command.append("-n") sample_countries = [area.strip() for area in settings_servers.split(',')] input_needed = 0 else: #3. If connecting to some specific group of servers# if opsys == "Windows": nordvpn_command.append("-g") #3.1 if asked for random sample, pull a sample.# if "random" in settings_servers: #determine sample size# samplesize = int(re.sub("[^0-9]", "", settings_servers).strip()) #3.1.1 if asked for random regions within country (e.g. random regions from United States,Australia,...)# if "regions" in settings_servers: try: sample_countries = country_dict[re.sub("random", "", settings_servers).rstrip('0123456789.- ').lower().strip()] input_needed = 0 except: input("\n\nThere are no specific regions available in this country, please try again.\nPress enter to continue.\n") if input_needed == 2: input_needed = 1 continue if re.compile(r'[^0-9]').search(settings_servers.strip()): sample_countries = random.sample(sample_countries, samplesize) #3.1.2 if asked for random countries within larger region# elif any(re.findall(r'europe|americas|africa east india|asia pacific', settings_servers)): larger_region = country_dict[re.sub("random|countries", "", settings_servers).rstrip('0123456789.- ').lower().strip()] sample_countries = random.sample(larger_region,samplesize) input_needed = 0 #3.1.3 if asked for random countries globally# else: if re.compile(r'[^0-9]').search(settings_servers.strip()): sample_countries = random.sample(country_dict['countries'], samplesize) input_needed = 0 else: sample_countries = country_dict['countries'] input_needed = 0 #4. If asked for specific region (e.g. europe)# elif settings_servers in country_dict.keys(): sample_countries = country_dict[settings_servers] input_needed = 0 #5. If asked for specific countries or regions (e.g.netherlands)# else: #check for empty input first.# if settings_servers == "": input("\n\nYou must provide some kind of input.\nPress enter to continue and then type 'help' to view the available options.\n") if input_needed == 2: input_needed = 1 continue else: sample_countries = [area.strip() for area in settings_servers.split(',')] #take into account possible superfluous spaces# approved_regions = 0 for region in sample_countries: if region in [area.lower() for area in areas_list]: approved_regions = approved_regions + 1 pass else: input("\n\nThe region/group " + region + " is not available. Please check for spelling errors.\nPress enter to continue.\n") if input_needed == 2: input_needed = 1 continue if approved_regions == len(sample_countries): input_needed = 0 ##if user does not use preloaded settings## if "instructions" not in locals(): #1.add underscore if spaces are present on Linux os# for number,element in enumerate(sample_countries): if element.count(" ") > 0 and opsys == "Linux": sample_countries[number] = re.sub(" ","_",element) else: pass #2.create insutrctions dict object# instructions = {'opsys':opsys,'command':nordvpn_command,'settings':sample_countries} if opsys == "Windows": instructions['cwd_path'] = cwd_path if opsys == "Linux": instructions['additional_settings'] = additional_settings_list if 'credentials' in locals(): instructions['credentials'] = credentials #3.save the settings if requested into .txt file in project folder# if save == 1: print("\nSaving settings in project folder...\n") try: os.remove("settings_nordvpn.txt") except FileNotFoundError: pass instructions_write = json.dumps(instructions) f = open("settings_nordvpn.txt", "w") f.write(instructions_write) f.close() print("\nDone!\n") return instructions
def file_exist(self, file_name): if not path.exists(file_name): return f"The file '{file_name}' does not exist or is outside of the script directory." self.files_to_delete.append(file_name) return False
def save_results(self, output_dir): if self._validation_results is None: raise Exception( "No results to save. Method validate() must be run before save_results()." ) for learning_point in range( self._validation_params['n_learning_points']): all_results_list = [] all_subjects_list = [] learning_point_dir = path.join( output_dir, 'learning_split-' + str(learning_point)) for iteration in range(self._validation_params['n_iterations']): iteration_dir = path.join(learning_point_dir, 'iteration-' + str(iteration)) if not path.exists(iteration_dir): os.makedirs(iteration_dir) iteration_subjects_df = pd.DataFrame({ 'y': self._validation_results[learning_point][iteration]['y'], 'y_hat': self._validation_results[learning_point][iteration] ['y_hat'], 'y_index': self._validation_results[learning_point][iteration] ['y_index'] }) iteration_subjects_df.to_csv(path.join(iteration_dir, 'subjects.tsv'), index=False, sep='\t', encoding='utf-8') all_subjects_list.append(iteration_subjects_df) iteration_results_df = pd.DataFrame( { 'balanced_accuracy': self._validation_results[learning_point][iteration] ['evaluation']['balanced_accuracy'], 'auc': self._validation_results[learning_point][iteration] ['auc'], 'accuracy': self._validation_results[learning_point][iteration] ['evaluation']['accuracy'], 'sensitivity': self._validation_results[learning_point][iteration] ['evaluation']['sensitivity'], 'specificity': self._validation_results[learning_point][iteration] ['evaluation']['specificity'], 'ppv': self._validation_results[learning_point][iteration] ['evaluation']['ppv'], 'npv': self._validation_results[learning_point][iteration] ['evaluation']['npv'], 'train_balanced_accuracy': self._validation_results[learning_point][iteration] ['evaluation_train']['balanced_accuracy'], 'train_accuracy': self._validation_results[learning_point][iteration] ['evaluation_train']['accuracy'], 'train_sensitivity': self._validation_results[learning_point][iteration] ['evaluation_train']['sensitivity'], 'train_specificity': self._validation_results[learning_point][iteration] ['evaluation_train']['specificity'], 'train_ppv': self._validation_results[learning_point][iteration] ['evaluation_train']['ppv'], 'train_npv': self._validation_results[learning_point][iteration] ['evaluation_train']['npv'] }, index=[ 'i', ]) iteration_results_df.to_csv(path.join(iteration_dir, 'results.tsv'), index=False, sep='\t', encoding='utf-8') mean_results_df = pd.DataFrame( iteration_results_df.apply(np.nanmean).to_dict(), columns=iteration_results_df.columns, index=[ 0, ]) mean_results_df.to_csv(path.join(iteration_dir, 'mean_results.tsv'), index=False, sep='\t', encoding='utf-8') all_results_list.append(mean_results_df) all_subjects_df = pd.concat(all_subjects_list) all_subjects_df.to_csv(path.join(learning_point_dir, 'subjects.tsv'), index=False, sep='\t', encoding='utf-8') all_results_df = pd.concat(all_results_list) all_results_df.to_csv(path.join(learning_point_dir, 'results.tsv'), index=False, sep='\t', encoding='utf-8') mean_results_df = pd.DataFrame(all_results_df.apply( np.nanmean).to_dict(), columns=all_results_df.columns, index=[ 0, ]) mean_results_df.to_csv(path.join(learning_point_dir, 'mean_results.tsv'), index=False, sep='\t', encoding='utf-8')
def load(vectors_path): if not path.exists(vectors_path): raise IOError("Can't find data directory: {}".format(vectors_path)) vector_map = VectorMap(128) vector_map.load(vectors_path) return vector_map
def save_results(self, output_dir): if self._validation_results is None: raise Exception( "No results to save. Method validate() must be run before save_results()." ) all_results_list = [] all_train_subjects_list = [] all_test_subjects_list = [] for iteration in range(len(self._validation_results)): iteration_dir = path.join(output_dir, 'iteration-' + str(iteration)) if not path.exists(iteration_dir): os.makedirs(iteration_dir) iteration_train_subjects_df = pd.DataFrame({ 'iteration': iteration, 'y': self._validation_results[iteration]['y_train'], 'y_hat': self._validation_results[iteration]['y_hat_train'], 'subject_index': self._validation_results[iteration]['x_index'] }) iteration_train_subjects_df.to_csv(path.join( iteration_dir, 'train_subjects.tsv'), index=False, sep='\t', encoding='utf-8') all_train_subjects_list.append(iteration_train_subjects_df) iteration_test_subjects_df = pd.DataFrame({ 'iteration': iteration, 'y': self._validation_results[iteration]['y'], 'y_hat': self._validation_results[iteration]['y_hat'], 'subject_index': self._validation_results[iteration]['y_index'] }) iteration_test_subjects_df.to_csv(path.join( iteration_dir, 'test_subjects.tsv'), index=False, sep='\t', encoding='utf-8') all_test_subjects_list.append(iteration_test_subjects_df) iteration_results_df = pd.DataFrame( { 'balanced_accuracy': self._validation_results[iteration]['evaluation'] ['balanced_accuracy'], 'auc': self._validation_results[iteration]['auc'], 'accuracy': self._validation_results[iteration]['evaluation'] ['accuracy'], 'sensitivity': self._validation_results[iteration]['evaluation'] ['sensitivity'], 'specificity': self._validation_results[iteration]['evaluation'] ['specificity'], 'ppv': self._validation_results[iteration]['evaluation']['ppv'], 'npv': self._validation_results[iteration]['evaluation']['npv'], 'train_balanced_accuracy': self._validation_results[iteration]['evaluation_train'] ['balanced_accuracy'], 'train_accuracy': self._validation_results[iteration]['evaluation_train'] ['accuracy'], 'train_sensitivity': self._validation_results[iteration]['evaluation_train'] ['sensitivity'], 'train_specificity': self._validation_results[iteration]['evaluation_train'] ['specificity'], 'train_ppv': self._validation_results[iteration]['evaluation_train'] ['ppv'], 'train_npv': self._validation_results[iteration]['evaluation_train'] ['npv'] }, index=[ 'i', ]) iteration_results_df.to_csv(path.join(iteration_dir, 'results.tsv'), index=False, sep='\t', encoding='utf-8') # mean_results_df = pd.DataFrame(iteration_results_df.apply(np.nanmean).to_dict(), # columns=iteration_results_df.columns, index=[0, ]) # mean_results_df.to_csv(path.join(iteration_dir, 'mean_results.tsv'), # index=False, sep='\t', encoding='utf-8') all_results_list.append(iteration_results_df) all_train_subjects_df = pd.concat(all_train_subjects_list) all_train_subjects_df.to_csv(path.join(output_dir, 'train_subjects.tsv'), index=False, sep='\t', encoding='utf-8') all_test_subjects_df = pd.concat(all_test_subjects_list) all_test_subjects_df.to_csv(path.join(output_dir, 'test_subjects.tsv'), index=False, sep='\t', encoding='utf-8') all_results_df = pd.concat(all_results_list) all_results_df.to_csv(path.join(output_dir, 'results.tsv'), index=False, sep='\t', encoding='utf-8') mean_results_df = pd.DataFrame(all_results_df.apply( np.nanmean).to_dict(), columns=all_results_df.columns, index=[ 0, ]) mean_results_df.to_csv(path.join(output_dir, 'mean_results.tsv'), index=False, sep='\t', encoding='utf-8') print("Mean results of the classification:") print("Balanced accuracy: %s" % (mean_results_df['balanced_accuracy'].to_string(index=False))) print("specificity: %s" % (mean_results_df['specificity'].to_string(index=False))) print("sensitivity: %s" % (mean_results_df['sensitivity'].to_string(index=False))) print("auc: %s" % (mean_results_df['auc'].to_string(index=False)))
def save_results(self, output_dir): if self._repeated_validation_results is None: raise Exception( "No results to save. Method validate() must be run before save_results()." ) all_results_list = [] all_subjects_list = [] for iteration in range(len(self._repeated_validation_results)): iteration_dir = path.join(output_dir, 'iteration-' + str(iteration)) if not path.exists(iteration_dir): os.makedirs(iteration_dir) iteration_subjects_list = [] iteration_results_list = [] folds_dir = path.join(iteration_dir, 'folds') if not path.exists(folds_dir): os.makedirs(folds_dir) for i in range(len(self._repeated_validation_results[iteration])): subjects_df = pd.DataFrame({ 'y': self._repeated_validation_results[iteration][i]['y'], 'y_hat': self._repeated_validation_results[iteration][i]['y_hat'], 'y_index': self._repeated_validation_results[iteration][i]['y_index'] }) subjects_df.to_csv(path.join( folds_dir, 'subjects_fold-' + str(i) + '.tsv'), index=False, sep='\t', encoding='utf-8') iteration_subjects_list.append(subjects_df) results_df = pd.DataFrame( { 'balanced_accuracy': self._repeated_validation_results[iteration][i] ['evaluation']['balanced_accuracy'], 'accuracy': self._repeated_validation_results[iteration][i] ['evaluation']['accuracy'], 'train_balanced_accuracy': self._repeated_validation_results[iteration][i] ['evaluation_train']['balanced_accuracy'], 'train_accuracy': self._repeated_validation_results[iteration][i] ['evaluation_train']['accuracy'] }, index=[ 'i', ]) results_df.to_csv(path.join(folds_dir, 'results_fold-' + str(i) + '.tsv'), index=False, sep='\t', encoding='utf-8') iteration_results_list.append(results_df) iteration_subjects_df = pd.concat(iteration_subjects_list) iteration_subjects_df.to_csv(path.join(iteration_dir, 'subjects.tsv'), index=False, sep='\t', encoding='utf-8') all_subjects_list.append(iteration_subjects_df) iteration_results_df = pd.concat(iteration_results_list) iteration_results_df.to_csv(path.join(iteration_dir, 'results.tsv'), index=False, sep='\t', encoding='utf-8') mean_results_df = pd.DataFrame( iteration_results_df.apply(np.nanmean).to_dict(), columns=iteration_results_df.columns, index=[ 0, ]) mean_results_df.to_csv(path.join(iteration_dir, 'mean_results.tsv'), index=False, sep='\t', encoding='utf-8') all_results_list.append(mean_results_df) all_subjects_df = pd.concat(all_subjects_list) all_subjects_df.to_csv(path.join(output_dir, 'subjects.tsv'), index=False, sep='\t', encoding='utf-8') all_results_df = pd.concat(all_results_list) all_results_df.to_csv(path.join(output_dir, 'results.tsv'), index=False, sep='\t', encoding='utf-8') mean_results_df = pd.DataFrame(all_results_df.apply( np.nanmean).to_dict(), columns=all_results_df.columns, index=[ 0, ]) mean_results_df.to_csv(path.join(output_dir, 'mean_results.tsv'), index=False, sep='\t', encoding='utf-8') print("Mean results of the classification:") print("Balanced accuracy: %s" % (mean_results_df['balanced_accuracy'].to_string(index=False)))
def colored_image(full_path): assert osp.exists(full_path), f"File {full_path} does not exist." return cv2.imread(full_path)
def save_results(self, output_dir): if self._validation_results is None: raise Exception( "No results to save. Method validate() must be run before save_results()." ) subjects_folds = [] results_folds = [] container_dir = path.join(output_dir, 'folds') if not path.exists(container_dir): os.makedirs(container_dir) for i in range(len(self._validation_results)): subjects_df = pd.DataFrame({ 'y': self._validation_results[i]['y'], 'y_hat': self._validation_results[i]['y_hat'], 'y_index': self._validation_results[i]['y_index'] }) subjects_df.to_csv(path.join(container_dir, 'subjects_fold-' + str(i) + '.tsv'), index=False, sep='\t', encoding='utf-8') subjects_folds.append(subjects_df) results_df = pd.DataFrame( { 'balanced_accuracy': self._validation_results[i]['evaluation'] ['balanced_accuracy'], 'auc': self._validation_results[i]['auc'], 'accuracy': self._validation_results[i]['evaluation']['accuracy'], 'sensitivity': self._validation_results[i]['evaluation']['sensitivity'], 'specificity': self._validation_results[i]['evaluation']['specificity'], 'ppv': self._validation_results[i]['evaluation']['ppv'], 'npv': self._validation_results[i]['evaluation']['npv'], 'train_balanced_accuracy': self._validation_results[i]['evaluation_train'] ['balanced_accuracy'], 'train_accuracy': self._validation_results[i]['evaluation_train'] ['accuracy'], 'train_sensitivity': self._validation_results[i]['evaluation_train'] ['sensitivity'], 'train_specificity': self._validation_results[i]['evaluation_train'] ['specificity'], 'train_ppv': self._validation_results[i]['evaluation_train']['ppv'], 'train_npv': self._validation_results[i]['evaluation_train']['npv'] }, index=[ 'i', ]) results_df.to_csv(path.join(container_dir, 'results_fold-' + str(i) + '.tsv'), index=False, sep='\t', encoding='utf-8') results_folds.append(results_df) all_subjects = pd.concat(subjects_folds) all_subjects.to_csv(path.join(output_dir, 'subjects.tsv'), index=False, sep='\t', encoding='utf-8') all_results = pd.concat(results_folds) all_results.to_csv(path.join(output_dir, 'results.tsv'), index=False, sep='\t', encoding='utf-8') mean_results = pd.DataFrame(all_results.apply(np.nanmean).to_dict(), columns=all_results.columns, index=[ 0, ]) mean_results.to_csv(path.join(output_dir, 'mean_results.tsv'), index=False, sep='\t', encoding='utf-8') print("Mean results of the classification:") print("Balanced accuracy: %s" % (mean_results['balanced_accuracy'].to_string(index=False))) print("specificity: %s" % (mean_results['specificity'].to_string(index=False))) print("sensitivity: %s" % (mean_results['sensitivity'].to_string(index=False))) print("auc: %s" % (mean_results['auc'].to_string(index=False)))
from os.path import exists from setuptools import find_packages, setup NAME = find_packages(exclude=['*.tests'])[0] with open('requirements.txt') as f: install_requires = f.read().strip().split('\n') if exists('README.rst'): with open('README.rst') as f: long_description = f.read() else: long_description = '' setup( name=NAME, packages=[NAME], install_requires=install_requires, author='Haiying Xu', author_email='*****@*****.**', description='A benchmarking results publish application repository', long_description=long_description, keywords='aiohttp benchmarking web_publish ', url='http://example.com/HelloWorld/', # project home page, if any project_urls={ 'Source Code': 'https://github.com/ncar-xdev/aiohttp_app_template', 'Documentation': 'https://ncar-xdev.github.io/aiohttp_app_template', 'Bug Tracker': 'https://github.com/ncar-xdev/aiohttp_app_template/issues', },
if __name__ == '__main__': args = parse_args() with open(args.json_file) as f: ann_dict = json.load(f) print(ann_dict.keys()) out_dir = osp.join(args.output_dir, osp.splitext(osp.basename(args.json_file))[0]) if not osp.exists(out_dir): os.makedirs(out_dir, exist_ok=True) i = 0 for img_annot in tqdm(ann_dict['images']): image_name = img_annot['file_name'] print(osp.join(args.imdir, image_name)) if not osp.exists(osp.join(args.imdir, image_name)): continue image_id = img_annot['id'] bboxes = [x['bbox'] for x in ann_dict['annotations'] \ if x['image_id'] == image_id] im = cv2.imread(osp.join(args.imdir, image_name)) assert im.size > 0 im_det = draw_detection_list(im, np.array(bboxes)) out_path = osp.join(out_dir, image_name.replace('/', '_'))
triggers = {} tfile = "triggers.json" tokenf = "token.txt" ignored = [] separator = '/' admins = [1115929317] def is_recent(m): return (time() - m.date) < 60 # Check if Triggers file exists. if exists(tfile): with open(tfile) as f: triggers = json.load(f) else: # print("Triggers file not found, creating.") with open(tfile, 'w') as f: json.dump({}, f) # Check if Token file exists, if not, create. if exists(tokenf): with open(tokenf) as f: token = f.readline().rstrip('\n') # print("Token = [" + token + "]") else: # print("Token File not found, creating.") with open(tokenf, 'w') as f:
def prep_image(full_path): assert osp.exists(full_path), f"File {full_path} does not exist." return cv2.imread(full_path, cv2.IMREAD_GRAYSCALE)
def save_to_db(opts, cmp_result, tad_result, reso, inputs, richA_stats, firsts, param_hash, launch_time, finish_time): if 'tmpdb' in opts and opts.tmpdb: # check lock while path.exists(path.join(opts.workdir, '__lock_db')): time.sleep(0.5) # close lock open(path.join(opts.workdir, '__lock_db'), 'a').close() # tmp file dbfile = opts.tmpdb copyfile(path.join(opts.workdir, 'trace.db'), dbfile) else: dbfile = path.join(opts.workdir, 'trace.db') con = lite.connect(dbfile) with con: cur = con.cursor() cur.execute("""SELECT name FROM sqlite_master WHERE type='table' AND name='JOBs'""") if not cur.fetchall(): cur.execute(""" create table PATHs (Id integer primary key, JOBid int, Path text, Type text, unique (Path))""") cur.execute(""" create table JOBs (Id integer primary key, Parameters text, Launch_time text, Finish_time text, Type text, Parameters_md5 text, unique (Parameters_md5))""") cur.execute("""SELECT name FROM sqlite_master WHERE type='table' AND name='SEGMENT_OUTPUTs'""") if not cur.fetchall(): cur.execute(""" create table SEGMENT_OUTPUTs (Id integer primary key, JOBid int, Inputs text, TADs int, Compartments int, richA_corr real, EV_index int, EValue real, Chromosome text, Resolution int)""") try: parameters = digest_parameters(opts, get_md5=False, extra=['fasta']) cur.execute( """ insert into JOBs (Id , Parameters, Launch_time, Finish_time, Type , Parameters_md5) values (NULL, '%s', '%s', '%s', 'Segment', '%s') """ % (parameters, time.strftime("%d/%m/%Y %H:%M:%S", launch_time), time.strftime("%d/%m/%Y %H:%M:%S", finish_time), param_hash)) except lite.IntegrityError: pass jobid = get_jobid(cur) for ncrm, crm in enumerate( max(list(cmp_result.keys()), list(tad_result.keys()), key=len)): if crm in cmp_result: add_path(cur, cmp_result[crm]['path_cmprt1'], 'COMPARTMENT', jobid, opts.workdir) add_path(cur, cmp_result[crm]['path_cmprt2'], 'COMPARTMENT', jobid, opts.workdir) add_path(cur, cmp_result[crm]['image_cmprt'], 'FIGURE', jobid, opts.workdir) if opts.savecorr: add_path(cur, cmp_result[crm]['path_cormat'], 'CROSS_CORR_MAT', jobid, opts.workdir) if crm in tad_result: add_path(cur, tad_result[crm]['path'], 'TAD', jobid, opts.workdir) if opts.rich_in_A: add_path(cur, opts.rich_in_A, 'BED', jobid, opts.workdir) if crm in firsts: evalue = firsts[crm][0][(opts.ev_index[ncrm] - 1) if opts.ev_index else 0] eindex = opts.ev_index[ncrm] if opts.ev_index else 1 else: evalue = 'NULL' eindex = 'NULL' try: cur.execute(""" insert into SEGMENT_OUTPUTs (Id , JOBid, Inputs, TADs, Compartments, richA_corr, EV_index, EValue, Chromosome, Resolution) values (NULL, %d, '%s', %s, %s, %s, %s, %s, '%s', %d) """ % (jobid, ','.join([str(i) for i in inputs]), tad_result[crm]['num'] if crm in tad_result else 'NULL', cmp_result[crm]['num'] if crm in cmp_result else 'NULL', (richA_stats[crm] if crm in richA_stats and richA_stats[crm] is not None else 'NULL'), eindex, evalue, crm, reso)) except lite.OperationalError: # TODO: remove this print_exc() try: cur.execute( "alter table SEGMENT_OUTPUTs add column 'richA_corr' 'real'" ) except: pass try: cur.execute( "alter table SEGMENT_OUTPUTs add column 'EValue' 'real'" ) except: pass try: cur.execute( "alter table SEGMENT_OUTPUTs add column 'EV_index', 'int'" ) except: pass cur.execute(""" insert into SEGMENT_OUTPUTs (Id , JOBid, Inputs, TADs, Compartments, richA_corr, EV_index, EValue, Chromosome, Resolution) values (NULL, %d, '%s', %d, %d, %s, %s, %s, '%s', %d) """ % (jobid, ','.join([str(i) for i in inputs]), tad_result[crm]['num'] if crm in tad_result else 0, cmp_result[crm]['num'] if crm in cmp_result else 0, (richA_stats[crm] if crm in richA_stats and richA_stats[crm] is not None else 'NULL'), eindex, evalue, crm, reso)) print_db(cur, 'PATHs') print_db(cur, 'JOBs') print_db(cur, 'SEGMENT_OUTPUTs') if 'tmpdb' in opts and opts.tmpdb: # copy back file copyfile(dbfile, path.join(opts.workdir, 'trace.db')) remove(dbfile) # release lock remove(path.join(opts.workdir, '__lock_db'))
def source_code(m): cid = m.chat.id if exists(__file__): bot.send_document(cid, open(__file__, 'rb')) else: bot.reply_to(m, "No source file found :x")
exit(1) if not os.path.exists(file) or not os.path.isfile(file): print 'File,', file, 'is not a valid file' exit(2) if name is None: print "Give a descriptive name for the data using -n '<descriptive name>'" exit(3) print 'Parsing file,', file fileName = os.path.split(file)[1] if fileName == "spotters.dat": workFile = "/tmp/spotters.dat" if path.exists(workFile) and path.isfile(workFile) and access( workFile, R_OK): print "Attempting to cleanup work directory" os.system("rm /tmp/spotters.dat") else: print "No preliminary cleanup needed - continuing" shutil.copy(file, workFile) os.system("sed -i -e 's/spotterName/spottersName/g' /tmp/spotters.dat") os.system("sed -i -e 's/spotterAddr/spottersAddress/g' /tmp/spotters.dat") os.system("sed -i -e 's/spotterCity/spottersCity/g' /tmp/spotters.dat") os.system("sed -i -e 's/spotterPhone/spottersPhone/g' /tmp/spotters.dat") file = workFile if exportFileName is None: exportFileName = fileName
def main(): warnings.warn("This script is aimed to demonstrate how to convert the\n" "JSON file to a single image dataset, and not to handle\n" "multiple JSON files to generate a real-use dataset.") parser = argparse.ArgumentParser() parser.add_argument('json_file') parser.add_argument('-o', '--out', default=None) args = parser.parse_args() json_file = args.json_file alist = os.listdir(json_file) for i in range(0, len(alist)): path = os.path.join(json_file, alist[i]) data = json.load(open(path)) out_dir = osp.basename(path).replace('.', '_') out_dir = osp.join(osp.dirname(path), out_dir) if not osp.exists(out_dir): os.mkdir(out_dir) if data['imageData']: imageData = data['imageData'] else: imagePath = os.path.join(os.path.dirname(path), data['imagePath']) with open(imagePath, 'rb') as f: imageData = f.read() imageData = base64.b64encode(imageData).decode('utf-8') img = utils.img_b64_to_arr(imageData) label_name_to_value = {'_background_': 0} for shape in data['shapes']: label_name = shape['label'] if label_name in label_name_to_value: label_value = label_name_to_value[label_name] else: label_value = len(label_name_to_value) label_name_to_value[label_name] = label_value # label_values must be dense label_values, label_names = [], [] for ln, lv in sorted(label_name_to_value.items(), key=lambda x: x[1]): label_values.append(lv) label_names.append(ln) assert label_values == list(range(len(label_values))) lbl = utils.shapes_to_label(img.shape, data['shapes'], label_name_to_value) captions = ['{}: {}'.format(lv, ln) for ln, lv in label_name_to_value.items()] lbl_viz = utils.draw_label(lbl, img, captions) PIL.Image.fromarray(img).save(osp.join(out_dir, 'img.png')) utils.lblsave(osp.join(out_dir, 'label.png'), lbl) PIL.Image.fromarray(lbl_viz).save(osp.join(out_dir, 'label_viz.png')) with open(osp.join(out_dir, 'label_names.txt'), 'w') as f: for lbl_name in label_names: f.write(lbl_name + '\n') warnings.warn('info.yaml is being replaced by label_names.txt') info = dict(label_names=label_names) with open(osp.join(out_dir, 'info.yaml'), 'w') as f: yaml.safe_dump(info, f, default_flow_style=False) print('Saved to: %s' % out_dir)
def auto_parse_schema(location): """ Attempts to automatically parse the file at location (if local) and extract a schema. This function will only parse files that have an extension in AUTO_PARSE_EXTS and are smaller than AUTO_PARSE_THRESHOLD bytes. The function returns a rendered YAML array if the schema was parsed successfully, None otherwise. TODO: The parsing is done by Pandas (using the various pd.read_* functions), and the schema is just a stringification of DataFrame.dtypes, which has several downsides (e.g. string columns have dtype 'object'). Returns: str -- A rendered YAML containing the schema, suitable for inclusion in the template. """ if not path.exists(location): return None if not check_pandas_installed(): click.secho("Pandas is not installed, skipping parsing.", fg="yellow") return None _, ext = path.splitext(location.lower()) if ext not in AUTO_PARSE_EXTS: click.secho("Unrecognized extension {}, skipping parsing.".format(ext), fg="yellow") size = path.getsize(location) if size > AUTO_PARSE_THRESHOLD: click.secho("File is large, skipping parsing.") return None import pandas as pd try: if ext == ".csv": df = pd.read_csv(location, parse_dates=True, nrows=10) elif ext == ".tsv": df = pd.read_csv(location, delimiter='\t', parse_dates=True, nrows=10) elif ext == ".json": df = pd.read_json(location) elif ext == ".parquet": df = pd.read_parquet(location) elif ext == ".xls" or ext == ".xlsx": df = pd.read_excel(location) schema = [ "- {}: {} [No description]".format(field, str(kind)) for field, kind in df.dtypes.to_dict().items() ] click.secho( "{} fields automatically parsed. Please check schema for accuracy." .format(len(schema))) return "\n".join(schema) except Exception as e: print(e) click.secho("Pandas could not parse this file, skipping parsing.") return None
def train(self, nb_steps=30000, verbose=1, visualize=False, log_interval=3000): if self.__istrained: raise RuntimeError('このモデルは既に訓練済みです。') print('訓練を行うので、お待ちください。') # 訓練実施 # Okay, now it's time to learn something! # We visualize the training here for show, but this slows down training quite a lot. # You can always safely abort the training prematurely using Ctrl + C. callbacks = [] if verbose == 1: self.train_interval_logger = TrainIntervalLogger2( interval=log_interval) callbacks.append(self.train_interval_logger) verbose = 0 elif verbose > 1: callbacks.append(TrainEpisodeLogger()) verbose = 0 hist = self.dqn.fit(self.env, nb_steps=nb_steps, callbacks=callbacks, verbose=verbose, visualize=visualize, log_interval=log_interval) self.__istrained = True if self.train_interval_logger is not None: # 訓練状況の可視化 interval = self.train_interval_logger.records['interval'] episode_reward = self.train_interval_logger.records[ 'episode_reward'] mean_q = self.train_interval_logger.records['mean_q'] if len(interval) > len(mean_q): mean_q = np.pad(mean_q, [len(interval) - len(mean_q), 0], "constant") plt.figure() plt.plot(interval, episode_reward, marker='.', label='報酬') plt.plot(interval, mean_q, marker='.', label='Q値') plt.legend(loc='best', fontsize=10) plt.grid() plt.xlabel('interval') plt.ylabel('score') plt.title('訓練状況') plt.xticks( np.arange(min(interval), max(interval) + 1, (max(interval) - min(interval)) // 7)) plt.show() # 重みの保存 if not exists(self.__class__.weightdir): try: mkdir(self.__class__.weightdir) except: print('重み保存フォルダの作成中にエラーが発生しました。') print('Unexpected error:', exc_info()[0]) raise try: # After training is done, we save the final weights. self.dqn.save_weights(self.weightfile, overwrite=True) except: print('重みの保存中にエラーが発生しました。') print('Unexpected error:', exc_info()[0]) raise return hist
def plot_interpolate_volspread(stock_index,ticker,date,**kwargs): """Merge all buy volumes, all sell volumes and interpolate results in order to be able to define the buy/sell volume spread. Returns are moving-averaged. """ vol_signal, tstamps = split_buy_sell_vols(stock_index,ticker,date) vol_sbuy = pd.DataFrame(columns=['Time','Vol Strong Buy']) vol_wbuy = pd.DataFrame(columns = ['Time','Vol Buy']) vol_mspbuy = pd.DataFrame(columns = ['Time','Vol Mid Spread Buy']) vol_mspsell = pd.DataFrame(columns = ['Time','Vol Mid Spread Sell']) vol_wsell = pd.DataFrame(columns = ['Time','Vol Sell']) vol_ssell = pd.DataFrame(columns = ['Time','Vol Strong Sell']) a = vol_signal.loc[vol_signal['Signal']==3][['Time','Vol']] vol_sbuy = pd.DataFrame({'Time': a['Time'].values, \ 'Vol Strong Buy': a['Vol'].values}) a = vol_signal.loc[vol_signal['Signal']==2][['Time','Vol']] vol_wbuy = pd.DataFrame({'Time': a['Time'].values, \ 'Vol Weak Buy': a['Vol'].values}) a = vol_signal.loc[vol_signal['Signal']==1][['Time','Vol']] vol_mspbuy = pd.DataFrame({'Time': a['Time'].values, \ 'Vol Mid Spread Buy': a['Vol'].values}) a = vol_signal.loc[vol_signal['Signal']==-1][['Time','Vol']] vol_mspsell = pd.DataFrame({'Time': a['Time'].values, \ 'Vol Mid Spread Sell': a['Vol'].values}) a = vol_signal.loc[vol_signal['Signal']==-2][['Time','Vol']] vol_wsell = pd.DataFrame({'Time': a['Time'].values, \ 'Vol Weak Sell': a['Vol'].values}) a = vol_signal.loc[vol_signal['Signal']==-3][['Time','Vol']] vol_ssell = pd.DataFrame({'Time': a['Time'].values, \ 'Vol Strong Sell': a['Vol'].values}) a = vol_signal.loc[vol_signal['Signal'].isin([3,2,1])][['Time','Vol']] vol_buy = pd.DataFrame({'Time': a['Time'].values, \ 'Vol Buy': a['Vol'].values}) a = vol_signal.loc[vol_signal['Signal'].isin([-3,-2,-1])][['Time','Vol']] vol_sell = pd.DataFrame({'Time': a['Time'].values, \ 'Vol Sell': a['Vol'].values}) # Interpolate buy and sell volumes to have trading events at common times vol_buy_interp = vol_interpolate(vol_buy,vol_sell,'buy').fillna(0) vol_sell_interp = vol_interpolate(vol_sell,vol_buy,'sell').fillna(0) # Volume spread Vol_spread = vol_buy_interp['Vol_buy'] - vol_sell_interp['Vol_sell'] vol_buy_interp['Vol_spread'] = Vol_spread # Set the format for Time: tseries (default) or timestamps intt2stamp = lambda df, ts: pd.Series(ts.loc[ts['Unnamed: 0'].isin(df['Time'].\ astype(int))]['Time'].values) if kwargs['tformat'] == 'tstamps': vol_buy_interp['Time'] = tstamps['Time'] vol_sell_interp['Time'] = tstamps['Time'] vol_wbuy['Time'] = intt2stamp(vol_wbuy,tstamps) vol_wsell['Time'] = intt2stamp(vol_wsell,tstamps) vol_sbuy['Time'] = intt2stamp(vol_sbuy,tstamps) vol_ssell['Time'] = intt2stamp(vol_ssell,tstamps) vol_mspbuy['Time'] = intt2stamp(vol_mspbuy,tstamps) vol_mspsell['Time'] = intt2stamp(vol_mspsell,tstamps) ax=vol_wbuy.plot(x='Time',y='Vol Weak Buy',marker='o',linestyle='dashed',\ color=(0.4,0.7,1),title='Buy and Sell Volumes for '+ ticker,legend=False) vol_wsell.plot(ax=ax,x='Time',y='Vol Weak Sell',marker='o',\ linestyle='dashed',color=(1,0.6,0.6),legend=False) if not vol_sbuy.empty: vol_sbuy.plot(ax=ax,x='Time',y='Vol Strong Buy',style='ob',legend=False) if not vol_ssell.empty: vol_ssell.plot(ax=ax,x='Time',y='Vol Strong Sell',style='or',legend=False) if not vol_mspbuy.empty: vol_mspbuy.plot(ax=ax,x='Time',y='Vol Mid Spread Buy',style='og',legend=False) if not vol_mspsell.empty: vol_mspsell.plot(ax=ax,x='Time',y='Vol Mid Spread Sell',style='og',legend=False) dir_save = img_dir +'vol_evolution/'+ date +'/' if not exists(dir_save): mkdir(dir_save) fig = ax.get_figure() fig.set_size_inches(8, 6) fig.savefig(dir_save+ticker+'.png',bbox_inches='tight',dpi=100) plt.close(fig)
def may_make_dir(path): if path in [None, '']: return if not osp.exists(path): os.makedirs(path)
def split_buy_sell_vols(stock_index,ticker,date): """ Build buy/sell volume signals and stock returns. Buy/Sell volumes are divided into the following categories: ----------------------------------------------------------- Strong Buy: 3, Buy: 2, Mid Spread Buy: 1, Mid Spread Sell: -1, Sell: -2, Strong Sell: -3. """ input_dir = data_dir + stock_index + '/' + date + '/' df = pd.read_csv(input_dir + 'order_book/' + ticker +'.csv') # Volume of signal vol_signal = pd.DataFrame(columns=['Time','Signal','Vol']) vol_buy = 0 vol_sell = 0 for k in range(df.shape[0]): bid, ask = df['Bid'][k], df['Ask'][k] last_price = df['Last Price'][k] last_size = df['Last Size'][k] t = df['Unnamed: 0'][k] if last_price == bid: # At the bid vol_sell += last_size vol_signal.loc[vol_signal.shape[0]] = [t,-3,vol_sell] elif last_price == ask: # At the ask vol_buy += last_size vol_signal.loc[vol_signal.shape[0]] = [t,3,vol_buy] elif abs(last_price-bid) < abs(last_price-ask): # Near the bid vol_sell += last_size vol_signal.loc[vol_signal.shape[0]] = [t,-2,vol_sell] elif abs(last_price-bid) > abs(last_price-ask): # Near the ask vol_buy += last_size vol_signal.loc[vol_signal.shape[0]] = [t,2,vol_buy] else: # When at the middle of bid-ask spread or (no ask | no bid) # Randomly assign a buy (1) or sell (0) signal if np.random.randint(2) == 1: vol_buy += last_size vol_signal.loc[vol_signal.shape[0]] = [t,1,vol_buy] else: vol_sell += last_size vol_signal.loc[vol_signal.shape[0]] = [t,-1,vol_sell] # Normalize volumes with respect to float shares fundm = input_dir + 'fundm_' + date + '.csv' if not exists(fundm): #sf.request_fundamentals(stock_index) sf.yahoof_key_statistics(stock_index) # Float from billions to millions Float = 1e9*pd.read_csv(fundm,index_col='Ticker').loc[ticker]['Float'] #Float = pd.read_csv(fundm,index_col='Ticker').loc[ticker]['Float'] # Express results in basis points (*10^4) bp = 1e4 vol_signal['Vol'] = bp * vol_signal['Vol']/Float # Time stamps of trading events tstamps = pd.concat([df['Unnamed: 0'],pd.to_datetime(df['Time']).dt.time],axis=1) return vol_signal, tstamps
new_folder = join(folder, file_title) new_path = join(new_folder, new_filename) makedirs(new_folder, exist_ok=True) copyfile(path, new_path) with open( join(new_folder, '.presentation'), mode='w', encoding="utf8") as f: dump({'Title': title, 'Author': author}, f) shell_call('git add "{}"'.format(new_folder)) return title, author if __name__ == '__main__': if not (exists('_tools') and isdir('_tools')): print("Run this from the CppCon{} root.".format(CPPCON_YEAR)) exit(1) TITLE = None AUTHOR = None if len(sys.argv) == 2 and sys.argv[1]: TITLE, AUTHOR = add_presentation(sys.argv[1]) with open('README.md', mode='w', encoding="utf8") as readme: make_readme(readme) shell_call('git add README.md') if TITLE and AUTHOR: shell_call('git commit -v -m "Add {} by {}" -e'.format(TITLE, AUTHOR)) else: