def generate_pipeline_hash(self, inputs, dependencies=[], filenames=True): """ Returns a hash representing this pipeline combined with its inputs """ from .glob import glob hasher = md5() join = lambda it: (y for x in it for y in x) expanded_inputs = list(join([ glob(x) for x in inputs])) for inp in sorted(expanded_inputs): if filenames: u = str(os.path.getmtime(inp)) hasher.update(u) else: hasher.update(inp) for dep in dependencies: # Update hasher for every file in dependencies for f in sorted(glob(dep)) if not os.path.isfile(dep) else [dep]: u = str(os.path.getmtime(f)) hasher.update(u) return hasher.hexdigest()
def compileWithAsh(self, tests): start_time = datetime.today() #print("starting compile of %d tests at %s" % (len(tests),start_time)) total=len(tests) if not pexpect: for test in tests: self.js_print('%d\tcompiling %s' % (total,test)) self.compile_test(test) (testdir, ext) = splitext(test) if exists(testdir+".abc")==False: print("ERROR abc files %s.abc not created" % (testdir)) self.ashErrors.append("abc files %s.abc not created" % (testdir)) total -= 1; else: #pexpect available child = pexpect.spawn("java -classpath %s macromedia.asc.embedding.Shell" % self.asc) child.logfile = None child.expect("\(ash\)") child.expect("\(ash\)") for test in tests: if self.debug: print cmd else: print "Compiling ", test if test.endswith(self.abcasmExt): self.compile_test(test) else: arglist = self.parseArgStringToList(self.ascargs) (dir, file) = split(test) # look for .asc_args files to specify dir / file level compile args arglist = self.loadAscArgs(arglist, dir, test) cmd = "asc -import %s " % (self.builtinabc) for arg in arglist: cmd += ' %s' % arg for p in self.parents(dir): shell = join(p,"shell.as") if isfile(shell): cmd += " -in " + shell break (testdir, ext) = splitext(test) deps = glob(join(testdir,"*.as")) deps.sort() for util in deps + glob(join(dir,"*Util.as")): cmd += " -in %s" % util #no need to prepend \ to $ when using ash cmd += " %s" % test if exists(testdir+".abc"): os.unlink(testdir+".abc") child.sendline(cmd) child.expect("\(ash\)") if not exists(testdir+".abc"): print("ERROR: abc file %s.abc not created, cmd used to compile: %s" % (testdir,cmd)) self.ashErrors.append("abc file %s.abc not created, cmd used to compile: %s" % (testdir,cmd)) total -= 1; #print("%d remaining, %s" % (total,cmd)) end_time = datetime.today()
def guess_language_from_files(path): from glob import glob with cd(path) as _: if len(glob('*.[ch]')) > 0: return 'c' elif len(glob('*.cxx') + glob('*.hxx')) > 0: return 'cxx' else: return None
def guess_language_from_files(path): from glob import glob with cd(path) as _: if len(glob("*.[ch]")) > 0: return "c" elif len(glob("*.cxx") + glob("*.hxx")) > 0: return "cxx" else: return None
def CR_bounds(): """ Cramer-Rao bounds for signal to noise of 100 spectra """ filenames = glob("./gradient*npz") + glob("./Gradient_Spectra_elem/*npz") for fn in filenames: data = np.load(fn) gradient_value = data["gradient_value"] crb = 1.0/np.sqrt(np.sum(gradient_value**2 * 1.e4) ) print crb, fn, crb
def build_incfiles(self, as_file): files=[] (dir, file) = split(as_file) for p in self.parents(dir): shell = join(p,'shell'+self.sourceExt) if isfile(shell): files.append(shell) (testdir, ext) = splitext(as_file) if not self.eval: for util in glob(join(testdir,'*'+self.sourceExt)) + glob(join(dir,'*Util'+self.sourceExt)): files.append(string.replace(util, "$", "\$")) return files
def can_folderise(folder): """ Check if corpus can be put into folders """ import os from glob import glob if os.path.isfile(folder): return False fs = glob(os.path.join(folder, '*.txt')) if len(fs) > 1: if not any(os.path.isdir(x) for x in glob(os.path.join(folder, '*'))): return True return False
def PDB_ITR(): PDB_DIRS = sorted(os.listdir("pdb")) for pdb_dir in PDB_DIRS: os.system("mkdir -p tmscores/{}".format(pdb_dir)) FILES = glob(os.path.join("pdb", pdb_dir, "*.pdb")) for f_pdb in sorted(FILES): yield f_pdb
def step2b(mincol=15): fraclist = np.array([1,0.2,0.02,0.005,0.4,2.5]) modellist = ['/d/monk/eigenbrot/WIYN/14B-0456/anal/models/bc03_{}_ChabIMF.fits'.format(i) for i in ['solarZ','004Z','0004Z','0001Z','008Z','05Z']] for p in range(6): basename = 'NGC_891_P{}'.format(p+1) for z in range(fraclist.size): name = glob('{}*_Z{:04}.dat'.format(basename,int(fraclist[z]*1000)))[0] print name try: tmp[z] = np.loadtxt(name) except UnboundLocalError: data = np.loadtxt(name) tmp = np.zeros((fraclist.size,data.shape[0],data.shape[1])) tmp[z] = data outfile = 'P{}_models.dat'.format(p+1) f = open(outfile,'w') bdx = np.argmin(tmp[:,:,mincol],axis=0) for i in range(tmp.shape[1]): f.write('{:5.3f} {:}\n'.format(fraclist[bdx[i]], modellist[bdx[i]])) f.close() del tmp return
def step2(): fraclist = np.array([1,0.2,0.02,0.005,0.4,2.5]) for p in range(6): basename = 'NGC_891_P{}'.format(p+1) for z in range(fraclist.size): name = glob('{}*_Z{:04}.dat'.format(basename,int(fraclist[z]*1000)))[0] print name try: tmp[z] = np.loadtxt(name) except UnboundLocalError: data = np.loadtxt(name) tmp = np.zeros((fraclist.size,data.shape[0],data.shape[1])) tmp[z] = data bdx = np.argmin(tmp[:,:,15],axis=0) h = open(name,'r') head = h.readlines()[4] outname = '{}_fit.dat'.format(name.split('_Z')[0]) f = open(outname,'w') f.write('# Generated on {}\n'.format(time.asctime())) f.write(head) for i in range(tmp.shape[1]): tmp[bdx[i],i,19] = fraclist[bdx[i]] f.write(str('{:11n}'+12*'{:13.3e}'+'{:7.2f}{:12.3f}'+4*'{:12.3e}'+'{:10.3f}'+'\n').format(*tmp[bdx[i],i,:])) f.close() h.close() del tmp return
def get_clubs(): data = [] datae=[] data = glob("Data/clubs/*.txt") for i in range (len(data)): a=[] t=0.0 b=0.0 c="" path = data[i].replace('\\',"/") infile = open(path,'r') info = infile.read().replace(" ","") info = info.replace('\n',',').split(",") a.append(data[i].replace("Data/clubs\\","").replace(".txt","")) a.append(info[0]) a.append(info[2]+" "+info[3]) num = (len(info)-1)/4 for i in range(num): c=info[(i*4)+4] c=c.split("/") t+=int(c[0]) b+=int(c[1]) a.append((round((t/b)*100.0,2))) a.append((info[4].split("/"))[1]) datae.append(a) infile.close() #print datae return datae
def _get_setup_modules( options, isfile = os.path.isfile, isdir = os.path.isdir, path_join = os.path.join, glob = glob.glob, _setup_modules = _setup_modules ): global _setup_path setup_path = options.setup_path.GetList() if setup_path == _setup_path: return _setup_modules setup_modules = [] for p in setup_path: if isfile( p ): if p not in setup_modules: setup_modules.append( p ) elif isdir( p ): for m in glob( path_join( p, '*.py') ): if m not in setup_modules: setup_modules.append( m ) _setup_path = setup_path return setup_modules
def read_red(): data = [] datae=[] data = glob("Data/students/*.txt") ##print data for i in range (len(data)): a=[] t=0.0 b=0.0 c="" path = data[i].replace('\\',"/") infile = open(path,'r') info = infile.read().replace(" ","") info = info.replace('\n',',').split(",") a.append(data[i].replace("Data/students\\","").replace(".txt","")) a.append(info[0]) a.append(info[1]) a.append(info[2]) num = ((len(info))-3)/2 for i in range(num): c=info[(i*2)+4] c=c.split("/") t+=int(c[0]) b+=int(c[1]) perc = (round((t/b)*100)) a.append(perc) if perc <= 70: datae.append(a) else: None infile.close() return datae
def backupFiles(taskFile, glob=glob.glob): root, ext = os.path.splitext(taskFile.filename()) datePattern = '[0-9]'*8 timePattern = '[0-9]'*6 files = glob('%s.%s-%s.tsk.bak'%(root, datePattern, timePattern)) files.sort() return files
def get_serials_info(): """.""" serials_info = {'ports': []} for port, desc, hwid in comports(): if port: serials_info['ports'].append(port) info = {'port': port, 'description': desc, 'hwid': hwid} vid = '' pid = '' if hwid: hwid_infos = hwid.split() for hwid_info in hwid_infos: if hwid_info.startswith('VID') and '=' in hwid_info: vid_pid = hwid_info.split('=')[-1] if ':' in vid_pid: vid, pid = vid_pid.split(':') vid = '0x' + vid.strip() pid = '0x' + pid.strip() break info['vid'] = vid info['pid'] = pid serials_info[port] = info os_name = sys_info.get_os_name() if not serials_info['ports'] and os_name == 'osx': for port in glob('/dev/tty.*'): serials_info['ports'].append(port) info = {'port': port, 'description': '', 'hwid': ''} info['vid'] = '' info['pid'] = '' serials_info[port] = info return serials_info
def getFiles(basePath, app, fileName): if app == 'gw': Index, file = getLastIndexFile(basePath, fileName) Files = file return Index, file allAppDirs = [f for f in glob(os.path.join(basePath, app+'*')) if os.path.isdir(f)] folderRegEx = re.compile(".*%s\d+" % app) indexs = [] for i in range(0, len(allAppDirs)): if not re.match(folderRegEx, allAppDirs[i]): indexs.append(i) for i in range(0, len(indexs)): allAppDirs.pop(i) if len(allAppDirs) == 0: raise Exception, 'Can\'t found folders %s under %s.' % (app, basePath) allAppDirs.sort(sortDirs) index = 1 Files = '' if len(allAppDirs) == 1: Index, file = getLastIndexFile(allAppDirs[0]+'/conf/', fileName) Files = file return Index, Files elif len(allAppDirs) > 1: for index in range(0, len(allAppDirs) - 1): Index, file = getLastIndexFile(allAppDirs[index]+'/conf/', fileName) if index == 0: retIndex = Index Files += file + ', ' Index, file = getLastIndexFile(allAppDirs[-1]+'/conf/', fileName) Files += file return retIndex, Files
def import_all_from_folder(path, excludes=[]): import os import sys from importlib import import_module from glob import glob modules = {} base_path = os.path.dirname(path) if not os.path.exists(base_path): raise ImportError("Cannot import modules from %s. Path does not exist." % base_path) add_path = base_path not in sys.path if add_path: sys.path.append(base_path) for file_path in glob('%s/*.py' % base_path): dir_name, file_name = os.path.split(file_path) module_name = file_name[:-3] modules[module_name] = import_module(module_name) if add_path: sys.path.remove(base_path) return modules
def process_images(self, docname, doctree): """ Process and rewrite image URIs. """ docdir = path.dirname(self.doc2path(docname, base=None)) for node in doctree.traverse(nodes.image): # Map the mimetype to the corresponding image. The writer may # choose the best image from these candidates. The special key * is # set if there is only single candidate to be used by a writer. # The special key ? is set for nonlocal URIs. node['candidates'] = candidates = {} imguri = node['uri'] if imguri.find('://') != -1: self.warn(docname, 'nonlocal image URI found: %s' % imguri, node.line) candidates['?'] = imguri continue # imgpath is the image path *from srcdir* if imguri.startswith('/') or imguri.startswith(os.sep): # absolute path (= relative to srcdir) imgpath = path.normpath(imguri[1:]) else: imgpath = path.normpath(path.join(docdir, imguri)) # set imgpath as default URI node['uri'] = imgpath if imgpath.endswith(os.extsep + '*'): for filename in glob(path.join(self.srcdir, imgpath)): new_imgpath = relative_path(self.srcdir, filename) if filename.lower().endswith('.pdf'): candidates['application/pdf'] = new_imgpath elif filename.lower().endswith('.svg'): candidates['image/svg+xml'] = new_imgpath elif ".latex." in filename.lower(): candidates['latex'] = new_imgpath elif ".html." in filename.lower(): candidates['html'] = new_imgpath else: try: f = open(filename, 'rb') try: imgtype = imghdr.what(f) finally: f.close() except (OSError, IOError), err: self.warn(docname, 'image file %s not ' 'readable: %s' % (filename, err), node.line) if imgtype: candidates['image/' + imgtype] = new_imgpath else: candidates['*'] = imgpath # map image paths to unique image names (so that they can be put # into a single directory) for imgpath in candidates.itervalues(): self.dependencies.setdefault(docname, set()).add(imgpath) if not os.access(path.join(self.srcdir, imgpath), os.R_OK): self.warn(docname, 'image file not readable: %s' % imgpath, node.line) continue self.images.add_file(docname, imgpath)
def processDatadir(datadir, stu, bindir=None, dryrun=True): bestzip is None zips = glob(os.path.join(datadir, 'zips', '*tabblock*zip')) for zname in zips: if betterShapefileZip(zname, bestzip): bestzip = zname linksname = os.path.join(datadir, stu + '.links') mppb_name = os.path.join(datadir, stu + '.mppb') mask_name = os.path.join(datadir, stu + 'mask.png') mppbsm_name = os.path.join(datadir, stu + '_sm.mppb') masksm_name = os.path.join(datadir, stu + 'mask_sm.png') args1 = [] if not os.path.exists(linksname): args1 += ['--links', linksname] if not os.path.exists(mppb_name): args1 += ['--rast', mppb_name, '--mask', mask_name, '--boundx', '1920', '--boundy', '1080'] if args1: command = makeCommand(args1, bindir) if dryrun: print ' '.join(command) else: subprocess.Popen(command, shell=False, stdin=None) if not os.path.exists(mppbsm_name): args2 = ['--rast', mppbsm_name, '--mask', masksm_name, '--boundx', '640', '--boundy', '480', bestzip]
def output_module_html(webpage_path): """Output an HTML page for each module""" icons_relpath = os.path.relpath(cellprofiler.icons.__path__[0]) all_png_icons = glob(os.path.join(icons_relpath, "*.png")) icon_names = [os.path.basename(f)[:-4] for f in all_png_icons] help_text = """ <h2>Help for CellProfiler Modules</a></h2> <ul>\n""" d = {} module_path = webpage_path if not (os.path.exists(module_path) and os.path.isdir(module_path)): try: os.mkdir(module_path) except IOError: raise ValueError("Could not create directory %s" % module_path) for module_name in sorted(cellprofiler.modules.get_module_names()): module = cellprofiler.modules.instantiate_module(module_name) location = os.path.split( module.create_settings.im_func.func_code.co_filename)[0] if location == cellprofiler.preferences.get_plugin_directory(): continue if isinstance(module.category, (str, unicode)): module.category = [module.category] for category in module.category: if not d.has_key(category): d[category] = {} d[category][module_name] = module result = module.get_help() if result is None: continue result = result.replace('<body><h1>', '<body><h1>Module: ') # Replace refs to icons in memory with the relative path to the image dir (see above) result = re.sub("memory:", os.path.join("images", "").encode('string-escape'), result) # Check if a corresponding image exists for the module if module_name in icon_names: # Strip out end html tags so I can add more stuff result = result.replace('</body>', '').replace('</html>', '') # Include images specific to the module, relative to html files ('images' dir) LOCATION_MODULE_IMAGES = os.path.join('images', '%s.png' % module_name) result += '\n\n<div><p><img src="%s", width="50%%"></p></div>\n' % LOCATION_MODULE_IMAGES # Now end the help text result += '</body></html>' fd = open(os.path.join(module_path, "%s.html" % module_name), "w") fd.write(result) fd.close() for category in sorted(d.keys()): sub_d = d[category] help_text += "<li><b>%s</b><br><ul>\n" % category for module_name in sorted(sub_d.keys()): help_text += "<li><a href='%s.html'>%s</a></li>\n" % (module_name, module_name) help_text += "</ul></li>\n" help_text += "</ul>\n" return help_text
def from_ganga(job_id): import os from glob import glob prefix = "$WORKDIR/gangadir/workspace/albarano/LocalXML/" pattern = prefix + str(job_id) + "/*/output/*.root" return glob(os.path.expandvars(pattern))
def get_index(pointing, data, datfile=None): if datfile is None: datfile = glob("{}/anal/indecies/NGC*P{}*dfkmid3*bands.dat".format(basepath, pointing))[0] print "Getting index info from {}".format(datfile) galdata = np.loadtxt( datfile, usecols=(0, 1, 5, 6), dtype={"names": ("aps", "bands", "index", "eqwidth"), "formats": ("S50", "S11", "f4", "f4")}, ) aps = np.unique(galdata["aps"]) sar = [int(s.split("(")[-1].split(")")[0]) for s in aps] sidx = np.argsort(sar) aps = aps[sidx] sar = np.array(sar)[sidx] for i in range(aps.size): if sar[i] != data["ap"][i] + 1: print "WARNING: index ap {} does not match data ap {}".format(sar[i], data["ap"][i]) raw_input("") idx = np.where(galdata["aps"] == aps[i]) bands = bcc.eat_index(galdata["eqwidth"][idx]) data["Hb"][i] = bands[0] data["HdA"][i] = bands[1] data["HgA"][i] = bands[2] data["HdF"][i] = bands[3] data["HgF"][i] = bands[4] data["Fe"][i] = bands[5] data["MgFe"][i] = bands[6] return data
def get_library_location(package): # get abs path of a package in the library, rather than locally library_package_paths = glob(os.path.join(get_path('platlib'), '*')) sys.path = library_package_paths + sys.path package_path = get_loader(package).filename sys.path = sys.path[len(library_package_paths):] return package_path
def clean_flist(raw_flist,s=False,lc=False): if not raw_flist: if (s or STRICT): raise Exception, "No file argument provided." else: perror("No file argument provided.") err("No file argument provided.") if type(raw_flist) != type(list()): raw_flist = [raw_flist] full_flist = [] clean_list = [] for f in raw_flist: full_flist.extend(glob(expand(f))) for entry in full_flist: valid = validate_path(entry) if exists(entry) and valid: clean_list.append(entry) elif not valid: warning("Invalid path: [%s] - skipping" % entry) else: warning("[%s] does not exist - skipping" % entry) if not clean_list: if (s or STRICT): raise Exception,"No such file or directory: %s" % raw_flist else: warn("No such file or directory: [%s]" % raw_flist) return clean_list
def make_movie_file(self): """ Create subdirectory based on casename, move all plot frame files to this directory, and generate an index.html for viewing the movie in a browser (as a sequence of PNG files). """ # Make HTML movie in a subdirectory directory = self.casename if os.path.isdir(directory): shutil.rmtree(directory) # rm -rf directory os.mkdir(directory) # mkdir directory # mv frame_*.png directory for filename in glob('frame_*.png'): os.rename(filename, os.path.join(directory, filename)) os.chdir(directory) # cd directory self.plt.movie('frame_*.png', encoder='html', output_file='index.html', fps=4) # Make other movie formats: Flash, Webm, Ogg, MP4 codec2ext = dict(flv='flv', libx64='mp4', libvpx='webm', libtheora='ogg') filespec = 'frame_%04d.png' movie_program = 'avconv' # or 'ffmpeg' for codec in codec2ext: ext = codec2ext[codec] cmd = '%(movie_program)s -r %(fps)d -i %(filespec)s '\ '-vcodec %(codec)s movie.%(ext)s' % vars() os.system(cmd) os.chdir(os.pardir) # move back to parent directory
def explosion(x, y): for file in glob("graphics/explosion_1/*.png"): img = image.load(file).convert_alpha() img = transform.scale(img, (200, 200)) for _ in range(10): screen.blit(img, (x - 80, y - 80)) display.update()
def list_serial_ports(): """.""" serial_ports = [port for port, d, h in comports() if port] os_name = sys_info.get_os_name() if not serial_ports and os_name == "osx": for port in glob("/dev/tty.*"): serial_ports.append(port) return serial_ports
def glob_tree(roots, patterns, exclude_patterns=None): """Recursive version of GLOB. Builds the glob of files while also searching in the subdirectories of the given roots. An optional set of exclusion patterns will filter out the matching entries from the result. The exclusions also apply to the subdirectory scanning, such that directories that match the exclusion patterns will not be searched.""" if not exclude_patterns: exclude_patterns = [] result = glob(roots, patterns, exclude_patterns) subdirs = [s for s in glob(roots, ["*"]) if s != "." and s != ".." and os.path.isdir(s)] if subdirs: result.extend(glob_tree(subdirs, patterns, exclude_patterns)) return result
def processDir(srcDir, target): allFiles = glob(srcDir + sep + "*") debug("dir = " + srcDir + ", target = " + target) for aFile in allFiles: if isfile(aFile): debug("is file = " + aFile) if aFile.endswith(".jar") or aFile.endswith(".pom") or aFile.endswith(".asc") or aFile.endswith(".war"): copy2(aFile, target)
def connect(): global s patt = '/dev/ttyUSB*' for port in glob(patt): s = Serial(port, baudrate, timeout=timeout) break else: raise ValueError("mmM board not found: no serial port")
def _read_data(data_path, channel): class_list = os.listdir(data_path) class_list.sort() n_classes = len(class_list) image_list = [] image_holder = [] label_list = [] for i in range(n_classes): img_class = glob(os.path.join(data_path, class_list[i]) + '/*.*') image_list += img_class for j in range(len(img_class)): label_list += [i] if channel == "1": flags = 0 else: flags = 1 length_data = len(image_list) for j in range(length_data): img = cv2.imread(image_list[j], flags=flags) if channel == 1: img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) img = cv2.resize(img, (32, 32)) img = img.astype(np.float32) img = np.reshape(img, [1, 32, 32, channel]) else: img = img.resize(img, (32, 32)) img = img.astype(np.float32) image_holder.append(img) image_holder = np.concatenate(image_holder, axis=0) label_holder = np.asarray(label_list, dtype=np.int32) idx = np.random.permutation(length_data) images = image_holder[idx] labels = label_holder[idx] ## preprocessing mean = np.mean(images, axis=(0, 1, 2)) std = np.std(images, axis=(0, 1, 2)) images = (images - mean) / std return images, labels
def update_hadoop_env(self): print("\nChecking hadoop-env.sh") try: if argv.hadoop_path: conf_base_path = argv.hadoop_path else: conf_base_path = glob( '/opt/mapr/hadoop/hadoop-*.*.*/etc/hadoop/')[-1] hadoop_env_sh = os.path.join(conf_base_path, 'hadoop-env.sh') preunravel_hadoop_env = os.path.join(conf_base_path, 'hadoop-env.sh.preunravel') if not os.path.exists(preunravel_hadoop_env) and not argv.dry_test: print("Backup original hadoop-env.sh") copyfile(hadoop_env_sh, preunravel_hadoop_env) with open(hadoop_env_sh, 'r') as f: content = f.read() f.close() if self.do_hive and self.configs['hadoop-env'].split( ':')[1] in content: print("{0} {1:>{width}}".format("HADOOP_CLASSPATH", "No change needed", width=80 - len('HADOOP_CLASSPATH'))) if argv.verbose: print_verbose(self.configs['hadoop-env']) elif self.do_hive: print("{0} {1:>{width}}".format("HADOOP_CLASSPATH", "Missing value", width=80 - len('HADOOP_CLASSPATH'))) if argv.verbose: print_verbose('None', self.configs['hadoop-env']) if not argv.dry_test: with open(hadoop_env_sh, 'a') as f: print("appending HADOOP_CLASSPATH") f.write('\n' + self.configs['hadoop-env']) f.close() else: print("Skip hadoop-env.sh") except Exception as e: print("Error: " + str(e)) self.do_hive = False
def par_crop(instance_size=511, num_threads=24, image_folder='data_folder', save_folder='siamrpn++_model'): dataDir = '.' crop_path = join(save_folder, 'crop{:d}'.format(instance_size)) if not isdir(crop_path): makedirs(crop_path) videos = [ x.split('/')[-2] for x in sorted(glob(join(image_folder, '*', '*.csv'))) ] n_videos = len(videos) with futures.ProcessPoolExecutor(max_workers=num_threads) as executor: fs = [ executor.submit(crop_video, video, image_folder, crop_path, instance_size) for video in videos ] for i, f in enumerate(futures.as_completed(fs)): printProgress(i, n_videos, suffix='Done ', barLength=40)
def add_posterior_predictive_data(resultdir, stats='hist'): """Generate posterior predictive data for all subjects in result and store in results file. """ subjects = [ int(os.path.basename(f)[1:3]) for f in glob(os.path.join(resultdir, '*%s.log' % stats)) ] for sub in subjects: print('\rProcessing subject %2d ...' % sub) # generate posterior predictive data try: choices_post, rts_post, data, model = ( generate_posterior_predictive_data(resultdir, sub, stats, 'r+')) except IOError: warnings.warn("Skipping subject {} due to IOError!".format(sub))
def event_list(startdate, enddate): event_list = [] reapath = os.path.join(SEISAN_DATA, 'REA', DB) years = list(range(startdate.year, enddate.year + 1)) for year in years: if year == enddate.year and year == startdate.year: months = list(range(startdate.month, enddate.month + 1)) elif year == startdate.year: months = list(range(startdate.month, 13)) elif year == enddate.year: months = list(range(1, enddate.month + 1)) else: months = list(range(1, 13)) for month in months: #print month dir = os.path.join(reapath, "%04d" % year, "%02d" % month) flist = glob(os.path.join(dir, "*L.S*")) event_list.extend(flist) return event_list
def glob_tree(roots, patterns, exclude_patterns=None): """Recursive version of GLOB. Builds the glob of files while also searching in the subdirectories of the given roots. An optional set of exclusion patterns will filter out the matching entries from the result. The exclusions also apply to the subdirectory scanning, such that directories that match the exclusion patterns will not be searched.""" if not exclude_patterns: exclude_patterns = [] result = glob(roots, patterns, exclude_patterns) subdirs = [ s for s in result if s != "." and s != ".." and os.path.isdir(s) ] if subdirs: result.extend(glob_tree(subdirs, patterns, exclude_patterns)) return result
def expand_globs(path_list, root_dir): files = [] for path in path_list: if not os.path.isabs(path): path = os.path.join(root_dir, path) if os.path.islink(path): files.append(path.replace(root_dir + os.path.sep, '')) elif os.path.isdir(path): files.extend(os.path.join(root, f).replace(root_dir + os.path.sep, '') for root, _, fs in os.walk(path) for f in fs) elif os.path.isfile(path): files.append(path.replace(root_dir + os.path.sep, '')) else: # File compared to the globs use / as separator indenpendently of the os glob_files = [f.replace(root_dir + os.path.sep, '') for f in glob(path)] files.extend(glob_files) files = [f.replace(os.path.sep, '/') for f in files] return files
def load_profile(self, widget): self.loadProfile_window = gtk.Window() self.loadProfile_window.set_border_width(1) self.loadProfile_window.set_default_size(330, 200) self.loadProfile_window.set_title("Load Profile") self.loadProfile_window.set_position(gtk.WIN_POS_CENTER) loadFixed = gtk.Fixed() self.loadProfile_window.add(loadFixed) # Scrollable treeview for displaying existing profiles self.loadList = gtk.ListStore(str, str) self.loadTreeview = gtk.TreeView(self.loadList) # Item Column column_item = gtk.TreeViewColumn() cell_icon = gtk.CellRendererPixbuf() cell_icon.set_property('cell-background', 'light blue') column_item.pack_start(cell_icon, False) column_item.set_attributes(cell_icon, stock_id=0) cell_item = gtk.CellRendererText() cell_item.set_property('cell-background', 'white') column_item.pack_start(cell_item, True) column_item.set_attributes(cell_item, text=1) #Set up treeview self.loadTreeview.append_column(column_item) self.loadTreeview.set_search_column(0) self.loadTreeview.set_size_request(300, 120) # Embed treeview in scrollable window sw = gtk.ScrolledWindow() sw.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_AUTOMATIC) sw.add(self.loadTreeview) loadFixed.put(sw, 15, 20) for f in glob("*.profile"): self.loadList.append([gtk.STOCK_FILE, str(f).split(".")[0]]) confirmButton = gtk.Button("Confirm") confirmButton.connect("clicked", self.confirmLoad) loadFixed.put(confirmButton, 100, 160) cancelButton = gtk.Button("Cancel") cancelButton.connect("clicked", self.cancelLoad) loadFixed.put(cancelButton, 190, 160) self.loadProfile_window.show_all()
def merge(path, blank_filename, output_filename): blank = PdfFileReader(file(blank_filename, "rb")) output = PdfFileWriter() for pdffile in glob('*.pdf'): if pdffile == output_filename: continue print("Parse '%s'" % pdffile) document = PdfFileReader(open(pdffile, 'rb')) for i in range(document.getNumPages()): output.addPage(document.getPage(i)) if document.getNumPages() % 2 == 1: output.addPage(blank.getPage(0)) print("Add blank page to '%s' (had %i pages)" % (pdffile, document.getNumPages())) print("Start writing '%s'" % output_filename) output_stream = file(output_filename, "wb") output.write(output_stream) output_stream.close()
def __init__(self, dataset, load_size, channels, augment_flag, transforms=None): "Initialize variables" self.load_size = load_size self.channels = channels self.augment_flag = augment_flag if transforms: self.transforms = transforms.Compose(transforms) else: self.transforms = None # Check if dataset exists check_exists(dataset) self.dataset_name = dataset self.train = glob("./dataset/{}/*.*".format(self.dataset_name))
def write_sessions_tsv(bids_dir, sessions_dict): """Create <participant_id>_sessions.tsv files. Write the content of the function create scans dict in several TSV files following the BIDS specification. Args: bids_dir: path to the bids directory sessions_dict: output of the function create_sessions_dict """ import os from glob import glob from os import path import pandas as pd bids_paths = glob(path.join(bids_dir, "sub-*")) for sp in bids_paths: bids_id = sp.split(os.sep)[-1] if bids_id in sessions_dict: session_df = pd.DataFrame( sessions_dict[bids_id]["M00"], index=[ "i", ], ) cols = session_df.columns.tolist() cols = cols[-1:] + cols[:-1] session_df = session_df[cols] else: print(f"No session data available for {sp}") session_df = pd.DataFrame(columns=["session_id"]) session_df["session_id"] = pd.Series("M00") session_df = session_df.fillna("n/a") session_df.to_csv( path.join(sp, bids_id + "_sessions.tsv"), sep="\t", index=False, encoding="utf8", )
def teste(): from glob import glob import os #import cv2 #pngs = glob('./**/*.png', recursive=True) pngs = glob('/home/users/lucas/DataMining/Yolo/darknet/data/*.jpg') os.chdir("/home/users/lucas/DataMining/Yolo/darknet/") os.system("./darknet detect cfg/yolov3.cfg yolov3.weights") for j in pngs: #img = cv2.imread(j) #cv2.imwrite(j[:-3] + 'jpg', img) #os.remove(j) #os.system(j) #print j #os.system("chmod +x "+j) os.system(j) print "IMG OK"
def execute(): message, result = show_entry_fields() print(result) print(message) if result == True: vbasename = basename.get() vsequence = str(sequence.get()) vdestname = destname.get() for i in vsequence.split(" "): if VERS == "NICOS": os.chdir(vdestname) filename = "%s%08d" % (vbasename, int(i)) # command="ln -s %s%08d.dat ." %(vbasename,int(i)) # os.system(command) # print( "command:",command command = "ln -s ../../%s%08d.dat ." % ( vbasename[vbasename.rfind("/data/") + 1:], int(i)) # print( "command neu:",command os.system(command) globname = "%s*_%08d_*" % (vbasename, int(i)) for item in glob(globname): #command="ln -s %s ." %(item) #os.system(command) #print( "command glob:",command command = "ln -s ../../%s ." % (item[item.rfind("/data/") + 1:]) os.system(command) # print( "command glob neu:",command else: filename = "%s%s" % (vbasename, i) rsync = "rsync -av %s/ %s" % (filename, vdestname) print("execute", rsync) os.system(rsync) command = "cd %s;%s" % (vdestname, message) print("executing command:", command) print("executing system(%s)" % (message)) os.system(command) print("finished") return
def check_multi_nodule(): pd_annotation = pd.read_csv(anno_path) count_image = 0 seriesuid_temp = None slice_list = [] multi_list = [] for each_annotation in pd_annotation.iterrows(): seriesuid = each_annotation[1].seriesuid coord_x = each_annotation[1].coordX coord_y = each_annotation[1].coordY coord_z = each_annotation[1].coordZ diameter_mm = each_annotation[1].diameter_mm mhd_name = '{}.mhd'.format(seriesuid) mhd_path = glob(os.path.join(working_path, '*', mhd_name), recursive=True)[0] numpyImage, numpyOrigin, numpySpacing = load_itk_image( mhd_path) # numpyImage.shape) 维度为(slice,w,h) # 将世界坐标下肺结节标注转换为真实坐标系下的坐标标注 worldCoord = np.asarray( [float(coord_x), float(coord_y), float(coord_z)]) voxelCoord = worldToVoxelCoord(worldCoord, numpyOrigin, numpySpacing) slice = int(voxelCoord[2] + 0.5) if seriesuid_temp == seriesuid: if slice in slice_list: print('multi nodules in one image') multi_list.append((seriesuid_temp, slice)) slice_list.append(slice) else: seriesuid_temp = seriesuid slice_list.clear() slice_list.append(slice) print(count_image) count_image += 1 print(multi_list)
def predict_separate(path, test_path=config.TEST_PATH, dat_file=config.SHAPE_PREDICTOR_DAT_PATH): print('Loading Model from: ', path) name_of_model = path.split('_')[1] model = load_model(path) images_processed = 0 images_with_no_face = 0 detector = dlib.get_frontal_face_detector() predictor = dlib.shape_predictor(dat_file) y_predProbs = np.empty((0, 7)) y_true = [] for label in config.CLASS_LABEL: files = sorted(glob(os.path.join(test_path, label.lower()) + '/*.jpg')) for images in files: img = cv2.resize(cv2.imread(images), (config.TARGET_SIZE['CV2_LANDMARKS_RESIZE'][0], config.TARGET_SIZE['CV2_LANDMARKS_RESIZE'][1])) img = cv2.cvtColor(src=img, code=cv2.COLOR_BGR2GRAY) faces = detector(img) if len(faces) == 0: images_with_no_face += 1 y_predProbs = np.vstack( (y_predProbs, np.zeros(config.NUM_CLASSES))) else: row = [] for face in faces: landmarks = predictor(image=img, box=face) for n in range(0, 68): row.append(landmarks.part(n).x) row.append(landmarks.part(n).y) break y_predProbs = np.vstack( (y_predProbs, model.predict(np.array(row).reshape(1, -1), verbose=1))) y_true.append(label) images_processed += 1 print("No of Images Processed:", images_processed) print('\n\nTotal Images:', images_processed, '\nFace detected in: ', images_processed - images_with_no_face, 'images\nFace not detected in: ', images_with_no_face, 'images') return y_predProbs, y_true
def task_progress(self): self.__progress_files = [] self.__progress_users = [] self.__progress_tasks = [] self.__path = str(self.__user.get_org()).strip() + '//' self.__progress_window = Tk() self.__progress_window.title("Company Name Task Progress List") self.__progress_title = Label(self.__progress_window, text = "Task List",\ font = ("Arial Bold", 25)) self.__progress_title.grid(column=2, row=0) count = 0 count2 = 0 for file in glob(path.join(self.__path, '*.txt')): self.__progress_files.append(FileHandler(file)) self.__progress_users.append( self.__progress_files[count].openFile()) count += 1 for x in range(len(self.__progress_users)): for y in range(len(self.__progress_users[x].get_tasks())): self.__progress_task_text = (str(self.__progress_users[x].get_name()).strip() \ + '(' + str(self.__progress_users[x].get_team()).strip() + ')' \ + ': ' + str(self.__progress_users[x].get_task(y).get_objective()) \ + ' | ' + str(self.__progress_users[x].get_task(y).get_urgency()) \ + ' | ' + str(self.__progress_users[x].get_task(y).get_date_due()) \ + ' | ' + str(self.__progress_users[x].get_task(y).get_status())) if (str(self.__progress_users[x].get_task( y).get_reported()).strip() == "True"): self.__progress_task_text += ' | Reported Completed' elif (str(self.__progress_users[x].get_task( y).get_reported()).strip() == "False"): self.__progress_task_text += ' | Not Reported Completed' self.__progress_tasks.append( Label(self.__progress_window, text=self.__progress_task_text)) self.__progress_tasks[count2].grid(column=x, row=y + 1) count2 += 1 size = 300 * count self.__progress_window.geometry(str(size) + 'x300')
def addOutputShouldBeToSolutions(): from glob import glob import subprocess, re for tdir in solutionDirs: dirmsg = tdir + "\n" + '=' * len(tdir) with visitDir(os.path.join(ROOT_DIR, tdir)): for sfile in glob("Solution-*.scala"): msg = sfile with open(sfile) as f: solution = f.read() if "OUTPUT_SHOULD" not in solution: if dirmsg: print dirmsg dirmsg = None if msg: print msg msg = None with open(sfile, 'w') as f: f.write(solution + output_should_be) subprocess.call([SUBLIME, sfile])
def readAllSeqLabelFigure(folderpath): file_paths = glob(folderpath + "*.tsv") all_texts = [] all_labels = [] all_figures = [] all_figure_appearance = [] str_seq_lens = [] for file_name in file_paths: str_seqs, label_seqs, figure_spans, figure_appearances, sentenceNums = readSeqLabelFigure( file_name) str_seqs, label_seqs, figure_spans, figure_appearances = clause2sentence( str_seqs, label_seqs, figure_spans, figure_appearances, sentenceNums) all_texts.extend(str_seqs) all_labels.extend(label_seqs) all_figures.extend(figure_spans) all_figure_appearance.extend(figure_appearances) str_seq_lens.append(len(str_seqs)) print(file_name, len(str_seqs), len(all_texts)) return all_texts, all_labels, all_figures, all_figure_appearance, str_seq_lens
def createFeatureDataset(nodules_files=None): if nodules_files == None: #noddir = "/Users/weichaozhou/Documents/Boston University/EC500/DSB3Tutorial/luna2016/" noddir = "../Luna_Data/full_data_mskextraction/" nodules_files = glob(noddir + "masks*.npy") truth_d = pickle.load(open("truthdict.pkl", 'rb')) num_of_features = 9 features = np.zeros((len(nodules_files), num_of_features)) truth = np.zeros((len(nodules_files))) for i, f in enumerate(nodules_files): patient_ID = f.replace(".", "_").split("_")[-2] try: truth[i] = truth_d[int(patient_ID)] except KeyError: truth[i] = 0 print(f) features[i] = getRegionMetricRow(f) np.save("dataY.npy", truth) np.save("dataX.npy", features)
def images_in_list(path): os.chdir(path) imglist = [] for i in range(0, len(glob("*png"))): imglist.append(glob("*png")[i]) img_medium_color(imglist[i]) for i in range(0, len(glob("*jpg"))): imglist.append(glob("*jpg")[i]) img_medium_color(imglist[i]) for i in range(0, len(glob("*jpeg"))): imglist.append(glob("*jpeg")[i]) img_medium_color(imglist[i])
def create_report(path='.'): paths = glob('*/stat*') stats = [] for p in paths: stats.append(json.load(open(p, 'r'))) stats[-1]['path'] = p stats = pd.DataFrame(stats) stats['kind'] = stats['path'].apply( lambda x: x.split('/')[1].split('_')[1]) stats['it'] = stats['path'].apply(lambda x: x.split('/')[0].strip('it')) stats.loc[stats['kind'] == 'test', 'it'] = 999 stats['it'] = stats['it'].astype(int) stats.loc[stats['kind'] == 'fit', 'it'] += 1 stats = stats.sort_values(['it', 'kind']) stats.loc[stats['kind'] == 'test', 'it'] = -1 stats = stats.reset_index(drop=True) stats = stats.drop(len(stats) - 2, axis=0).drop('path', axis=1).drop('mean deviation', axis=1) stats.columns = [{ 'it': 'Iteration', 'kind': 'Kind' }.get(x, x.upper() + ' (eV)') for x in stats.columns] stats.loc[stats['Kind'] == 'test', 'Iteration'] = '' stats.index = [ k + '_' + str(it) for k, it in zip(stats['Kind'], stats['Iteration']) ] stats.index = [{'test_': 'test'}.get(i, i) for i in stats.index] stats['Kind'] = stats['Kind'].apply(lambda x: { 'sc': 'Self-consistent', 'fit': 'Fitting', 'test': 'Testing' }[x]) return stats
def load_data_by_id(id: int, video_df): """Given an frame ID, and a dataset description""" video = video_df[(video_df.start < id) & (video_df.end > id)] if len(video) == 0: return None, None, None, None paths = glob( "./data/*/{videoID}.npz".format(videoID=video.iloc[0].videoID)) if len(paths) == 0: return None, None, None, None path = paths[0] frame_id = int(id - video.start) q_color_images, q_bounding_box, q_landmarks_2d, q_landmarks_3d = load_data( path) return ( q_color_images[..., frame_id], q_bounding_box[..., frame_id], q_landmarks_2d[..., frame_id], q_landmarks_3d[..., frame_id], )
def renameWidget(srcWidget, oldName, newName): if oldName == newName: return # first rename parent if possible newPath = os.path.dirname(srcWidget) + "/" + newName os.system("mv {} {}".format(srcWidget, newPath)) # now we need to rename each of the files # and after renaming replace the instances of oldName in the data structures and python script with newName oldFiles = glob("{}/{}.*".format(newPath, oldName)) for oldFile in oldFiles: basename, extension = os.path.splitext(oldFile) newFile = newPath + "/" + newName + extension os.rename(oldFile, newFile) if extension == ".py": replaceNamePy(newFile, oldName, newName) else: replaceName(newFile, "name", newName) oldPyPath = "{}/{}.py".format(srcWidget, oldName) newPyPath = "{}/{}.py".format(newPath, newName) renameWidgetInToolDock(oldPyPath, newPyPath)
def createFeatureDataset(nodules_files=None): if nodules_files == None: noddir = "../data/tutorial4/" nodules_files = glob(noddir +"masks*.npy") truth_d = pickle.load(open("truthdict.pkl",'r')) num_of_features = 9 features = np.zeros((len(nodules_files),num_of_features)) truth = np.zeros((len(nodules_files))) for i,f in enumerate(nodules_files): patient_ID = f.replace(".","_").split("_")[2] try: truth[i] = truth_d[int(patient_ID)] except KeyError: truth[i] =0 print(f) features[i] = getRegionMetricRow(f) np.save("dataY.npy", truth) np.save("dataX.npy", features)
def __init__( self, casename='tmp', # prefix in filenames umin=-1, umax=1, # fixed range of y axis pause_between_frames=None, # movie speed backend='matplotlib', # or 'gnuplot' screen_movie=True, # show movie on screen? every_frame=1): # show every_frame frame self.casename = casename self.yaxis = [umin, umax] self.pause = pause_between_frames module = 'scitools.easyviz.' + backend + '_' exec('import %s as st' % module) self.st = st self.screen_movie = screen_movie self.every_frame = every_frame # Clean up old movie frames for filename in glob('frame_*.png'): os.remove(filename)
def count_freq(limit = 0): X = [] Y = [] max_words = word_dic["_MAX"] cat_names = [] for cat in os.listdir(root_dir): cat_dir = root_dir + "/" + cat if not os.path.isdir(cat_dir): continue cat_idx = len(cat_names) cat_names.append(cat) files = glob(cat_dir+"/*.wakati") i = 0 for path in files: print(path) cnt = count_file_freq(path) X.append(cnt) Y.append(cat_idx) if limit > 0: if i > limit: break i += 1 return X,Y
def test_can_resume_job_twice(self): from glob import glob import time job_bundle_name, _ = self._submit_and_schedule_job() self._pause_job(job_bundle_name) time.sleep(self.wait_time) # First Request to resume self._resume_job(job_bundle_name) # Check if second request to resume produces response as expected response = self._resume_job(job_bundle_name) time.sleep(self.wait_time) runs_from_scheduled_job = glob( f'{self.archives_dir_path}/{job_bundle_name}_*') self.assertEqual(204, response.status_code) self.assertIn(len(runs_from_scheduled_job), [3, 4, 5])
def main(): parser = argparse.ArgumentParser(description='PyTorch Wikitext-2 RNN/LSTM Language Model') parser.add_argument('--gt_file', type=str, default='data/eltec_gt_splits/eng/test.txt', help='random seed') parser.add_argument('--results_dir', type=str, default='data/results', help='use CUDA') args = parser.parse_args() print(args) gold = [l.rstrip().split('\t') for l in open(args.gt_file)] _, gold = zip(*gold) for fn in glob(f'{args.results_dir}/*.txt'): print('=' * 64) print(f'-> results for {fn}') silver = [l.rstrip().split('\t') for l in open(fn)] _, silver = zip(*silver) report = classification_report(gold, silver) print(report)
def packMap(dest, folder, mode=-1, keys=0): os.chdir(folder) assets = set() for file in glob("*.osu"): try: f = open(folder + file, 'r', encoding="utf-8") lines = f.readlines() f.close() if (-1 == mode or getMode(lines) == mode): checks = True if (3 == mode and 0 != keys and keys != getCS(lines)): checks = False if checks: assets.add(getAudio(lines)) assets.add(file) addAll(assets, getDesignAssets(lines)) packAssets(dest, assets, folder) except Exception as e: errors.append("Error while reading \"" + folder + file + "\" (" + str(e) + ")")
def curatecores(): for file in glob('*_unique.txt'): named = file.rsplit("_", 1)[0] outtie = open(named + '_unique-curated.txt', 'w') with open(file, 'r') as innie: for line in innie: if ">" in line: outtie.write(line) elif "TRINITY" in line: contigtag = line.split(" ")[0] spacey = line.split(" ")[1:] putrid = [tears for tears in spacey if "GO" in tears] print "file: " + named purrid = str(putrid) print purrid twosie = purrid.split(";")[1] print "twosie: " + twosie outtie.write(twosie) outtie.write("\n") else: continue