def get_matching_excluded_paths(context_path: [str], excluded_paths: [str], included_paths: [str], exclude_gitignored_files: bool) -> [str]: def abs_path_glob_including_snapshot(p): return os.path.abspath(os.path.join(context_path, p)) def expand_if_dir(path): if os.path.isdir(path): # Return the dir as well as the files inside return itertools.chain( iter([path]), glob2.iglob(os.path.join(path, '**'), recursive=True, include_hidden=True)) else: return iter([path]) included_paths = set( ip for p in included_paths for ip in glob2.iglob(abs_path_glob_including_snapshot(p), recursive=True, include_hidden=True)) # Get the files inside the directories included_paths = set(p for ip in included_paths for p in expand_if_dir(ip)) all_included_prefixes = set( os.sep.join(ip.split(os.sep)[:n + 1]) for ip in included_paths for n in range(len(ip.split(os.sep)))) # Expand the globs excluded_paths = [ ep for p in excluded_paths for ep in glob2.iglob(abs_path_glob_including_snapshot(p), recursive=True, include_hidden=True) ] # Add the git ignored files git_ignored_files = [] # A value of None for exclude_gitignored_files means "exclude if git is # available" use_git = exclude_gitignored_files or (exclude_gitignored_files is None and is_git_present(context_path)) if use_git: git_ignored_files = [abs_path_glob_including_snapshot('.git')] + \ get_ignored_git_files(context_path) excluded_paths += git_ignored_files excluded_and_not_included_paths = [] for ep in excluded_paths: if ep in all_included_prefixes: excluded_and_not_included_paths += ( p for p in expand_if_dir(ep) if p not in all_included_prefixes) else: excluded_and_not_included_paths.append(ep) return [ p[len(os.path.abspath(context_path)) + 1:] for p in excluded_and_not_included_paths ]
def main(): logger.info('started') notDone=True while notDone: #anyone done? for process,file in readers: if not process.is_alive(): logger.info('{0}/{1} marked as done'.format(process,file)) readers.remove((process,file)) for afile in glob2.iglob(options.filemask): #logger.debug('noticed file {0}'.format(afile)) #if we aren't reading a file #spin up a process to read the file activefiles=[] for process,filename in readers: activefiles.append(filename) if afile not in activefiles: logger.info('starting a reader for {0}'.format(afile)) readingProcess=Process(target=readCEFFile,args=(afile,),name="cef2mozdefReadFile") readers.append((readingProcess,afile)) readingProcess.start() #if we are reading a file no longer in the list of active files (rotated, etc), #tell it's reader it's ok to stop. for process,filename in readers: if filename not in glob2.iglob(options.filemask): logger.info('{0} no longer valid, stopping file reader'.format(filename)) process.terminate() process.join() #change this if you want it to stop (cronjob, or debugging), else it will run forever (ala /etc/init service) #notDone=False time.sleep(2) logger.info('finished')
def build_trigger_pd(): run_num = [ int(num[0]) for file, num in ( glob2.iglob(f"{data_folder}/raw_dat/RUN_*/", with_matches=True)) ] run_num = [num for num in run_num if num > 106] gemroc = [] run = [] subrun = [] trigger = [] time_end = [] for run_n in run_num: for filename, sub in glob2.iglob( f"{data_folder}/raw_dat//RUN_{run_n}/ACQ_log_*", with_matches=True): with open(filename, 'r') as filelog: for line in filelog.readlines(): if "total" in line: run.append(int(run_n)) subrun.append(int(sub[0])) gemroc.append(int(line.split()[11])) trigger.append(int(line.split()[-1])) time_end.append(line.split("--")[0]) trigger_pd = pd.DataFrame({ "gemroc": gemroc, "run": run, "subrun": subrun, "triggers": trigger, "time": time_end }) return trigger_pd
def directoryU(): global counter images = list(glob2.iglob("static/img/*.*")) path = images[counter] Invoice_No = request.form['Invoice_No'] Invoice_Date = request.form['Invoice_Date'] Supplier_Name = request.form['Supplier_Name'] Amount = request.form['Amount'] CGST = request.form['CGST'] SGST = request.form['SGST'] GST_No_TVS = request.form['GST_No_TVS'] GST_No_SUPPLIER = request.form['GST_No_SUPPLIER'] PAN_No_SUPPLIER = request.form['PAN_No_SUPPLIER'] PAN_NO_TVS = request.form['PAN_NO_TVS'] PO_No = request.form['PO_No'] PO_Date = request.form['PO_Date'] sample = { "Invoice_No": Invoice_No, "Invoice_Date": Invoice_Date, "Supplier_Name": Supplier_Name, "Amount": Amount, "CGST": CGST, "SGST": SGST, "GST_No_TVS": GST_No_TVS, "GST_No_SUPPLIER": GST_No_SUPPLIER, "PAN_No_SUPPLIER": PAN_No_SUPPLIER, "PAN_NO_TVS": PAN_NO_TVS, "PO_No": PO_No, "PO_Date": PO_Date } #{"Invoice_No","Invoice_Date","Supplier_Name",Amount","CGST","SGST","GST_No_TVS","GST_No_SUPPLIER","PAN_No_SUPPLIER","PAN_NO_TVS","PO_No","PO_Date"} print("In custom : ", sample) Iname = path.split('/')[2].split('.')[0] path = 'uploads/user_response/' + Iname # if not os.path.isfile(path): # os.mkdir(path) #uuid = randint(0, 99) book = xlwt.Workbook(encoding="utf-8") sheet1 = book.add_sheet("sheet1") colunm_count = 0 data = sample source_photo = os.path.join(os.getcwd(), images[counter]) dest = app.config['FINAL_DEST_OF_PHOTOS'] shutil.move(source_photo, dest) for title, value in data.items(): sheet1.write(0, colunm_count, title) sheet1.write(1, colunm_count, value) colunm_count += 1 file_name = path + ".xls" % () book.save(file_name) images = list(glob2.iglob("static/img/*.*")) if len(images) == 0: return render_template("index.html") counter = 0 return render_template("extract.html", image_path=images[counter])
def find_praat_test_files(): for dir in glob.iglob(os.path.join(PRAAT_TEST_BASE_DIR, "*", "")): if os.path.basename(os.path.dirname(dir)) in PRAAT_TEST_IGNORE_SUBDIRS: continue for fn in glob.iglob(os.path.join(dir, "**", "*.praat"), recursive=True): if "_GUI_" in fn: continue rel_fn = os.path.relpath(fn, PRAAT_TEST_BASE_DIR) yield pytest.param(fn, id=rel_fn)
def archive_files(from_pattern, archive_dir, to_pattern, batch_id): """ Archives files matching from_pattern and renames to to_pattern based on uid in a batch folder :param from_pattern: :param archive_dir :param to_pattern: :param batch_id: :return: """ # place archived files in a timestamped folder archive_folder = os.path.join(archive_dir, batch_id) if not os.path.isdir(archive_folder): os.mkdir(archive_folder) os.mkdir(os.path.join(archive_folder, 'unsigned_certs')) os.mkdir(os.path.join(archive_folder, 'signed_certs')) os.mkdir(os.path.join(archive_folder, 'sent_txs')) os.mkdir(os.path.join(archive_folder, 'receipts')) os.mkdir(os.path.join(archive_folder, 'blockchain_certificates')) archived_file_pattern = os.path.join(archive_folder, to_pattern) [ shutil.move(filename, models.convert_file_name(archived_file_pattern, uid)) for filename, (uid, ) in glob2.iglob(from_pattern, with_matches=True) ]
def sort_allrepeat(dir): sort_dir = os.path.join(dir, 'sort') if os.path.exists(sort_dir): shutil.rmtree(sort_dir) os.mkdir(sort_dir) for path in glob2.iglob(dir + '/*.jpg'): image = Image.open(path)
def extract(): images = list(glob2.iglob("static/img/*.*")) if len(images) == 0: return render_template("index.html") print(images) return render_template("extract.html", image_path=images[counter])
def elab_on_run_dict(self, run): time_dict_start = {} time_dict_end = {} for filename, (subrun, ) in glob2.iglob( self.path + "/RUN_{}/ACQ_log_*".format(run), with_matches=True): lenght_list = [] with open(filename, "r") as flog: lines = flog.readlines() if len(lines) != 0: if "--" in lines[0]: utc_time = time.strptime(lines[0].split("--")[0], "%a %b %d %H:%M:%S %Y ") start_time = timegm(utc_time) else: start_time = -1 if "--" in lines[-1]: utc_time = time.strptime(lines[-1].split("--")[0], "%a %b %d %H:%M:%S %Y ") end_time = timegm(utc_time) else: end_time = start_time time_dict_start[subrun] = start_time time_dict_end[subrun] = end_time else: time_dict_start[subrun] = -1 time_dict_end[subrun] = -1 return time_dict_start, time_dict_end
def main(): logger.info('started') notDone = True while notDone: #anyone done? for process, file in readers: if not process.is_alive(): logger.info('{0}/{1} marked as done'.format(process, file)) readers.remove((process, file)) for afile in glob2.iglob(options.filemask): #if we aren't reading a file #or at the end of the file #spin up a process to read the file activefiles = [] for process, filename in readers: activefiles.append(filename) if afile not in activefiles: pygtail = Pygtail(afile, pretend=False) if getsize(afile) > pygtail._offset: logger.info('starting a reader for {0}'.format(afile)) readingProcess = Process(target=readCEFFile, args=(afile, pygtail), name="cef2mozdefReadFile") readers.append((readingProcess, afile)) readingProcess.start() #change this if you want it to stop (cronjob, or debugging), else it will run forever (ala /etc/init service) #notDone=False time.sleep(2) logger.info('finished')
def app_css(template_reference): """Generate app css link tags for Flask app.""" link_tags = "" # Excluded css libs for ex_lib in G.css_config['excluded_libs']: if ex_lib.startswith('http'): link_tags += link(ex_lib, absolute=True) else: link_tags += link(ex_lib) # if False: if G.debug: # libs link_tags += ''.join(map(link, G.css_config['libs'])) # layout for layout_path in G.css_config['layout']: # 支持通配符 if '*' in layout_path: for path in glob2.iglob( os.path.join(G.static_path, layout_path)): relative_path = path.split(G.static_path)[1][1:] link_tags += link(relative_path) else: link_tags += link(layout_path) # page template_name = _get_template_name(template_reference) page_css_path = os.path.join(G.css_config['page'], template_name.replace('html', 'css')) link_tags += link(page_css_path) else: link_tags += link(APP_CSS) return Markup(link_tags)
def MAE_Value(resDir, gtDir): p = parameter() gtThreshold = p['gtThreshold'] gtImgs = glob2.iglob(gtDir + '/*.png') MAE = [] for gtName in gtImgs: dir, name = os.path.split(gtName) mapName = os.path.join(resDir, name) if len(name) > 7: continue #if method!='CGL' and method!='MFSR' and method!='TSAA'and method!='PDNet' and method!='MFCF' and method!='Ours': #mapName = mapName.replace('.png', '_' + 'RT_' + '.png') curMap = im2double(cv2.imread(mapName, cv2.IMREAD_GRAYSCALE)) curGT = im2double(cv2.imread(gtName, cv2.IMREAD_GRAYSCALE)) curGT = (curGT >= gtThreshold).astype(np.float32) if curMap.shape[0] != curGT.shape[0]: curMap = cv2.resize(curMap, (curGT.shape[1], curGT.shape[0])) diff = np.abs(curMap - curGT) MAE.append(np.mean(diff)) return np.mean(MAE)
def scan_keys(keys_dir=KEYS_DIR): keys_map = KeysMap() patterns = ['Key-6.dat', '*.jks', '*.zs2', '*.ZS2'] if glob2: files = sum( (list(glob2.iglob(os.path.join(KEYS_DIR, '**/%s' % pattern))) for pattern in patterns), []) else: files = sum((list( glob.iglob(os.path.join(KEYS_DIR, '**/%s' % pattern), recursive=True)) for pattern in patterns), []) log.info('Keys (%s) in %s', len(files), keys_dir) for filename in files: filename = os.path.abspath(filename) if filename not in keys_map.values(): log.info('Checking new key %s', filename) cabinet = Cabinet() try: inn, fio, expires = cabinet.pre_login_cert(filename) except Exception as e: log.exception('Error occured on key processing %s %s', filename, repr(e)) if DEBUG: import pdb pdb.set_trace() # noqa continue finally: cabinet.quit() log.info('Adding key inn=%s fio=%s expires=%s filename=%s', inn, fio, expires, filename) keys_map.add_key(inn, fio, filename, expires)
def main(): logger.info('started') notDone=True while notDone: #anyone done? for process,file in readers: if not process.is_alive(): logger.info('{0}/{1} marked as done'.format(process,file)) readers.remove((process,file)) for afile in glob2.iglob(options.filemask): #if we aren't reading a file #or at the end of the file #spin up a process to read the file activefiles=[] for process,filename in readers: activefiles.append(filename) if afile not in activefiles: pygtail = Pygtail(afile,pretend=False) if getsize(afile)> pygtail._offset: logger.info('starting a reader for {0}'.format(afile)) readingProcess=Process(target=readCEFFile,args=(afile,pygtail),name="cef2mozdefReadFile") readers.append((readingProcess,afile)) readingProcess.start() #change this if you want it to stop (cronjob, or debugging), else it will run forever (ala /etc/init service) #notDone=False time.sleep(2) logger.info('finished')
def elab_on_run(self, run): rname = self.elabpath + "{}".format(run) rname = rname + '/time_run_{}.root'.format(run) rootFile = R.TFile(rname, 'recreate') tree = R.TTree('tree', '') tree_struct = R.TreeStruct() tree.Branch('runNo', R.AddressOf(tree_struct, 'runNo'), 'runNo/I') tree.Branch('subRunNo', R.AddressOf(tree_struct, 'subRunNo'), 'subRunNo/I') tree.Branch('triggers', R.AddressOf(tree_struct, 'triggers'), 'triggers/I') tree.Branch('start_time', R.AddressOf(tree_struct, 'start_time'), 'start_time/I') tree.Branch('end_time', R.AddressOf(tree_struct, 'end_time'), 'end_time/I') tree_struct.runNo = int(run) for filename, (subrun, ) in glob2.iglob( self.path + "/RUN_{}/ACQ_log_*".format(run), with_matches=True): tree_struct.subRunNo = int(subrun) lenght_list = [] with open(filename, "r") as flog: lines = flog.readlines() for line in lines: if "packets" in line: lenght_list.append( int(line.split("TM.dat, total packets= ")[1])) if len(lines) != 0: if "--" in lines[0]: utc_time = time.strptime(lines[0].split("--")[0], "%a %b %d %H:%M:%S %Y ") start_time = timegm(utc_time) else: start_time = -1 if "--" in lines[-1]: utc_time = time.strptime(lines[-1].split("--")[0], "%a %b %d %H:%M:%S %Y ") end_time = timegm(utc_time) else: end_time = start_time tree_struct.start_time = start_time tree_struct.end_time = end_time else: tree_struct.start_time = -1 tree_struct.end_time = -1 lenght_list = np.asarray(lenght_list) if len(lenght_list) > 0 and (np.count_nonzero(lenght_list) ): #interrotto da timeout, con dati tree_struct.triggers = int( np.min(lenght_list[np.nonzero(lenght_list)])) else: tree_struct.triggers = -1 tree.Fill() rootFile.Write() rootFile.Close()
def _lookup_savegame_path(self, name, pathname): path = self.configurator.settings.get(name, 'path', fallback=None) if not path or not os.path.exists(path): path = os.path.normpath( os.path.expandvars( os.path.expanduser(pathname))) if not os.path.isabs(path): for search_path in self.search_paths: glob_pattern = os.path.join( os.path.normpath(search_path), path) glob_path = list(itertools.islice( glob2.iglob(glob_pattern), 1)) if glob_path: path = glob_path[0] break if os.path.exists(path): if not self.configurator.settings.has_section(name): self.configurator.settings.add_section(name) self.configurator.settings.set(name, 'path', path) self.configurator.save_settings_file() else: path = None # TODO logging # raise RuntimeError( # f"Savegame path could not be found: {pathname}") return path
def in_valid_paths(self, root, path, valid_paths): for valid_path in [p for p in valid_paths if isinstance(p, str)]: if path in list(map(normalize_path, glob(valid_path))): return True for valid_path in [p for p in valid_paths if not isinstance(p, str)]: for nested_valid_path in valid_path: for found_nested_path, matches in iglob(nested_valid_path, with_matches=True): found_nested_path = normalize_path(found_nested_path) if found_nested_path == path: # check matches matches = map(normalize_path, matches) for match in matches: for related_path in valid_path: if related_path != found_nested_path: related_path = related_path.replace( '*', match, 1) if not os.path.isfile(related_path): rel_path = normalize_path( os.path.relpath(path, root)) rel_related_path = normalize_path( os.path.relpath( related_path, root)) raise ValidationError( '{file} missing related file {related}' .format(file=rel_path, related=rel_related_path)) return True raise ValidationError('{file} is not allowed'.format(file=path))
def MAE_Value(resDir, gtDir): p = parameter() gtThreshold = p['gtThreshold'] gtImgs = glob2.iglob(gtDir + '/*.jpg') MAE = [] for gtName in gtImgs: dir, name = os.path.split(gtName) mapName = os.path.join(resDir, name[:-4] + '.png') ###### #print(mapName) if os.path.exists(mapName) is False: mapName = mapName.replace('.png', '.jpg') if os.path.exists(mapName) is False: mapName = mapName.replace('.jpg', '.bmp') curMap = im2double(cv2.imread(mapName, cv2.IMREAD_GRAYSCALE)) curGT = im2double(cv2.imread(gtName, cv2.IMREAD_GRAYSCALE)) curGT = (curGT >= gtThreshold).astype(np.float32) if curMap.shape[0] != curGT.shape[0]: curMap = cv2.resize(curMap, (curGT.shape[1], curGT.shape[2])) diff = np.abs(curMap - curGT) MAE.append(np.mean(diff)) return np.mean(MAE)
def preview(self, request, pk=None): job = self.get_object() ips = job.rule.information_packages.all() files = [] for ip in ips: datadir = os.path.join(ip.policy.cache_storage.value, ip.object_identifier_value) for pattern, spec in six.iteritems(job.rule.specification): for path in iglob(datadir + '/' + pattern): if os.path.isdir(path): for root, dirs, files in walk(path): rel = os.path.relpath(root, datadir) for f in files: files.append({ 'ip': ip.object_identifier_value, 'document': os.path.join(rel, f) }) elif os.path.isfile(path): rel = os.path.relpath(path, datadir) files.append({ 'ip': ip.object_identifier_value, 'document': rel }) return Response(files)
def buildListOfFiles(searchGlob): """ Build a master list of all files that we should check. """ return [ fpath for fpath in glob2.iglob(searchGlob) if os.path.isfile(fpath) ]
def convertpdftoimage(): for i, pages in enumerate(glob2.iglob('uploads/*.pdf')): print(pages) pg = pdf2image.convert_from_path(pages, 300) os.remove(pages) for j in range(len(pg)): pg[j].save('uploads/skp{}{}.jpg'.format(i, j), 'JPEG')
def files(self): """ a generator to return all files watched by this dog """ for path in self.paths: for file in glob2.iglob(path): yield file
def dec_run_header_trailer(self): """ Decodes one run in parallel :return: """ input_list = [] return_list_hd = [] return_list_tr = [] return_list_udp = [] self.decoder = pl_lib.decoder(1, self.run_number, downsamplig=self.downsampling) if not self.silent: print ("Decoding") for filename ,(subrun,gemroc) in glob2.iglob(self.data_folder+"/raw_dat/RUN_{}/SubRUN_*_GEMROC_*_TM.dat".format(self.run_number), with_matches=True): input_list.append((filename, int(subrun), int(gemroc))) if len(input_list)>0: with Pool(processes=self.cpu_to_use) as pool: with tqdm(total=len(input_list), disable=self.silent) as pbar: for i, x in enumerate(pool.imap_unordered(self.decode_on_file_header_trailer, input_list)): return_list_hd.append(x[0]) return_list_tr.append(x[1]) return_list_udp.append(x[2]) pbar.update() header_pd = pd.concat(return_list_hd) trailer_pd = pd.concat(return_list_tr) UDP_pd = pd.concat(return_list_udp) header_pd.to_pickle(os.path.join(self.data_folder,"raw_root",f"{self.run_number}","header_pd.gzip"), compression ="gzip") trailer_pd.to_pickle(os.path.join(self.data_folder,"raw_root",f"{self.run_number}","trailer_pd.gzip"), compression ="gzip") UDP_pd.to_pickle(os.path.join(self.data_folder,"raw_root",f"{self.run_number}","UDP_pd.gzip"), compression ="gzip") else: print (f"Can't find any .dat file in {self.data_folder}/raw_dat/RUN_{self.run_number}")
def get_features(paths, **kwargs): """Get features for given paths. :param list paths: `list` of paths (file or dirs) :return: `list` of `Feature` objects. """ seen_names = set() features = [] for path in paths: if path not in seen_names: seen_names.add(path) if op.isdir(path): features.extend( get_features( glob2.iglob(op.join(path, "**", "*.feature")), **kwargs ) ) else: base, name = op.split(path) feature = Feature.get_feature(base, name, **kwargs) features.append(feature) features.sort(key=lambda feature: feature.name or feature.filename) return features
def app_css(template_reference): """Generate app css link tags for Flask app.""" link_tags = "" # Excluded css libs for ex_lib in G.css_config['excluded_libs']: if ex_lib.startswith('http'): link_tags += link(ex_lib, absolute=True) else: link_tags += link(ex_lib) # if False: if G.debug: # libs link_tags += ''.join(map(link, G.css_config['libs'])) # layout for layout_path in G.css_config['layout']: # 支持通配符 if '*' in layout_path: for path in glob2.iglob(os.path.join(G.static_path, layout_path)): relative_path = path.split(G.static_path)[1][1:] link_tags += link(relative_path) else: link_tags += link(layout_path) # page template_name = _get_template_name(template_reference) page_css_path = os.path.join(G.css_config['page'], template_name.replace('html', 'css')) link_tags += link(page_css_path) else: link_tags += link(APP_CSS) return Markup(link_tags)
def process_data(partition): """ Reads audio waveform and transcripts from a dataset partition and generates mfcc featues. Args: parition - represents the dataset partition name. Returns: feats: dict containing mfcc feature per utterance transcripts: dict of lists representing transcript. utt_len: dict of ints holding sequence length of each utterance in time frames. """ feats = {} transcripts = {} utt_len = {} # Required for sorting the utterances based on length for filename in glob2.iglob(partition + '/**/*.txt'): with open(filename, 'r') as f: for line in f: parts = line.split() audio_file = parts[0] file_path = os.path.join(os.path.dirname(filename), audio_file + '.flac') audio, sample_rate = sf.read(file_path) feats[audio_file] = compute_mfcc(audio, sample_rate) utt_len[audio_file] = feats[audio_file].shape[0] target = ' '.join(parts[1:]) transcripts[audio_file] = [CHAR_TO_IX[i] for i in target] return feats, transcripts, utt_len
def extract_num_triggers_run(data_raw_folder, run_number): return_dict = {} for filename, subrun in glob2.iglob("{}/RUN_{}/ACQ_log_*".format( data_raw_folder, run_number), with_matches=True): return_dict[subrun[0]] = int(extract_num_triggers_subrun(filename)) return return_dict
def add_patterns(self, patterns): for p in patterns: for log_file in glob2.iglob(p): try: self.add_available(log_file) except LogError, e: logger.warning(e)
def _process(self): for source_path in iglob(os.path.join(self.source, '**')): path = os.path.relpath(source_path, self.source) target_path = os.path.join(self.target, path) try: target_time = os.path.getmtime(target_path) except OSError: target_time = 0 try: self.timestamp = os.path.getmtime(source_path) if self.timestamp == target_time: raise ValueError item = next(x for x in self.handlers if x['regex'].match(path)) _in = open(source_path, 'rb') except(OSError, StopIteration, ValueError): continue for _filter in item['filters']: out = BytesIO() _filter(_in, out, path) _in = out _in.seek(0) for action in item['actions']: out = BytesIO() target_path = action(_in, out, target_path) _in.seek(0) out.seek(0) if target_path: self._write(out, target_path) _in.close()
def merg_dec_func(self, subrun ): pd_list = [] for filename, (gemroc) in glob2.iglob(self.data_folder + "/raw_root/{}/SubRUN_{}_GEMROC_*_TM.pickle.gzip".format(self.run_number, subrun), with_matches=True): pd_list.append(pd.read_pickle(filename, compression="gzip")) if len(pd_list)>0: subrun_pd = pd.concat(pd_list, ignore_index=True) subrun_pd.to_pickle(self.data_folder + "/raw_root/{}/Sub_RUN_dec_{}.pickle.gzip".format(self.run_number, subrun), compression="gzip")
def find_praat_test_files(): for dir in glob.iglob(os.path.join(PRAAT_TEST_BASE_DIR, "*", "")): if os.path.basename(os.path.dirname(dir)) in PRAAT_TEST_IGNORE_SUBDIRS: continue for fn in glob.iglob(os.path.join(dir, "**", "*.praat"), recursive=True): rel_fn = os.path.relpath(fn, PRAAT_TEST_BASE_DIR) marks = [] if rel_fn == os.path.join("num", "mean.praat"): marks.append( pytest.mark.xfail( sys.platform == 'win32', reason= "`long double` is only double precision when compiling with MSVC, causing `assert mean ({ -1e18, 3, 1e18 }) = 1` to fail" )) yield pytest.param(fn, id=rel_fn, marks=marks)
def import_raw_hands(hands_dir): for hands_filepath in glob2.iglob(os.path.join(hands_dir, '*.txt')): with open(hands_filepath, 'rt', encoding='latin-1') as hands_file: print(hands_filepath) file_text = replace_euro_symbol(hands_file.read()) for raw_hand in file_text.split('\n\n\n\n\n'): if raw_hand and is_valid_hand(raw_hand): yield raw_hand
def glob(self, basedir, expr): """Runs when a glob expression needs to be resolved. """ expr = path.join(basedir, expr) for filename in glob.iglob(expr): if path.isdir(filename): continue yield filename
def extract_num_triggers_run(self, run_number): tot = 0 for filename, subrun in glob2.iglob( "/{}/raw_dat/RUN_{}/ACQ_log_*".format(self.data_folder, run_number), with_matches=True): tot += int(extract_num_triggers_subrun(filename)) return tot
def start_encrypt(key): #Encrypts all files recursively starting from startPath for filename in glob2.iglob(startPath): if (os.path.isfile(filename)): #print('Encrypting> ' + filename) if (os.path.splitext(filename)[1] != extension): encrypt_file(key, filename) os.remove(filename)
def migrate_tests(): """Migrate outdated tests to the most recent form.""" if len(sys.argv) != 2: print 'Usage: pytestbdd_migrate_tests <path>' sys.exit(1) path = sys.argv[1] for file_path in glob2.iglob(os.path.join(os.path.abspath(path), '**', '*.py')): migrate_tests_in_file(file_path)
def add_patterns(self, patterns): for p in patterns: for log_file in glob2.iglob(p): try: self.add_available(log_file) except LogError as e: logger.warning(e) logger.info("Added %d log file(s)", len(self.available))
def glob(self, basedir, expr): """Generator that runs when a glob expression needs to be resolved. Yields a list of absolute filenames. """ expr = path.join(basedir, expr) for filename in glob.iglob(expr): if path.isdir(filename): continue yield filename
def _iglob_within(self, root, pathname): try: from glob2 import iglob except ImportError: from glob import iglob import os for path in iglob(os.path.join(root, pathname)): yield path.replace(root, '').lstrip('/')
def prepare_issuance_batch(unsigned_certs_dir, signed_certs_dir, blockchain_certs_dir, work_dir, file_extension=JSON_EXT): """ Prepares file system for issuing a batch of certificates. Copies inputs to work_dir, and ensures that all output dirs required for processing the batch exist. :param unsigned_certs_dir: input certificates :param signed_certs_dir: output dir :param blockchain_certs_dir: output dir :param work_dir: work dir :return: """ # create work dir if it doesn't already exist os.makedirs(work_dir, exist_ok=True) # create final output dirs if they don't already exist os.makedirs(blockchain_certs_dir, exist_ok=True) os.makedirs(signed_certs_dir, exist_ok=True) # ensure previous processing state, if any, is cleaned up for item in os.listdir(work_dir): file_path = os.path.join(work_dir, item) if os.path.isdir(file_path): shutil.rmtree(file_path) # define work subdirs unsigned_certs_work_dir = os.path.join(work_dir, UNSIGNED_CERTIFICATES_DIR) signed_certs_work_dir = os.path.join(work_dir, SIGNED_CERTIFICATES_DIR) blockchain_certs_work_dir = os.path.join(work_dir, BLOCKCHAIN_CERTIFICATES_DIR) # copy input certs to unsigned certs work subdir and create output subdirs shutil.copytree(unsigned_certs_dir, unsigned_certs_work_dir) os.makedirs(signed_certs_work_dir, exist_ok=True) os.makedirs(blockchain_certs_work_dir, exist_ok=True) cert_info = collections.OrderedDict() input_file_pattern = str(os.path.join(unsigned_certs_work_dir, '*' + file_extension)) matches = glob2.iglob(input_file_pattern, with_matches=True) if not matches: logging.warning('No certificates to process') raise NoCertificatesFoundError('No certificates to process') # create certificate metadata for each certificates for filename, (uid,) in sorted(matches): certificate_metadata = CertificateMetadata(uid=uid, unsigned_certs_dir=unsigned_certs_work_dir, signed_certs_dir=signed_certs_work_dir, blockcerts_dir=blockchain_certs_work_dir, final_blockcerts_dir=blockchain_certs_dir, file_extension=file_extension) cert_info[uid] = certificate_metadata logging.info('Processing %d certificates', len(cert_info)) return cert_info
def eglob(path, exclude=None): """Like glob.glob, but supports "/path/**/{a,b,c}.txt" lookup""" fi = itertools.chain.from_iterable paths = list(fi(glob2.iglob(d) for d in expand_paths(path))) if exclude: cached_regex = cached_regices.get(exclude, None) if not cached_regex: cached_regex = cached_regices[exclude] = re.compile(exclude) paths = [x for x in paths if not cached_regex.search(x)] return paths
def match(patterns): for pattern in patterns: if os.path.isfile(pattern): yield pattern elif os.path.isdir(pattern): for path, names, filenames in os.walk(pattern): for filename in filenames: yield os.path.join(path, filename) else: for filename in glob2.iglob(pattern): yield filename
def add_patterns(self, patterns): i = 0 for p in patterns: for log_file in glob2.iglob(p): try: self.add_available(log_file) i += 1 except LogError as e: logger.warning(e) logger.info('Added %d log file(s)', i) return i
def list(self, path): for root in self.locations: for absolute_path in glob2.iglob(os.path.join(root, path)): if os.path.isfile(absolute_path): logical_path = os.path.relpath(absolute_path, root) # Filter out any files that don't end in the specified extensions if not self.ext_pattern.search(absolute_path): continue # Filter out ignored paths if self.ignore_pattern.search(absolute_path): continue yield logical_path, absolute_path
def find_unsigned_certificates(app_config): cert_info = collections.OrderedDict() for filename, (uid,) in sorted(glob2.iglob( app_config.unsigned_certs_file_pattern, with_matches=True)): with open(filename) as cert_file: cert_raw = cert_file.read() cert_json = json.loads(cert_raw) certificate_metadata = make_certificate_metadata(app_config, uid, cert_json['recipient']['publicKey']) cert_info[uid] = certificate_metadata return cert_info
def cp(sources, dest): """Copy a file or a list of files. Copy one or more files. Also is possible copy directories. There is one different between this two sources, `some/folder` and `/some/folder/*`. In the first case the folder is copied to the destination, in the second case the files inside the directory are copied, but not the directory itself. """ logging.info('-----') for source in _as_list(sources): for path in iglob(source): copy_rec(path, dest)
def glob(self, expr): """Globs the given expression with the base path of the module. This uses the glob2 module and supports recursive globs ('**/*'). Args: expr: Glob expression. Returns: A list of all files that match the glob expression. """ if not expr or not len(expr): return [] base_path = os.path.dirname(self.path) glob_path = os.path.join(base_path, expr) return list(glob2.iglob(glob_path))
def find_unsigned_certificates(app_config): cert_info = {} for filename, (uid,) in glob2.iglob(app_config.unsigned_certs_file_pattern, with_matches=True): with open(filename) as cert_file: cert_raw = cert_file.read() cert_json = json.loads(cert_raw) certificate_metadata = CertificateMetadata(app_config, uid, cert_json['recipient']['givenName'] + ' ' + cert_json['recipient'][ 'familyName'], cert_json['recipient']['pubkey']) cert_info[uid] = certificate_metadata return cert_info
def make_archive(self, task, test_dir): # Return early if no test_dir is specified if test_dir is None: return None # Return early if there are no globs specified if task.task.artifact_archive_globs is None or len(task.task.artifact_archive_globs) == 0: return None all_matched = set() total_size = 0 for g in task.task.artifact_archive_globs: try: matched = glob2.iglob(test_dir + "/" + g) for m in matched: canonical = os.path.realpath(m) if not canonical.startswith(test_dir): LOG.warn("Glob %s matched file outside of test_dir, skipping: %s" % (g, canonical)) continue total_size += os.stat(canonical).st_size all_matched.add(canonical) except Exception as e: LOG.warn("Error while globbing %s: %s" % (g, e)) if len(all_matched) == 0: return None max_size = 200*1024*1024 # 200MB max uncompressed size if total_size > max_size: # If size exceeds the maximum size, upload a zip with an error message instead LOG.info("Task %s generated too many bytes of matched artifacts (%d > %d)," \ + "uploading archive with error message instead.", task.task.get_id(), total_size, max_size) archive_buffer = cStringIO.StringIO() with zipfile.ZipFile(archive_buffer, "w") as myzip: myzip.writestr("_ARCHIVE_TOO_BIG_", "Size of matched uncompressed test artifacts exceeded maximum size" \ + "(%d bytes > %d bytes)!" % (total_size, max_size)) return archive_buffer # Write out the archive archive_buffer = cStringIO.StringIO() with zipfile.ZipFile(archive_buffer, "w", zipfile.ZIP_DEFLATED) as myzip: for m in all_matched: arcname = os.path.relpath(m, test_dir) while arcname.startswith("/"): arcname = arcname[1:] myzip.write(m, arcname) return archive_buffer
def _get_mtimes(paths): files = {} for path in paths: if '*' not in path and not os.path.exists(path): logging.error("WARNING, file not found: '{}'".format(path)) if os.path.isdir(path): path = '{}/{}'.format(path.rstrip('/'), '**') for file_path in iglob(path): if os.path.isfile(file_path): files[file_path] = os.path.getmtime(file_path) return files
def main(): '''For each file matching our filemask read in lines until we've got a complete slapd connection/termination parse out the interesting transactions and move on to the next sleeping in between to be nice to our cpu for really large files ''' for afile in glob2.iglob(options.filemask): #cache list of log lines to help deal with the multi-line slapd format linecache=[] #defaults ipAddress='0.0.0.0' bindName='Unknown' errCode=None errName='Unknown' #dictionary to store bits of bind transactions as we find them in the multi-line logs bindDetails={} if exists(afile): #sometimes files can move/archive while we iterate the list #have pygtail feed us lines without incrementing the offset until we've posted any results we've found. pt = Pygtail(afile,pretend=True) for line in pt: if 'slapd' in line and ('ACCEPT' in line or 'RESULT' in line or 'BIND' in line ): #store lines until we match a multi line slapd connection structure #ACCEPT yields the source IP, BIND yields the dn [email protected], RESULT yields the success/failure if len(linecache)>options.cachelength: linecache.remove(linecache[0]) linecache.append(line) searchforBind(line,bindDetails) if 'RESULT' in line: #maybe it's the termination of an earlier bind attempt searchforBindResult(line,bindDetails,linecache) postBindResults(bindDetails,pt,linecache) trimBindDetails(bindDetails,linecache) sleep(.00001) #be nice, but not too nice #post any remaining bindDetails postBindResults(bindDetails,pt,linecache,True)
def get_features(paths): """Get features for given paths. :param paths: `list` of paths (file or dirs) :return: `list` of `Feature` objects """ seen_names = set() features = [] for path in paths: if path in seen_names: continue seen_names.add(path) if os.path.isdir(path): features.extend(get_features(glob2.iglob(os.path.join(path, '**', '*.feature')))) else: base, name = os.path.split(path) feature = Feature.get_feature(base, name) features.append(feature) features.sort(key=lambda feature: feature.name or feature.filename) return features
def archive_files(from_pattern, archive_dir, to_pattern, batch_id): """ Archives files matching from_pattern and renames to to_pattern based on uid in a batch folder :param from_pattern: :param archive_dir :param to_pattern: :param batch_id: :return: """ # place archived files in a timestamped folder archive_folder = os.path.join(archive_dir, batch_id) if not os.path.isdir(archive_folder): os.mkdir(archive_folder) os.mkdir(os.path.join(archive_folder, 'unsigned_certs')) os.mkdir(os.path.join(archive_folder, 'signed_certs')) os.mkdir(os.path.join(archive_folder, 'sent_txs')) os.mkdir(os.path.join(archive_folder, 'receipts')) os.mkdir(os.path.join(archive_folder, 'blockchain_certificates')) archived_file_pattern = os.path.join(archive_folder, to_pattern) [shutil.move(filename, models.convert_file_name(archived_file_pattern, uid)) for filename, (uid,) in glob2.iglob(from_pattern, with_matches=True)]
def app_css(template_reference): """Generate app css link tags for Flask app.""" link_tags = "" # if False: if G.debug: # layout for layout_path in G.css_config['layout']: # 支持通配符 if '*' in layout_path: for path in glob2.iglob(os.path.join(G.static_path, layout_path)): relative_path = path.split(G.static_path)[1][1:] link_tags += link(relative_path) else: link_tags += link(layout_path) # page template_name = _get_template_name(template_reference) page_css_path = os.path.join(G.css_config['page'], template_name.replace('html', 'css')) link_tags += link(page_css_path) else: link_tags += link(APP_CSS) return Markup(link_tags)
def archive_files(from_pattern, to_pattern, timestamp): [shutil.copyfile(filename, convert_file_name(to_pattern, uid) + '-' + timestamp) for filename, (uid,) in glob2.iglob(from_pattern, with_matches=True)]
# construct the argument parse and parse the arguments ap = argparse.ArgumentParser() ap.add_argument("-i", "--input", required = True, help = "input directory of images") ap.add_argument("-o", "--output", required = True, help = "output directory") ap.add_argument("-c", "--csv", required = True, help = "path to CSV file for image counts") args = vars(ap.parse_args()) # open the output file for writing output = open(args["csv"], "w") # loop over the input images for imagePath in glob2.iglob(args["input"] + "/*/*.jpg"): # generate a random filename for the image and copy it to # the output location filename = str(uuid.uuid4()) + ".jpg" shutil.copy(imagePath, args["output"] + "/" + filename) # there is a 1 in 500 chance that multiple copies of this # image will be used if random.randint(0, 500) == 0: # initialize the number of times the image is being # duplicated and write it to the output CSV file numTimes = random.randint(1, 8) output.write("%s,%d\n" % (filename, numTimes)) # loop over a random number of times for this image to # be duplicated
def get(self, pattern): _folder = CleverString(self._folder).value _pattern = CleverString(pattern).value for item in glob2.iglob(str(f(_folder)/_pattern)): yield item
def eglob(path): """Like glob.glob, but supports "/path/**/{a,b,c}.txt" lookup""" fi = itertools.chain.from_iterable paths = expand_paths(path) return list(fi(glob2.iglob(d) for d in paths))