def __call__(self, path=None): if path is None or path == ".": l = os.ls() else: l = os.ls(path) for f in l: print(f)
def readUNIPENdB(location): weHave=ls(location) if 'data' in weHave: sdB=[] tagClassification={} folders=ls(location+'/data') for curFolder in folders: altFol=ls(location+'/data/'+curFolder) for curAltFold in altFol: if curAltFold=='aga': dataFiles=ls(location+'/data/'+curFolder+'/'+curAltFold) for curFile in dataFiles: fullPath=location+'/data/'+curFolder+'/'+curAltFold+'/'+curFile print fullPath+':' sdB=sdB+tds.mountDS(fullPath,'UNIPEN') for symbol in sdB: symbol.draw() psdB=spp.preprocessing(sdB) for symbol in psdB: symbol.computeFeatures() if symbol.tag not in tagClassification: tagClassification[symbol.tag]=[] tagClassification[symbol.tag].append(symbol) print type(tagClassification) return psdB,tagClassification
def check_all_dirs_have_rootfiles(n, rootfile_ext = ".root"): """Checks that there are no missing subdirs and each have a .root file""" print "check_all_dirs_have_rootfiles(%i)" % n # get list of numeric dirs from os import listdir as ls list_of_subdir_numbers = [int(sdir) for sdir in ls('.') if sdir.isdigit()] print len(list_of_subdir_numbers) print list_of_subdir_numbers # check there are no missing subdir numbers consecutive_numbers = range(max(list_of_subdir_numbers) + 1) print consecutive_numbers list_of_subdir_numbers.sort() if consecutive_numbers != list_of_subdir_numbers: print "There seems to be a problem with a subjob directory missing" return False print "All subdirectories exist." # check the presence of a rootfile for sdir in list_of_subdir_numbers: subdir_files = ls("%i/output/" % sdir) rootfiles = map(has_ext, subdir_files) if sum(rootfiles) == 0: print "Subdirectoy %i does not have a rootfile" % sdir return False if sum(rootfiles) > 1: print "Warning: subdirectoy %i has %i root files" % (sdir, sum(rootfiles)) print "All subdirectories have a root file." return True
def load_cib_short(imdir='../images'): ''' Returns nested dictionairy with category infos''' # animate_being, object, place baselvl = [join(imdir, item) for item in ls(imdir)] mapping = df.from_dict(\ df((((bname(top), len(loadimages(top))).__str__(), (dict( ((sub1, len(loadimages(join(top, sub1)))).__str__(),\ dict( ((sub2, len(loadimages(join(top, sub1, sub2)))).__str__(), dict( ((sub3, len(loadimages(join(top, sub1, sub2, sub3)))).__str__(),\ dict( (item, len(loadimages(join(top, sub1, sub2, sub3, item)))) for item in ls(join(top, sub1, sub2, sub3)))) for sub3 in ls(join(top,sub1,sub2)) )) for sub2 in ls(join(top, sub1)) ))\ for sub1 in ls(top)) )) for top in baselvl), dtype='object').set_index(0).transpose( ).to_dict()).transpose().to_dict()[1] return mapping
def load_objects(self, dirs=[], callwith={}): ''' Call this to load resources from each dir in @dirs. Code resources will receive @callwith as an argument. ''' for d in dirs: contents = ls(d) for t in contents: first = join(d, t) if t.startswith('.') or os.path.isfile(first): continue t_l = t.lower() for fl in ls(first): full = join(first, fl) if fl.startswith('.') or os.path.isdir(full): continue fl_n = fl.lower().rsplit('.', 1)[0] ty = guess_type(full) if ty == T_IMAGE: self.load_image(full, fl_n, t) elif ty == T_SOUND: self.load_sound(full, fl_n, t) elif ty == T_MUSIC: self.load_music(full, fl_n, t) elif ty == T_CODE and fl_n == '__init__': self.load_code(d, t, callwith) elif ty == T_PLAYLIST: self.load_playlist(full, fl_n)
def readUNIPENdB(location): weHave = ls(location) if 'data' in weHave: sdB = [] tagClassification = {} folders = ls(location + '/data') for curFolder in folders: altFol = ls(location + '/data/' + curFolder) for curAltFold in altFol: if curAltFold == 'aga': dataFiles = ls(location + '/data/' + curFolder + '/' + curAltFold) for curFile in dataFiles: fullPath = location + '/data/' + curFolder + '/' + curAltFold + '/' + curFile print fullPath + ':' sdB = sdB + tds.mountDS(fullPath, 'UNIPEN') for symbol in sdB: symbol.draw() psdB = spp.preprocessing(sdB) for symbol in psdB: symbol.computeFeatures() if symbol.tag not in tagClassification: tagClassification[symbol.tag] = [] tagClassification[symbol.tag].append(symbol) print type(tagClassification) return psdB, tagClassification
def join_per_year(root_folder=GDOT_SAVE_FOLDER): """ Uses all years images informations to make a info file with all csv. The function will read the images.csv files, join then and put the new file in the root_folder.s Arguments: root_folder: Folder with the year/month folders. """ infos = [] for year_folder in ls(root_folder): print(year_folder) year_folder = join(root_folder, year_folder) if not isdir(year_folder): continue image_info_file = join(year_folder, "images.csv") try: info = pd.read_csv(image_info_file) info = create_info_cols(info) except: print("image.csv from {} does not exists!".format(year_folder)) continue for month_folder in ls(year_folder): month_folder = join(year_folder, month_folder) if not isdir(month_folder): continue images = ls(month_folder) for image in images: image_filepath = join(month_folder, image) with Image.open(image_filepath) as img: width, height = img.size info = put_image_info(info, image, width=width, height=height) info.to_csv(image_info_file, index=False) infos.append(info) infos = pd.concat(infos, axis=0) infos.to_csv(join(root_folder, "images.csv"), index=False)
def inv_map(imdir=IMDIR): levelA = [join(IMDIR, item) for item in ls(IMDIR)] # animate_being, object, place mappin = df.from_records(df.from_dict(\ df.from_records((((bname(top), (df.from_records((((sub1, dict((sub2, df.from_records( ( ( pd.Index(pd.Series(col), name=col[0]).to_frame().set_index([[0, 1]]) for col in pd.Index(df.from_dict(dict((item, {'n_files':len(sorted(ls(join(top, sub1, sub2, sub3, item)))), 'files': sorted(ls(join(top, sub1, sub2, sub3, item)))}) for item in ls(join(top,sub1,sub2,sub3))), orient='index' ) ).to_frame().iteritems() ) for sub3 in ls(join(top,sub1,sub2)) ) ).set_index(pd.Index(ls(join(top,sub1,sub2)))) ) for sub2 in ls(join(top, sub1)))) for sub1 in ls(top)))).set_index(0).to_dict()[1])) for top in levelA))).set_index(0).transpose().to_dict()).transpose().to_dict()[1]).to_dict() return mappin
def search(region, root): files_in_lists = [search(f, path.join(root, f)) for f in ls(root) if path.isdir(path.join(root, f))] +\ [[(region, path.join(root, f)) for f in ls(root) if path.isfile(path.join(root, f))]] files = reduce(lambda a, b: a + b, files_in_lists) return files
def check_prereqs(): ''' Validate environment & dependencies exist for script ''' error_count = 0 print("Checking script environment...") compatible_os = regex(r'Windows') if not compatible_os.match(get_os()): eprint( "FAILED: This script only works on Windows with administrator privileges." ) error_count -= 1 else: try: # Check running as administrator (only Administrator rights can see the temp dir) ls(p.join(ossep, 'Windows', 'temp')) except PermissionError: eprint( '\n' + "FAILED: Script needs to be run with administrator privileges." ) eprint( "PowerShell w/ Admin Privalages command: \n\t Start-Process powershell -Verb RunAs" ) error_count -= 1 # TODO: check for required cygwin_pubring.asc public key file # VALID Environment if error_count != 0: return (error_count) else: print("PASS") return (0)
def windows_filenames(): """ As filenames are actually binary data we should be careful when our scripts read eg from a vfat filesystem. In this example we're creating 3 windows-encoded filenames like wuerstelstrasse.X.txt os.path functions fail mangling them as encoded strings so we ask it to mangle as bytes. moreover we use "{!r}".format to avoid further encoding issues in the printing part """ from os.path import join as pjoin from os import listdir as ls win = 'cp1252' prefix = "w\u00fcrstelstra\u00dfe" touch_encoded_filenames("/tmp/course", prefix, encoding=win) for f in ls(basedir): try: utf_path = pjoin(basedir, f) print("file: {!r}".format(utf_path)) except UnicodeDecodeError as e: print("Error decoding {!r}".format(f)) bytebasedir = bytes(basedir) for b in ls(bytebasedir): try: byte_path = pjoin(bytebasedir, b) print("file: {!r}".format(utf_path)) except UnicodeDecodeError as e: print("Error decoding {!r}".format(b))
def square_resize(fname, dimensions=(500, 500)): ''' Resizes square aspect-ratio images to desired dimensions. Doesn't overwrite the images; instead, a prefix corresponding to 'dimensions' parameter is added before each image's and folder's name (i.e: folder 'aquatic_mammal' --> '500_aquatic_mammal'). Parameters ---------- fname: type = str Name of category images directory (ex: 'outdoor_sport') dimensions: type = tuple Tuple (width, length) indicating desired size in pixels type(width) and type(length) = int 'width' & 'length' should be equal Returns ------- None''' imdirect.monkey_patch() #Fixes unexpected image rotation while saving prefix = str(dimensions[0]) + '_' nfpath = join(getcwd(), prefix + fname) os.system("mkdir {}".format(nfpath)) for categs in ls(join(getcwd(), fname)): newcatpath = join(nfpath, prefix + categs) os.system("mkdir {}".format(newcatpath)) for synname in ls(join(getcwd(), fname, categs)): os.system("mkdir {}".format(join(newcatpath, prefix + synname))) for allim in os.walk(join(getcwd(), fname, categs, synname)): for subd in allim[1]: os.system("mkdir {}".format( join(newcatpath, prefix + synname, prefix + subd))) for im_name in tqdm(allim[2]): impath = join(allim[0], im_name) image = Image.open(impath) newpath = impath.replace(fname, prefix + fname, 1) newpath = newpath.replace(categs, prefix + categs, 1) newpath = newpath.replace(synname, prefix + synname, 1) newpath = newpath.replace(bname(dname(newpath)), prefix + bname(dname(newpath)), 1) image.convert("RGB") if image.size[0] > image.size[1]: image = image.crop( ((image.size[0] - image.size[1]) / 2, 0, (image.size[0] + image.size[1]) / 2, image.size[1])) elif image.size[0] < image.size[1]: image = image.crop( (0, (image.size[1] - image.size[0]) / 2, image.size[0], (image.size[1] + image.size[0]) / 2)) im_resized = image.resize(dimensions, Image.ANTIALIAS) im_resized.save(join(dname(newpath), prefix + im_name), 'JPEG', quality=90)
def file_iterdir(relpath=None): cwd = getcwd() if relpath is not None: if relpath in ls(cwd): paths = join(cwd, relpath) else: files = ls(cwd) paths = [join(cwd, file) for file in files] return paths
def search(region, root): """ Finds all the subdirectories of a directory, and pairs them with the name of their parent directory. """ files_in_lists = [search(f, path.join(root, f)) for f in ls(root) if path.isdir(path.join(root, f))] +\ [[(region, path.join(root, f)) for f in ls(root) if path.isfile(path.join(root, f))]] return reduce(lambda a, b: a + b, files_in_lists)
def getConfigList(configListFilename, configDir): if configListFilename: if glob(configListFilename): with open(configListFilename, "r") as configFile: configList = configFile.read().splitlines() print "Configuration list read" else: print "WARNING: Given configuration does not exist. " \ + "Will use all configurations in configuration directory." configList = ls(configDir) configList = sorted(configList) with open(configListFilename, "w") as configFile: for config in configList: configFile.write(str(config) + "\n") print "Configuration list written" else: configList = ls(configDir) configList = sorted(configList) configFilename = "./confs.txt" if glob(configFilename): print "WARNING: Configuration list already exists in this directory. " \ + "Will not overwrite." else: with open(configFilename, "w") as configFile: for config in configList: configFile.write(str(config) + "\n") print "Configuration list written" print "Number of configurations: " + str(len(configList)) return configList
def get_size(self): sizes = {k:v for k,v in self.paths.items()} for k,v in sizes.items(): try: sizes[k] = sum([getsize(join(v, name)) for name in ls(v)]) except NotADirectoryError: sizes[k] = getsize(v) return sizes return sum([getsize(join(path, name)) for name in ls(path)])
def play(rootDir, playerPath=None): cd(rootDir) files = [i for i in ls('.') if not ignore in i or not (isdir(i) and not(ls(i)))] choice = files[rand(0, len(files)-1)] if isdir(choice): play(choice) else: if not playerPath: cmd("%s %s" %(getPlayer(), choice)) else: cmd("%s %s" %(playerPath, choice))
def _get_package_data(): """Iterates over the `init` dir for directories and returns all files within them. Only files within `binaries` and `templates` will be added. """ from os.path import join as jn from os import listdir as ls x = 'init' b = jn('serv', x) dr = ['binaries', 'templates'] return [jn(x, d, f) for d in ls(b) if d in dr for f in ls(jn(b, d))]
def parse(self, str_dir): frames_path = path.join(str_dir, 'frame') frames = dict() files_names = ls(frames_path) for count, file_name in enumerate(ls(frames_path)): if file_name.endswith('.xml'): file_path = path.join(frames_path, file_name) logger.info( '{percent:3d}% reading frame "{absolute}"'.format(percent=((count + 1) * 100) / len(files_names), absolute=file_name)) frame = self._fparser.parse(file_path) frames[frame.name] = frame return Net(frames)
def check_prereqs(): print("Checking script environment...") compatible_os = regex(r'Windows') if not compatible_os.match(get_os()): eprint( "FAILED: This script only works on Windows with cygwin installed.") exit(-1) else: try: # Check cygwin's bash.exe exist subprocess.check_call([ 'powershell.exe', 'try {{ if (-not ([System.IO.File]::Exists("{cygwin_path}"))) {{ throw "Error" }} }} catch {{ }};' .format(cygwin_path=p.join(dir_cygwin, 'bin', "bash.exe")) ], shell=False) # Check running as administrator (only Administrator rights can see the temp dir) ls(p.join(ossep, 'Windows', 'temp')) except subprocess.CalledProcessError: eprint("MISSING PREREQ: cygwin not found @ {}/".format(dir_cygwin)) exit(-2) except PermissionError: eprint( '\n' + "FAILED: Script needs to be run with administrator privileges." ) eprint( "PowerShell w/ Admin Privalages command: \n\t Start-Process powershell -Verb RunAs" ) exit(-3) # Check cygwin's package manager exists pkgmgnr = p.join(dir_cygwin, 'cyg-get', "cyg-pkg-mgnr.exe") if not p.isfile(pkgmgnr): eprint("MISSING PREREQ: cyg-pkg-mgnr.exe not found @ {}".format( p.join(dir_cygwin, 'cyg-get') + ossep)) if p.isfile( p.join( p.dirname(pkgmgnr), 'setup-x86{}.exe'.format("_64" if is_64bit else ""))): # HELPING: original download file found, not renamed. eprint( " Please rename setup-x86{}.exe to cyg-pkg-mgnr.exe and try again.\n" .format("_64" if is_64bit else "")) exit(-4) # VALID Environment print("PASS")
def __init__(self, **kwargs): super(Test, self).__init__(**kwargs) self._path = op.dirname(op.abspath(__file__)) try: if sys.argv[1] == '--folder': self.path = sys.argv[2] if self.path.startswith('\'') or self.path.startswith('"'): self.path = self.path[1:] if self.path.endswith('\'') or self.path.endswith('"'): self.path = self.path[:-1] self.path = self.path.strip('.py') self.modules = [] m = [f for f in ls(self.path) if isfile(op.join(self.path, f))] for mod in m: if (mod.startswith('test_') and mod.endswith('.py')): self.modules.append(mod.strip('.py')) else: self.path = self._path self.modules = [ 'test_text', ] except IndexError: self.path = self._path self.modules = [ 'test_text', ] self.outputs = []
def collect(fdr): """ collect encoder output in a folder. """ eot = [] hof = [] pcs = [] # __n = 0 for fname in sorted(ls(fdr)): if not fname.endswith('pgz'): continue # if not __n < 10: # break # __n = __n + 1 fname = pt.join(fdr, fname) print(fname) output = lpz(fname) eot.append(output['eot']) hof.append(output['hof']) if not isinstance(output['pcs'], Exception): pcs.append(output['pcs'][:, 0:16]) import numpy as np eot = np.array(eot, 'f') hof = np.array(hof, 'f') pcs = np.array(pcs, 'f') hof = np.transpose(hof, [1, 2, 0]) pcs = np.transpose(pcs, [1, 2, 0]) ret = {'eot': eot, 'hof': hof, 'pcs': pcs} return ret
def get_pages(self): """Sort source files by its created time, and then chunk all posts into 9 pages""" if not exists(Post.src_dir): logger.error(SourceDirectoryNotFound.__doc__) sys.exit(SourceDirectoryNotFound.exit_code) source_files = [join(Post.src_dir, fn) for fn in ls(Post.src_dir) if fn.endswith(src_ext)] posts = [] # all posts objects for filepath in source_files: try: data = parser.parse_filename(filepath) except ParseException as e: # skip single post parse exceptions logger.warn(e.__doc__ + ", filepath '%s'" % filepath) else: posts.append(Post(**data)) # sort posts by its created time, from new to old posts.sort(key=lambda post: post.datetime.timetuple(), reverse=True) # each page has 9 posts groups = chunks(posts, self.POSTS_COUNT_EACH_PAGE) pages = [Page(number=idx, posts=list(group)) for idx, group in enumerate(groups, 1)] # mark the first page and the last page if pages: # !important: Not empty list pages[0].first = True pages[-1].last = True return pages
def proband_pickles() -> Union[List[str], List]: """Returns a sorted list of probands IDs determined by contents of the proband data directory. """ return sorted( p.replace(".pkl", "") for p in ls(proband_path) if p.endswith(".pkl"))
def collect(fdr='.', nrt=None, out=None, csv=None): """ collect simulation report in a folder. """ fns = sorted(f for f in ls(fdr) if f.endswith('pgz')) # configuration, training history, and benchmarks cfg, hst, bmk = [], [], [] for i, f in enumerate(fns): if nrt is not None and not i < nrt: break f = pt.join(fdr, f) print(f) pgz = lpz(f) # 1) collect training history hs = pgz.pop('hst') hst.append(hs) # 2) collect simulation configuration cf = ['fam', 'xtp', 'frq', 'mdl', 'rsq', 'gdy', 'gtp'] cf = dict((k, v) for k, v in pgz.items() if k in cf) cf['nxp'] = '{}x{}'.format(pgz['gmx'].shape[0], pgz['gmx'].shape[2]) cf['nwk'] = pgz['dim'] cf = pd.Series(cf) cfg.append(cf) # 3) collect reference benchmarks, also append the performance of NNT bmk.append(pgz.pop('bmk').reset_index()) # concatenation _df = [] for c, b in zip(cfg, bmk): _df.append(pd.concat([pd.DataFrame([c] * b.shape[0]), b], 1)) bmk = pd.concat(_df) # non-NNT methods do not rely on these parameters bmk.loc[bmk.mtd != 'nnt', ['gtp', 'nwk', 'xtp']] = '-' # configuration keys and report keys cfk = cf.index.tolist() + ['mtd', 'par', 'key'] _gp = bmk.groupby(cfk) # means, stds, and iteration count of 'val' _mu = _gp.val.mean().rename('mu') _sd = _gp.val.std().rename('sd') _it = _gp.val.count().rename('itr') rpt = pd.concat([_mu, _sd, _it], 1).reset_index() rpt = rpt.loc[:, cfk + ['mu', 'sd', 'itr']] # do the same for training history hst = pd.concat(hst) _gp = hst.groupby('ep') _it = _gp.terr.count().rename('itr') hst = pd.concat([_gp.mean(numeric_only=True), _it], 1).reset_index() # save and return ret = Bunch(bmk=bmk, hst=hst, rpt=rpt) if out: spz(out, ret) if csv: rpt.to_csv(csv) return ret
def __init__(self, **kwargs): super(Test, self).__init__(**kwargs) self.path = [] self.modules = [] self.log_level = ['info', 'debug', 'trace'] # override --folder & --pythonpath args if running demo if parse.demo: parse.folder = [join(dirname(abspath(__file__)), 'examples')] parse.pythonpath = [dirname(abspath(__file__))] self.path.extend(parse.folder) # get files only from requested folders mods = [ f for path in self.path for f in ls(path) if isfile(join(path, f)) ] # unittest.TestLoader.loadTestsFromName doesn't want raw filenames for mod in mods: if (mod.startswith('test_') and mod.endswith('.py')): self.modules.append(mod.strip('.py')) # put desired folders with modules first if parse.pythonpath: self.path = parse.pythonpath + self.path
def workdir_view(request): # по аналогии с `time_view`, напишите код, # который возвращает список файлов в рабочей # директории current_list = ls(path=".") msg = f'Список файлов в рабочей директории: {current_list}' return HttpResponse(msg)
def __init__(self, **kwargs): super(Test, self).__init__(**kwargs) self.path = [] self.modules = [] self.log_level = ['info', 'debug', 'trace'] # override --folder & --pythonpath args if running demo if parse.demo: parse.folder = [ join(dirname(abspath(__file__)), 'examples') ] parse.pythonpath = [ dirname(abspath(__file__)) ] self.path.extend(parse.folder) # get files only from requested folders mods = [ f for path in self.path for f in ls(path) if isfile(join(path, f)) ] # unittest.TestLoader.loadTestsFromName doesn't want raw filenames for mod in mods: if (mod.startswith('test_') and mod.endswith('.py')): self.modules.append(mod.strip('.py')) # put desired folders with modules first if parse.pythonpath: self.path = parse.pythonpath + self.path
def list_sub_dir( directory: str, debug=False ): # lista subdiretórios e retorna lista com arquivos encontrados if debug: print('Listando diretórios...') try: for item in ls(directory): d = path.join(directory, item) if path.isdir(d): if debug: print('Pasta encontrada') list_sub_dir(d, debug) else: if debug: print('Arquivo Encontrado') items.append(str(d)) else: if len(items) == 0: print('Não encontrados arquivos') exit(3) except PermissionError: if debug: print('Permissão Negada à Pasta') exit(1) except Exception as e: if debug: print('Erro não tratado list_sub_dir: ' + str(e)) exit(2)
def get_files(self, directory=getcwd(), **definitions): """ Returns list of files which correpond to the keyword arguments. Parameters ---------- directory : string Files directory. (default: current working directory) Optional keyword arguments -------------------------- definitions : float, int or bool Defined parameters (e.g., density=0.1). Returns ------- files : list of strings List of files correponding to parameters. """ length = self.filename_length() # length of filename name_parts = self.filename(**definitions) # list of defined name parts return [ file for file in ls(directory) if all(part in file for part in name_parts) and len(file) == length ]
def get_samples_in_order(directory: str) -> List: """ Get all files corresponding to a sample in a directory. Return value is a list of file names ordered starting with sample 1, then sample 2, then sample 3 and so on... """ # get all files in directory (this will collect even non-csv files) data_files = ls(directory) """ Build a list of file names that correspond to samples. Samples are files containing the string Potentiostatic EIS. """ sample_files = [] for file_name in data_files: if "Potentiostatic EIS" in file_name: sample_files.append(path.join(directory, file_name)) """ Sort samples, this will order the files by their index So, we get that the first element is the file starting with 2__, the second is the file starting with 4__, the third is the file starting with 6__ and so on """ sample_files = sorted(sample_files) return sample_files
def fs_migrations(self): files = [ f[:-3] for f in ls(self.migrate_dir) if self.filemask.match(f) ] return sorted(files, key=lambda f: int(f.split('_')[0]))
def __del__(self): try: tmp = ls('compDicts') except: mkdir('compDicts') with open('compDicts/'+self.type+'dict','wb+') as out: out.write(self.dictData)
def lsmsm(subd, rrtype): ps_all = [] msm = [] bad = 0 if subd is None: subd = 'nicolas' for fn in ls(outdir): if not fn.startswith(subd + '-' + rrtype.lower()): continue fo = open(outdir + fn, 'r') print 'Opening ' + fn + '...' fps = dpkt.pcap.Reader(fo) for _, pkt in fps: try: # Rule out bad packets ipkt = dpkt.ethernet.Ethernet(pkt).data p = { 'src': ipkt.src, 'dst': ipkt.dst, 'udp': ipkt.p == 17, 'dns': dpkt.dns.DNS(ipkt.data.data) } t = p['dns'].qd[0].name.lower() if (subd == 'nicolas' and t.endswith(dn)) or t.endswith(subd + '.' + dn): ps_all += [p] except (IndexError, AttributeError, dpkt.UnpackError, dpkt.NeedData): bad += 1 msm += [int(fn.split('-')[2].split('.')[0])] print 'Closing ' + fn + '...' fo.close() if bad > 0: print str(bad) + ' bad packets' return ps_all, set(msm)
def genRandomSequence(numDoms): """ Generates a random zf-C2H2 protein sequence with the given number of domains. Includes sequence before and after zf domain. """ files = ls(DATAPATH) f = list(open(DATAPATH + choice(files)))[1::2] sequence = choice(f).strip() sequence.translate(None, '-') starts, ends, seqs = findDomains(sequence, hmmfile) if len(starts) < numDoms: return genRandomSequence(numDoms) prefix = sequence[:starts[0]] suffix = sequence[ends[-1]:] if prefix == '' or suffix == '': return genRandomSequence(numDoms) linkers = [] for i in range(len(starts) - 1): linkers.append(sequence[ends[i] + 1:starts[i + 1]]) middle = '' for _ in range(numDoms - 1): middle += choice(seqs) + choice(linkers) middle += choice(seqs) newSeq = prefix + middle + suffix newSeq = ''.join(newSeq.split('-')) #Deletes all lowercase letters newSeq = newSeq.translate(None, string.ascii_lowercase) #Deletes all illegal AA characters newSeq = newSeq.translate(None, 'BJOUXZ') return newSeq
def readCROHMEdB(locationList): #Fa una llista amb els arxius a extreure la base de dades i ho ordena al sistema sdB = [] tagClassification = {} os.remove('toTrain.txt') trainListFile = open('toTrain.txt', 'w') for destiny in locationList: filesInkML = ls(destiny) for filenom in filesInkML: if '.inkml' in filenom: trainListFile.write(destiny + '/' + filenom + '\n') trainListFile.close() trainListFile = open('toTrain.txt', 'r') linea = 'o' lcont = 0 while linea != '': lcont += 1 linea = trainListFile.readline() if linea != '': if os.path.exists('segmentedData/' + 'Folder000' + str(lcont)): shutil.rmtree('segmentedData/' + 'Folder000' + str(lcont)) os.makedirs('segmentedData/' + 'Folder000' + str(lcont)) sg.main([ 'GO', linea, 'segmentedData/' + 'Folder000' + str(lcont) + '/trainFile' ]) #Escaneja els arxius de la base de dades i guarda les coordenades for cases in os.listdir('segmentedData'): filesHere = sorted(os.listdir('segmentedData/' + cases)) filesHere.remove('trainFile_GT.txt') filesHere = [ 'trainFile' + str( sorted([ int(filesHere[i].strip('trainFile').strip('.inkml')) for i in range(len(filesHere)) ])[j]) + '.inkml' for j in range(len(filesHere)) ] Gfile = open('segmentedData/' + cases + '/trainFile_GT.txt') linea = 'o' filco = 0 while linea != '': linea = Gfile.readline() if linea != '': whereInd = linea.index(',') etiq = linea[whereInd + 1:].rstrip('\n') sdB.append( ink2Traces.i2trained( 'segmentedData/' + cases + '/' + filesHere[filco], etiq)) filco += 1 #Preprocessa i troba els atributs dels simbols de la base de dades for symbol in sdB: symbol.draw() psdB = spp.preprocessing(sdB) for symbol in psdB: symbol.computeFeatures() if symbol.tag not in tagClassification: tagClassification[symbol.tag] = [] tagClassification[symbol.tag].append(symbol) return psdB, tagClassification
def __init__(self, kernel, agent, params=None): self.agent = agent self.input_link_wme = agent.GetInputLink() self.structure_wme = agent.CreateIdWME(self.input_link_wme, "structure") self.request_wme = agent.CreateIdWME(self.input_link_wme, "request") self.request_path_wme = agent.CreateIdWME(self.request_wme, "path") self.feedback_wme = agent.CreateIdWME(self.input_link_wme, "feedback") self.status_wme = None if "corpus-path" in params: WSD.CORPUS_PATH = params["corpus-path"] self.sentence_files = [f for f in ls(WSD.CORPUS_PATH) if re.match("[0-9]*$", f)] #self.sentence_files = self.sentence_files + self.sentence_files if "max-sentences" in params and params["max-sentences"] > 0: self.sentence_files = self.sentence_files[:params["max-sentences"]] #self.sentence_files = ["0042", "0047", "0057"] self.cur_file_index = 0 self.cur_tuples = [] self.cur_word = 0 self.node_map = {"0":self.structure_wme} self.structure_wmes = [] self.attr_map = {} self.request_wmes = [] self.feedback_wmes = [] self.answer = None self.path = "" self.responded = False self.last_response = None self.last_answer = None self.last_source = None self.last_epmem_recog = None self.last_smem_recog = None # register for events soar_exp.register_output_callback(kernel, agent, output_callback, self) agent.Commit()
def write_file(name, contents, override=False): if override == True or name[:-3] not in ls(get_directory()): if not isdir(os.path.join(get_directory(), name[:-3])): mkdir(os.path.join(get_directory(), name[:-3])) with open(os.path.join(get_directory(), name[:-3], name), 'wb') as f: f.write(contents) return True return False
def main(dirpath): out = {"TOTAL": set()} driter = (f for f in ls(dirpath) if f.endswith('.csv')) for fl in driter: token_set = getSet(fl) out[f] = token_set out["TOTAL"].add(token_set) writeMetaDataToXml(out)
def add_label_to_pictures(directory): for file in ls(directory): print(file) if file.lower().endswith("jpg") or file.lower().endswith("jpeg"): complete_filename = directory + "/" + file if (show_and_treat(complete_filename)): return clear_output()
def _get_aud_stim_paths(s: str) -> dict: """Returns a dictionary containing whose keys are the file names of the values are absolute paths to auditory stimuli for test `s`.""" p = pj(aud_stim_path, s) try: return {n: pj(p, n) for n in ls(p) if n.endswith(".wav")} except FileNotFoundError: return {}
def get_month_num(month_suffix='_months.csv'): file_list = ls('.') month_csvs = [f_name for f_name in file_list if f_name.endswith(month_suffix)] month_nums = [int(str(month_file[:-len(month_suffix)])) for month_file in month_csvs] return np.max(month_nums), month_nums, month_csvs
def get_solved(): from os import listdir as ls from os.path import isfile from os import path import re p = config['directory'] solved = [i.split('.')[0] for i in ls(p) if isfile(path.join(p, i))] solved = [re.sub("^\d+", re.findall("^\d+", i)[0] + '/', i) for i in solved] return solved
def list_sub_dir(directory: str): items = [] for item in ls(directory): d = path.join(directory, item) if path.isdir(d): list_sub_dir(d) else: items.append(d) return items
def RAW(nodes): protos = { ATWNetwork: ATWNode, } #nodes = [i * 10 ** pow for pow in range(1, 4) for i in range(1, 6)] numTrials = 30 for numNodes in nodes: if 'ATW%d.pkl' % numNodes not in ls('.'): results = runAllProtos(protos, [numNodes], numTrials) with open('ATW%d.pkl' %numNodes, 'wb') as logfile: pickle.dump(results, logfile)
def Gather(self): if self.ZipObject == None: return 1 #ZipFileObject is NULL try: for files in ls(self.dir): self.ZipObject.write(self.dir + files) except: return 2 #ZipObject cannot add files to archive self.ZipObject.close() #Close ZipObject return 0
def get_vars(): path = "less/" nw_path = "static/css/" paths=[path] # Need to fix path = fix_path(path) nw_path = fix_path(nw_path) Lakefile, config, ignore = get_file() paths_l = [] for n in range(len(paths)): if n == 0: paths_l.append(paths[n]) elif n + 1 == len(paths): paths_l.append("and " + paths[n]) else: paths_l.append(", " + paths[n]) lesses = [i for o in [[(path, (f[:2] if f.startswith("./") else f)) for f in ls(path) if f.endswith('.less')] for path in paths] for i in o] files = [] for less in lesses: if less not in ignore: old_file = "/".join(less) nw_file = nw_path + (less[1][:-5] if less[1].endswith('.less') else less[1]) nw_file += ".css" files.append((old_file, nw_file)) parser = OptionParser() parser.add_option("-o", "--orign", dest="orign", default=path, help="Orign Folder of less files", metavar="FILE") parser.add_option("-d", "--destination", dest="dest", default=nw_path, help="Destination Folder for less files", metavar="FILE") parser.add_option("-f", "--force", action="store_true", dest="force", default=False, help="ignore warnings") parser.add_option("-q", "--quite", action="store_true", dest="quite", default=False, help="suppress output") force = parser.parse_args()[0].force quite = parser.parse_args()[0].quite nw_path = parser.parse_args()[0].dest paths=[path] # Need to fix return vars()
def glob_zh_src_files(cls): """Glob zh source files return filepath to name dict""" if not exists(Post.zh_src_dir): raise SourceDirectoryNotFound dct = {} for fn in ls(Post.zh_src_dir): if fn.endswith(src_ext): name = fn[:-len(src_ext)] path = join(Post.zh_src_dir, fn) dct[path] = name return dct
def update(): db = TinyDB("data/db.json") cur = './Archive' problems = [] for f in ls(cur): if f in ignore: continue if f.startswith('.'): continue if not path.isdir(cur + '/' + f): continue # print 'find round: %s' % f for problem_name in ls(cur + '/' + f): problems.append(problem_name.split('.')[0]) counter = 0 for p in problems: q = Query() dp = db.search(q.name == p) if (len(dp) > 0 and dp[0]['solved'] is False): print 'db update, problem :`%s` solved' % p db.update({'solved': True}, (q.name == p)) counter += 1 print '>>>> %d rows have been updated.' % counter
def __init__(self, feat, folder, codes): self.codes = codes self.book = {} self.train_set = [] self.features=feat for item in ls(folder): self.book[item] = ls(folder + "/" + item) self.doc_count = len(self.book[self.codes.keys()[0]]) for i in range(self.doc_count): for item in self.book: f = open(folder + "/" + item + "/" + self.book[item][i]) text = f.read() vector = [] for feature in feat.feature_list: vector.append(feat.vectorize(feature, text)) vector.append(self.codes[item]) self.train_set.append(vector) f.close()
def vcf2npz(vcf='../data/1kg_gpk'): """ cache input genomic data in zipped numpy format. """ if pt.isdir(vcf): vcfs = [pt.join(vcf, f) for f in ls(vcf) if f.endswith('vcf.gz')] else: vcfs = vcf if isinstance(vcf, list) else [vcf] for vcf in vcfs: npz = vcf.replace('vcf.gz', 'npz') if pt.exists(npz): print(vcf, npz, 'exists.') continue print(vcf, npz, 'to cache.') gmx, sbj = loadVCF(vcf) np.savez_compressed(npz, gmx=gmx, sbj=sbj)
def get_posts(): posts = [Post(fn) for fn in ls(Post.src_dir) if fn.endswith(se)] # parse posts and update posts' attr for post in posts: c = open(post.src).read().decode(charset) dct = parse(c) dct.setdefault("tags", []) dct.setdefault("title", "Untitled") # update post's attrs with parsed content post.__dict__.update(dct) # cast post.tags = set(post.tags) return posts
def readCROHMEdB(locationList): #Fa una llista amb els arxius a extreure la base de dades i ho ordena al sistema sdB=[] tagClassification={} os.remove('toTrain.txt') trainListFile=open('toTrain.txt','w') for destiny in locationList: filesInkML=ls(destiny) for filenom in filesInkML: if '.inkml' in filenom: trainListFile.write(destiny+'/'+filenom+'\n') trainListFile.close() trainListFile=open('toTrain.txt','r') linea='o' lcont=0 while linea!='': lcont+=1 linea=trainListFile.readline() if linea!='': if os.path.exists('segmentedData/'+'Folder000'+str(lcont)): shutil.rmtree('segmentedData/'+'Folder000'+str(lcont)) os.makedirs('segmentedData/'+'Folder000'+str(lcont)) sg.main(['GO',linea,'segmentedData/'+'Folder000'+str(lcont)+'/trainFile']) #Escaneja els arxius de la base de dades i guarda les coordenades for cases in os.listdir('segmentedData'): filesHere=sorted(os.listdir('segmentedData/'+cases)) filesHere.remove('trainFile_GT.txt') filesHere=['trainFile'+str(sorted([int(filesHere[i].strip('trainFile').strip('.inkml')) for i in range(len(filesHere))])[j])+'.inkml' for j in range(len(filesHere))] Gfile=open('segmentedData/'+cases+'/trainFile_GT.txt') linea='o' filco=0 while linea!='': linea=Gfile.readline() if linea!='': whereInd=linea.index(',') etiq=linea[whereInd+1:].rstrip('\n') sdB.append(ink2Traces.i2trained('segmentedData/'+cases+'/'+filesHere[filco],etiq)) filco+=1 #Preprocessa i troba els atributs dels simbols de la base de dades for symbol in sdB: symbol.draw() psdB=spp.preprocessing(sdB) for symbol in psdB: symbol.computeFeatures() if symbol.tag not in tagClassification: tagClassification[symbol.tag]=[] tagClassification[symbol.tag].append(symbol) return psdB,tagClassification
def f(type): global counter if isfile('images/' + type): return files = ls('images/' + type) counter2 = 0 for name in files: counter += 1 counter2 += 1 f = open("_urunler/%.3d.md" % counter, 'w') f.write('---\n') f.write('tip: %s\n' % type) f.write('referans_ismi: %d\n' % counter2) f.write('dosya_ismi: %s\n' % name) f.write('---\n') f.close()
def get_posts(self): if not exists(Post.src_dir): logger.error(SourceDirectoryNotFound.__doc__) sys.exit(SourceDirectoryNotFound.exit_code) source_files = [join(Post.src_dir, fn) for fn in ls(Post.src_dir) if fn.endswith(src_ext)] for filepath in source_files: try: data = parser.parse_filename(filepath) except ParseException as e: # skip single post parse exception logger.warn(e.__doc__ + ', filepath: %r' % filepath) else: self.posts.append(Post(**data)) # sort posts by its created time, from new to old self.posts.sort(key=lambda post: post.datetime.timetuple(), reverse=True)
def get_files_stat(self): """get source files' update time""" if not exists(Post.src_dir): logger.error(SourceDirectoryNotFound.__doc__) sys.exit(SourceDirectoryNotFound.exit_code) paths = [] for fn in ls(Post.src_dir): if fn.endswith(src_ext): paths.append(join(Post.src_dir, fn)) # config.toml if exists(config.filepath): paths.append(config.filepath) # files: a <filepath to updated time> dict files = dict((p, stat(p).st_mtime) for p in paths) return files
def __init__(self, Distribution): self.dist = Distribution self.dist_dict = self.dist.__dict__ self.fancy_name = self.dist.project_name self.name = self.dist.key self.project_name = self.dist.project_name self.location = self.dist.location self.version = self.dist.version self.path = join(self.location, self.name) self.stats = os.stat(self.path) self.modified_last = self.stats.st_mtime self.meta_change = self.stats.st_ctime self.last_accessed = self.stats.st_atime if os.path.isdir(self.path): self.size = sum([getsize(join(self.path, name)) for name in ls(self.path)]) elif os.path.isfile(self.path): self.size = getsize(self.path) else: self.size = None self.fmt = time.strftime("%B %d, %Y %H:%M:%S", time.gmtime(self.meta_change))
def parse_posts(self, sender): """Parse and sort posts""" if not exists(Post.src_dir): logger.error(SourceDirectoryNotFound.__doc__) sys.exit(1) files = [] for fn in ls(Post.src_dir): if fn.endswith(src_ext): files.append(join(Post.src_dir, fn)) for filepath in files: try: data = parser.parse_file(filepath) except ParseException, e: logger.warn(e.__doc__ + ": filepath '%s'" % filepath) pass else: self.posts.append(Post(**data))
def finish(dirname): fnames = [] for f in ls(dirname): if f.split('.')[-1] == 'cpp': fnames.append(f) if not path.exists('./Archive/%s' % dirname): pathname = './Archive/%s' % dirname makedirs(pathname) print '>>>> create directory:[%s]' % pathname db = TinyDB("data/db.json") for f in fnames: oldpath = '%s/%s' % (dirname, f) newpath = './Archive/%s/%s' % (dirname, f) move(oldpath, newpath) print '>>>> move [%s] to [%s]' % (oldpath, newpath) q = Query() pname = path.splitext(f)[0] if db.search(q.name == pname): db.update({'solved': True}, (q.name == pname)) print '>>>> db update, problem: `%s` solved' % pname rmtree(dirname)