def create(dataset, work_dir): # Find all the pages in the dataset img_dir = Path(Path.cwd().ancestor(1), 'data/hwr_data/pages', dataset) ann_dir = Path(Path.cwd().ancestor(1), 'data/charannotations') images = img_dir.listdir('*.jpg') annotations = ann_dir.listdir(dataset + '*.words') files = merge(images, annotations) # Create character segmentations stats = {} for f in files: # Preprocess logging.info("Preprocessing %s", str(f[0])) pagesPathFolder = Path(work_dir, 'pages') pagesPathFolder.mkdir() pagePath = Path(pagesPathFolder, f[0].stem + '.ppm') img = cv2.imread(f[0], cv2.IMREAD_GRAYSCALE) img = preprocess(img) cv2.imwrite(pagePath, img) # Segment segmentPathFolder = Path(work_dir, 'segments') segmentPathFolder.mkdir() e = ET.parse(f[1]).getroot() logging.info("Segmenting %s", str(f[0])) segment(img, e, segmentPathFolder, stats) print_statistics(stats, dataset)
def get(project=None): """Get the data from different experiments. Warning! Experiment directories are expected in a particular location outside of this (words-in-transition) directory. Options are: telephone-app acoustic-similarity learning-sound-names """ if project is None or project == 'telephone-app': app_dir = Path('../telephone-app') snapshot_dir = Path(app_dir, 'words-in-transition') if src_dir.exists(): src_dir.rmtree() copytree(snapshot_dir, src_dir) if project is None or project == 'acoustic-similarity': # src proj_dir = Path('../acoustic-similarity/data') judgments = Path(proj_dir, 'judgments') # dst acoustic_similarity_dir = Path(data_raw, 'acoustic-similarity') if not acoustic_similarity_dir.isdir(): acoustic_similarity_dir.mkdir() # copy the csvs in the root proj data dir for csv in proj_dir.listdir('*.csv'): csv.copy(Path(acoustic_similarity_dir, csv.name)) # concat and save judgments files judgments_csv = Path(acoustic_similarity_dir, 'judgments.csv') judgments = [pd.read_csv(x) for x in judgments.listdir('*.csv')] if judgments: (pd.concat(judgments, ignore_index=True).to_csv(judgments_csv, index=False)) if project is None or project == 'learning-sound-names': src = Path('../learning-sound-names/data') dst = Path(data_raw, 'learning_sound_names.csv') data = pd.concat([pd.read_csv(x) for x in src.listdir('LSN*.csv')]) data['is_correct'] = data.is_correct.astype(int) data.to_csv(dst, index=False) # also get subject info and questionnaire data to_get = ['questionnaire_v1', 'subject_info'] for x in to_get: src_file = Path(src, '{}.csv'.format(x)) dst_file = Path(data_raw, 'learning_sound_names_{}.csv'.format(x)) run('cp {} {}'.format(src_file, dst_file))
def get(project=None): """Get the data from different experiments. Warning! Experiment directories are expected in a particular location outside of this (words-in-transition) directory. Options are: telephone-app acoustic-similarity learning-sound-names """ if project is None or project == 'telephone-app': app_dir = Path('../telephone-app') snapshot_dir = Path(app_dir, 'words-in-transition') if src_dir.exists(): src_dir.rmtree() copytree(snapshot_dir, src_dir) if project is None or project == 'acoustic-similarity': # src proj_dir = Path('../acoustic-similarity/data') judgments = Path(proj_dir, 'judgments') # dst acoustic_similarity_dir = Path(data_raw, 'acoustic-similarity') if not acoustic_similarity_dir.isdir(): acoustic_similarity_dir.mkdir() # copy the csvs in the root proj data dir for csv in proj_dir.listdir('*.csv'): csv.copy(Path(acoustic_similarity_dir, csv.name)) # concat and save judgments files judgments_csv = Path(acoustic_similarity_dir, 'judgments.csv') judgments = [pd.read_csv(x) for x in judgments.listdir('*.csv')] if judgments: (pd.concat(judgments, ignore_index=True) .to_csv(judgments_csv, index=False)) if project is None or project == 'learning-sound-names': src = Path('../learning-sound-names/data') dst = Path(data_raw, 'learning_sound_names.csv') data = pd.concat([pd.read_csv(x) for x in src.listdir('LSN*.csv')]) data['is_correct'] = data.is_correct.astype(int) data.to_csv(dst, index=False) # also get subject info and questionnaire data to_get = ['questionnaire_v1', 'subject_info'] for x in to_get: src_file = Path(src, '{}.csv'.format(x)) dst_file = Path(data_raw, 'learning_sound_names_{}.csv'.format(x)) run('cp {} {}'.format(src_file, dst_file))
class JarBuilder(object): """Holds the information needed for building a Java Archive (`.jar`) file. """ def __init__(self, jarfile, sourcedir, tsa): self.libjars = [] self.tsa = tsa self.jarfile = Path(jarfile) self.sourcedir = Path(sourcedir) self.sources = list(self.sourcedir.listdir('*.java')) self.jarcontent = [Path('Manifest.txt')] self.jarcontent += [ Path(x) for x in self.sourcedir.listdir('*.class')] def add_lib(self, pth): self.libjars.append(Path(pth)) def build_jar(self, ctx, outdir, alias): flags = '-storepass "`cat ~/.secret/.keystore_password`"' if self.tsa: flags += ' -tsa {0}'.format(self.tsa) def run_signer(jarfile): ctx.run("jarsigner %s %s %s" % (flags, jarfile, alias), pty=True) ctx.run("jarsigner -verify %s" % jarfile, pty=True) outdir = Path(outdir) jarfile = outdir.child(self.jarfile) if jarfile.needs_update(self.jarcontent): jarcontent = [x.replace("$", r"\$") for x in self.jarcontent] ctx.run("jar cvfm %s %s" % (jarfile, ' '.join(jarcontent)), pty=True) run_signer(jarfile) for libfile in self.libjars: jarfile = outdir.child(libfile.name) if not jarfile.exists() or libfile.needs_update([jarfile]): libfile.copy(jarfile) run_signer(jarfile) def build_classes(self, ctx): flags = "-Xlint:unchecked" if len(self.libjars): cp = ':'.join(self.libjars) flags += " -classpath %s" % cp for src in self.sources: ctx.run("javac %s %s" % (flags, src), pty=True)
def get_local(code, ktype='d', path='.'): """get last file and demo Parameters ---------- code : str ktype : {'d', '5', '15', '30', '60', 'w', 'm'} path : str where to search file Returns ------- filename_demo : filename of demo """ path = Path(path) pattern = CONSTS.filename_sep.join([ktype, CONSTS.demo]) pattern = '%s%s*%s%s' % (code, CONSTS.filename_sep, CONSTS.filename_sep, pattern) file_list = path.listdir(pattern, names_only=True) if len(file_list) == 1: return file_list[0] elif not file_list: # empty means False return warn('NO ITEM') else: raise Exception('MORE THAN 1 ITEM')
class JarBuilder(object): """Holds the information needed for building a Java Archive (`.jar`) file. """ def __init__(self, jarfile, sourcedir, tsa): self.libjars = [] self.tsa = tsa self.jarfile = Path(jarfile) self.sourcedir = Path(sourcedir) self.sources = list(self.sourcedir.listdir('*.java')) self.jarcontent = [Path('Manifest.txt')] self.jarcontent += [Path(x) for x in self.sourcedir.listdir('*.class')] def add_lib(self, pth): self.libjars.append(Path(pth)) def build_jar(self, ctx, outdir, alias): flags = '-storepass "`cat ~/.secret/.keystore_password`"' if self.tsa: flags += ' -tsa {0}'.format(self.tsa) def run_signer(jarfile): ctx.run("jarsigner %s %s %s" % (flags, jarfile, alias), pty=True) ctx.run("jarsigner -verify %s" % jarfile, pty=True) outdir = Path(outdir) jarfile = outdir.child(self.jarfile) if jarfile.needs_update(self.jarcontent): jarcontent = [x.replace("$", r"\$") for x in self.jarcontent] ctx.run("jar cvfm %s %s" % (jarfile, ' '.join(jarcontent)), pty=True) run_signer(jarfile) for libfile in self.libjars: jarfile = outdir.child(libfile.name) if not jarfile.exists() or libfile.needs_update([jarfile]): libfile.copy(jarfile) run_signer(jarfile) def build_classes(self, ctx): flags = "-Xlint:unchecked" if len(self.libjars): cp = ':'.join(self.libjars) flags += " -classpath %s" % cp for src in self.sources: ctx.run("javac %s %s" % (flags, src), pty=True)
def __init__(self, path): try: path = Path(path) self.__parents__ = path self.__contents__ = [d.name for d in path.listdir()] except: pass
def load_sounds(sound_dir, pattern): sound_dir = Path(sound_dir) sound_names = sound_dir.listdir(pattern=pattern) sounds = {} for snd_name in sound_names: snd_stem = str(snd_name.stem) sounds[snd_stem] = sound.Sound(snd_name) return sounds
def install_skeletton(self): logger.info('Installing %s' % self.flavor) source = Path(self.repo_dir, self.flavor) #move all items in the directory into the install_path for item in source.listdir(): item.move(self.install_path) self.delete_tmp_dir() logger.info('...done')
def find_labels(): files = {} work_dir = Path("tmp/segments") if not work_dir.exists(): raise Exception("You must first create the labels") files = {} labels = work_dir.listdir(filter=DIRS_NO_LINKS) for label in labels: files[str(label.name)] = label.listdir(pattern='*.ppm') return files
def dump_path(path, prefix="", tab=" ", file=None): if file is None: file = sys.stdout p = Path(path) if p.islink(): print >>file, "%s%s -> %s" % (prefix, p.name, p.read_link()) elif p.isdir(): print >>file, "%s%s:" % (prefix, p.name) for p2 in p.listdir(): dump_path(p2, prefix+tab, tab, file) else: print >>file, "%s%s (%d)" % (prefix, p.name, p.size())
def dream_on_dir(images_dir, out_dir): images_dir = Path(images_dir) out_dir = Path(out_dir) if not out_dir.exists(): os.mkdir(out_dir) for image in images_dir.listdir(): config = NightmareConfig(out_dir=out_dir) config.force_all_random() deep_dream(image, config) for d in config.list_output_for_image(image): print(" New dream: {}".format(d))
class JarBuilder(object): """ Used by my Java projects :ref:`davlink` and :ref:`eidreader`. """ def __init__(self, jarfile, sourcedir): self.jarfile = Path(jarfile) self.sourcedir = Path(sourcedir) self.sources = list(self.sourcedir.listdir('*.java')) self.jarcontent = [Path('Manifest.txt')] self.jarcontent += list(self.sourcedir.listdir('*.class')) self.jarcontent = [ Path(x.replace("$", "\\$")) for x in self.jarcontent] self.libjars = [] def add_lib(self, pth): self.libjars.append(Path(pth)) def build_jar(self, outdir, alias): flags = '-storepass "`cat ~/.secret/.keystore_password`"' flags += ' -tsa http://timestamp.globalsign.com/scripts/timestamp.dll' outdir = Path(outdir) jarfile = outdir.child(self.jarfile) if jarfile.needs_update(self.jarcontent): local("jar cvfm %s %s" % (jarfile, ' '.join(self.jarcontent))) local("jarsigner %s %s %s" % (flags, jarfile, alias)) for libfile in self.libjars: jarfile = outdir.child(libfile.name) if libfile.needs_update([jarfile]): libfile.copy(jarfile) local("jarsigner %s %s %s" % (flags, jarfile, alias)) def build_classes(self): flags = "-Xlint:unchecked" if len(self.libjars): cp = ':'.join(self.libjars) flags += " -classpath %s" % cp for src in self.sources: local("javac %s %s" % (flags, src))
def __init__(self, path): self.__parents__ = None self.__contents__ = None try: path = Path(path) self.__parents__ = path self.__contents__ = sorted([dir_.absolute() for dir_ in path.listdir() if dir_.isdir() or dir_.isfile()], key=lambda v: parse_version(v)) except: pass
def parse(path='./flat_files/'): path = Path(path) print "parsing records at {}".format(path.absolute()) records = [] for p in path.listdir(): try: gbr = GenBank.read(open(p)) records.append(gbr) except: print 'error with file', p print "parsed %s records.." % len(records) return records
def cli(darknet_dir, images_dir, temp_dir): darknet_dir = Path(darknet_dir) images_dir = Path(images_dir) temp_dir = Path(temp_dir) if not temp_dir.exists(): print('Dir "{}" does not exist'.format(temp_dir)) return with multiprocessing.Pool(PROCESSES) as pool: args = [] for image_file in images_dir.listdir(): args.append((image_file, temp_dir, darknet_dir)) pool.starmap(yolo_detect_image_task, args)
def recursive_listdir(path): """ Lists recursively all files inside a given directory and its subdirectories :param path: (unipath.Path or string) path to the directory :return: (list of unipath.Path) list containing the path to all the files inside the given directory """ folder = Path(path) files = list() for f in folder.listdir(): if f.isdir(): files.extend(recursive_listdir(f)) else: files.append(f.absolute()) return files
def __init__(self, path): self.__parents__ = None self.__contents__ = None try: path = Path(path) self.__parents__ = path self.__contents__ = sorted([ dir_.absolute() for dir_ in path.listdir() if dir_.isdir() or dir_.isfile() ], key=lambda v: parse_version(v)) except: pass
def parse_auth_log(log_folder='/var/log/'): directory = Path(log_folder) for auth_file in directory.listdir('auth*.*', FILES): if auth_file.ext == '.gz': auth = gzip.open(auth_file, 'rb') else: auth = open(auth_file, 'rU') prev_id = '' attempt = None for line in auth.readlines(): line = line.strip() # Skip sudo entries if line.find('sudo') >= 0: continue # Grab the date/time of the attempt and remove that from line attempted, line = get_date_from_line(line) # Grab the attempt ID attempt_id = get_id_from_line(line) # Is this a dup? if SshHackAttempt.objects.filter(ssh_id=attempt_id).count() > 0: continue # Is this a new attempt? new_attempt = prev_id != attempt_id if new_attempt: if getattr(attempt, 'has_all_keys', None): # Store the previous attempt and start a new one attempt.save() attempt = AuthAttempt() attempt.ssh_id = attempt_id attempt.attempted = attempted # Grab the IP address attempt.set_ip_from_line(line) # Is the username in this line? attempt.set_username_from_line(line) # Does the ID need updated? if prev_id != attempt_id: prev_id = attempt_id auth.close()
def convert_pdfs(pdfs_dir, output_dir): output_dir = Path(output_dir) if not output_dir.exists(): os.makedirs(output_dir) pdfs_dir = Path(pdfs_dir) args = [] for pdf in tqdm.tqdm(pdfs_dir.listdir("*.pdf")): f_name = pdf.name.split('.')[0] out = output_dir.child(f_name) outfile = out + '-1.png' if not outfile.exists(): args.append((pdf, out)) print('Converting {} pdfs'.format(len(args))) with Pool(NUM_OF_PROCESSES) as pool: pool.starmap(convert_to_png, args)
def get_package_files(eggbasket, package): package_path = get_package_path(eggbasket) package = package_path.listdir()[[x.name.lower() for x in package_path.listdir()].index(package.lower())] package_dir = Path(package_path, package) files = [] for package_file in package_dir.listdir(): p = Package() p.filepath = package_file p.eggbasket = eggbasket try: p.name = SDist(package_file).name p.author = SDist(package_file).metadata['PKG-INFO']['Author'] except Exception: pass files.append(p) return sorted(files, reverse=True)
def create_own_lexicon(): lexicon = {} for dataset in ['KNMP', 'Stanford']: # Find all the annotated pages in the dataset ann_dir = Path(Path.cwd().ancestor(1), 'data/hwr_data/words/' + dataset) annotations = ann_dir.listdir( '*.words') for f in annotations: # Segment annotation = ET.parse(f).getroot() for word in annotation.iter('Word'): text = word.get('text') # Add word to lexicon if lexicon.has_key(text): lexicon[text] += 1 else : lexicon[text] = 1 return lexicon
def check_circular_imports(path, prefix): path = Path(path) python_files_paths = path.listdir(filter=FILES, pattern='*.py') relative_import_modules = {} pattern = re.compile( r'(from {0} import )?(\w+),?[\s+]?(as\s\w+)?'.format(prefix), re.IGNORECASE) for pyf in python_files_paths: with open(pyf, 'r') as f: for line in f.read().splitlines(): matchs = pattern.findall(line) if not matchs: continue modules_names = [m[1] for m in matchs] relative_import_modules[pyf.stem] = modules_names for module, next_module in combinations(relative_import_modules.keys(), 2): module_modules = relative_import_modules[module] next_module_modules = relative_import_modules[next_module] if module.name in next_module_modules and next_module.name in module_modules: print "Circular imports in modules {0} and {1}".format( module, next_module)
from __future__ import print_function from unipath import Path import numpy as np if __name__ == '__main__': workdir = Path('tmp/') datasets = workdir.listdir('window_stats_*.csv') total_widths = [] total_heights = [] for data in datasets: with open(data, 'r') as f: widths = [] heights = [] for line in f.readlines(): letter, width, height = line.strip().split(',') if letter == 'label' and width == 'width' and height == 'height': continue # skip header widths.append(int(width)) heights.append(int(height)) total_widths.append(int(width)) total_heights.append(int(height)) print(str(data), 'statistics:') print('Mean width: ', np.mean(widths)) print('Stdev width: ', np.std(widths)) print('Mean height:', np.mean(heights)) print('Stdev height:', np.std(heights)) print('Median width:', np.median(widths)) print('Median height:', np.median(heights)) print()
class ReleaseSorter(object): def __init__(self, sort_dir): self.sort_dir = Path(sort_dir) # Make sure the sort dir is a dir and cd into it if not self.sort_dir.isdir(): raise ReleaseSorterError('Invalid sort-dir {}'.format(sort_dir)) os.chdir(sort_dir) self.files_to_sort = {} def relative_path(self, path, root_path): relative_path = path.replace(root_path, '') if relative_path[0:1] == '/': return relative_path[1:] else: return relative_path def check_extension(self, extension): if extension in ('.mkv', '.avi'): return True else: return False def check_modified_time(self, time_since_modified): if time_since_modified < timedelta(minutes=20): return False else: return True def create_series_folders(self, sorter_file): if sorter_file.series_dir and not sorter_file.series_dir.exists(): log.info('Creating series dir {}'.format( sorter_file.relative_path(sorter_file.series_dir))) sorter_file.series_dir.mkdir() if sorter_file.season_dir and not sorter_file.season_dir.exists(): log.info('Creating season dir {}'.format( sorter_file.relative_path(sorter_file.season_dir))) sorter_file.season_dir.mkdir() def move_subtitle_files(self, sorter_file): """Check for existing subtitle files matching media file and move them to sort folder too. """ for ext in ('.srt', '.sub', '.idx'): subtitle_path = Path(sorter_file.path.parent, '{}{}'.format( sorter_file.path.stem, ext)) if subtitle_path.exists(): log.info('Moving subtitle file {} to {}'.format( self.relative_path(subtitle_path, self.sort_dir), sorter_file.season_dir)) subtitle_path.move(Path(self.sort_dir, sorter_file.season_dir)) def move_sorter_file(self, sorter_file): log.info('Moving {} to {}'.format(sorter_file.relative_path(), sorter_file.season_dir)) sorter_file.path.move(Path(self.sort_dir, sorter_file.season_dir)) def get_sorter_files(self): """List sort dir and find all files to sort""" log.debug('Sorting dir {}'.format(self.sort_dir)) file_list = self.sort_dir.listdir(filter=FILES) for file in file_list: sorter_file = SorterFile(file, self.sort_dir) # File extension if not self.check_extension(sorter_file.extension): log.debug('Skipping {}, wrong file extension'.format( sorter_file.relative_path())) continue # Modifed time, only process files who hasen't been modified the # in the last 20 min time_since_modified = datetime.now() - sorter_file.mtime if not self.check_modified_time(time_since_modified): log.debug('Skipping {}, has been modified in the last 20 min ' '({})'.format(sorter_file.relative_path(), human(time_since_modified))) continue # Skip if file is not a TV release if not sorter_file.release.tv_release: log.debug('Skipping {}, not a TV release'.format( sorter_file.relative_path())) continue # Add file to sorter list series_name = sorter_file.release.tv_series_data['series_name'] series_episodes = self.files_to_sort.get(series_name) if not series_episodes: series_episodes = {} series_episodes[unicode(sorter_file)] = sorter_file self.files_to_sort[series_name] = series_episodes def sort_files(self): # If a season dir already exist use that when sorting. Else if there # is only one file found for the series skip processing and moving. for series in self.files_to_sort.keys(): series_episodes = self.files_to_sort[series] for episode_file in series_episodes: sorter_file = series_episodes[episode_file] # Episode already has a season dir if sorter_file.season_dir.exists(): log.info('Season dir for {} already exists {}'.format( episode_file, sorter_file.season_dir)) # No season dir for episode. Skip if only one episode was found else: # Skip if only one episode was found if len(series_episodes) < 2: log.debug('Skipping {}, only one episode found'.format( series_episodes.iterkeys().next())) del(self.files_to_sort[series]) # Loop remaining files for folder creating and moving for series in self.files_to_sort: series_episodes = self.files_to_sort[series] for episode_file in series_episodes: sorter_file = series_episodes[episode_file] # Create series folder if needed self.create_series_folders(sorter_file) # Move the file self.move_sorter_file(sorter_file) # Move subtitle files self.move_subtitle_files(sorter_file) def sort(self): self.get_sorter_files() self.sort_files() def cleanup_empty_folders(self): log.debug('Cleanup empty folders in {}'.format(self.sort_dir)) dirs_to_check_for_removal = [] for dir in self.sort_dir.walk(filter=DIRS, top_down=False): # Skip all dirs in _ dir if '/_' in dir: log.debug('Skipping cleanup on {}, _ dir'.format(dir)) continue dirs_to_check_for_removal.append(dir) for dir in dirs_to_check_for_removal: # If dir is empty, remove it if dir.isdir() and len(dir.listdir()) == 0: log.info('Removing empty dir {}'.format(self.relative_path( dir, self.sort_dir))) dir.rmtree()
from unipath import Path import os # CircleCI will run this script 3rd # This will add an "_" to all the fresh .md files in v3-1 # All markdown files in the 'includes' folder in Slate portion of the Dev Center repo #os.chdir(r"C:\Users\I860605\Desktop\Dev_Center_New\dev_concur\src\api-explorer\v3-1") v3_1 = Path(r"./src/api-explorer/v3-1") #Add "_" to the beginning of every markdown file that does not have it for markdown in v3_1.listdir(): if "_" not in markdown: new_name = "_" + markdown os.rename(markdown, new_name) print("Adding '_' to file...Filename has just been updated to: ", new_name) print("All filenames already have correct formatting")
from unipath import Path import os # CircleCI will run this script 4th # This will add an "_" to all the fresh .md files in v3-2 # All markdown files in the 'includes' folder in Slate portion of the Dev Center repo #os.chdir(r"C:\Users\I860605\Desktop\Dev_Center_New\dev_concur\src\api-explorer\v3-2") v3_2 = Path(r"./src/api-explorer/v3-2") #Add "_" to the beginning of every markdown file that does not have it for markdown in v3_2.listdir(): if "_" not in markdown: new_name = "_" + markdown os.rename(markdown, new_name) print("Adding '_' to file...Filename has just been updated to: ", new_name) print("All filenames already have correct formatting")
class Decoder(object): def __init__(self,base_dir, cal=None, cal_each_mask=False): ''' A Decoder for a Vector Coded Aperture Measurment System Parameters ----------- base_dir : str, or `unipath.Path` the base directory which holds all data cal: `skrf.Calibration` object or None the calibration template that is copied for each mask's calibration. The `measurements` attribute provided by the calibraition template is never used, only the `ideals`. if None, then no calibration is performed cal_each_mask : bool should a calibration be performed for each mask? If True, this requires that `cal` represents a calibration template, for which the `measurements` are provided for each mask dir ''' self.base_dir = Path( base_dir) self.cal = cal self.cal_each_mask = cal_each_mask # determine rank max_dec = max([int(d) for d in self.decs.keys()]) self.rank = int(sqrt(len('{0:b}'.format(max_dec)))) self.frequency = rf.ran(str(self.decs.values()[0])).values()[0].frequency @property def decs(self): ''' list of decimal values for each mask A dictionary with key:values as string:Path for each dec value ''' return {str(k.name):k for k in self.base_dir.listdir()} def pixel2decs(self, m, n, half_on_only=False): ''' list of the masks which have a given pixel `on`. the masks are given in decimal representations ''' out=[] for d in self.decs.keys(): mask = dec2mask(d, rank=self.rank) if mask[m,n] ==1: # pixel is on if half_on_only: if sum(mask) == self.rank: out.append(d) else: out.append(d) return out def cal_of(self,dec): ''' Calibration for a given mask, or pixel ''' ##TODO: for no-cal or static cal this could be only calculated # once to improve performance if self.cal is None: freq = self.frequency n = len(freq) coefs ={'directivity':zeros(n), 'source match': zeros(n), 'reflection tracking':ones(n)} cal =OnePort.from_coefs(frequency=freq, coefs=coefs) return cal if not self.cal_each_mask: return self.cal else: # we want a calibration for each mask, so create the calbration # for this mask,or pixel cal = deepcopy(self.cal) ideals = cal.ideals if isinstance(dec, tuple): # decode the measurements measured =[] for ideal in ideals: m = self.raw_ntwk_of(dec,ideal.name) measured.append(m) else: measured = rf.ran(self.decs[dec]).values() cal.measured, cal.ideals = rf.align_measured_ideals(measured,ideals) cal.name = str(dec) return cal def error_ntwk_of(self,dec): ''' error ntwk for a given mask, or pixel ''' if isinstance(dec, tuple): ntwks = [self.error_ntwk_of(k) for k in self.pixel2decs(*dec)] return rf.average(ntwks) ntwk = self.cal_of(dec).error_ntwk ntwk.name = dec return ntwk def raw_ntwk_of(self,dec,name): ''' raw ntwk for a given mask, or pixel ''' if isinstance(dec, tuple): ntwks = [self.raw_ntwk_of(k,name) for k in self.pixel2decs(*dec)] return rf.average(ntwks) ntwk = rf.ran(str(self.decs[dec]), contains=name).values()[0] return ntwk def cor_ntwk_of(self,dec, name, loc='corrected'): ''' corrected ntwk for a given mask, or pixel ''' if isinstance(dec, tuple): if loc == 'corrected': # decode in corrected-space ntwks = [self.cor_ntwk_of(k,name) for k in self.pixel2decs(*dec)] return rf.average(ntwks) elif loc =='measured': # decode in measured space m = self.raw_ntwk_of(dec,name) return self.cal_of(dec).apply_cal(m) # correct a measurement for a single mask return self.cal_of(dec).apply_cal(self.raw_ntwk_of(dec,name)) def cor_cube(self,name,attr='s_db'): ''' a corrected datacube constructs a `corrected` 3D data cube with dimensions (FxMXN), where F is frequency axis, M and N are pixels starting from upper left. Parameters -------------- name : str name of network attr: 's', 's_db', 's_deg', any skrf.Network property the attribute to put in the cube ''' rank = self.rank z = array([getattr(self.cor_ntwk_of((m,n),name),attr) \ for m in range(rank) for n in range(rank)]) z = z.T.reshape(-1,rank,rank) return z def interact_cor_cube(self,name,attr='s_db', clims=None): ''' an interactive image projection of the cor_cube ''' z = self.cor_cube(name=name, attr=attr) if clims == None: if attr =='s_db': clims = (-20,10) elif attr=='s_deg': clims = (-180,180) freq = self.frequency def func(n): plt.matshow(z[n]) plt.title('%i%s'%(freq.f_scaled[n],freq.unit)) plt.grid(0) plt.colorbar() if clims is not None: plt.clim(clims) return interactive (func,n =(0,len(freq)) )
from __future__ import print_function from unipath import Path import numpy as np if __name__ == "__main__": workdir = Path("tmp/") datasets = workdir.listdir("window_stats_*.csv") total_widths = [] total_heights = [] for data in datasets: with open(data, "r") as f: widths = [] heights = [] for line in f.readlines(): letter, width, height = line.strip().split(",") if letter == "label" and width == "width" and height == "height": continue # skip header widths.append(int(width)) heights.append(int(height)) total_widths.append(int(width)) total_heights.append(int(height)) print(str(data), "statistics:") print("Mean width: ", np.mean(widths)) print("Stdev width: ", np.std(widths)) print("Mean height:", np.mean(heights)) print("Stdev height:", np.std(heights)) print("Median width:", np.median(widths)) print("Median height:", np.median(heights)) print()
from unipath import Path import os # CircleCI will run this script 5th # This will add an "_" to all the fresh .md files in v4-0 #Note for testing on your local machine, you may uncomment either of the below paths, either will work. #They've been changed for use in CircleCI to specify the specific path as listed in the repository, rather than your machine #So once you uncomment either path, kindly adjust the portion after the r" to wherever v3-0 is on your machine # Once done, simply comment out the 'v3_0 = Path(r"./src/api-explorer/v3-0")' path and then it will run on your machine #These are the 2 paths you may edit for testing on your machine: #os.chdir(r"C:\Users\I860605\Desktop\Dev_Center_New\dev_concur\src\api-explorer\v4-0") # v4_0 = Path(r"C:\Users\I860605\Desktop\Dev_Center_New\dev_concur\src\api-explorer\v4-0" v4_0 = Path(r"./src/api-explorer/v4-0") #Add "_" to the beginning of every markdown file that does not have it for markdown in v4_0.listdir(): if "_" not in markdown: new_name = "_" + markdown os.rename(markdown, new_name) print("Adding '_' to file...Filename has just been updated to: ", new_name) print("All filenames already have correct formatting")
def get_current_package(eggbasket, package): package_dir = Path(get_package_path(eggbasket), package) contents = sorted(package_dir.listdir(), reverse=True) return contents[0] if contents else 'unknown'
import os # CircleCI will run this script 2nd # This will add an "_" to all the fresh .md files in v3-0 # All markdown files in the 'includes' folder in Slate portion of the Dev Center repo #Note for testing on your local machine, you may uncomment either of the below paths, either will work. #They've been changed for use in CircleCI to specify the specific path as listed in the repository, rather than your machine #So once you uncomment either path, kindly adjust the portion after the r" to wherever v3-0 is on your machine # Once done, simply comment out the 'v3_0 = Path(r"./src/api-explorer/v3-0")' path and then it will run on your machine #These are the 2 paths you may edit for testing on your machine: #os.chdir(r"C:\Users\I860605\Desktop\Dev_Center_New\dev_concur\src\api-explorer\v3-0") #v3_0 = Path(r"C:\Users\I860605\Desktop\Dev_Center_New\dev_concur\src\api-explorer\v3-0") v3_0 = Path(r"./src/api-explorer/v3-0") #Add "_" to the beginning of every markdown file that does not have it for markdown in v3_0.listdir(): #os.path.join(v3_0, markdown) print(markdown) if "_" not in markdown: new_name = "_" + markdown os.rename(markdown, new_name) print("Adding '_' to file...Filename has just been updated to: ", new_name) print("All filenames already have correct formatting")
for app in app_list: log.info("Retrieving log data for {}".format(app.name)) # Set up the directory containing all the logs log_directory = Path(app.log_location) log.debug("Log Location = {}".format(log_directory)) # Cycle through all retrieved files looking for files # matching the file name regex log.debug("File name regex = {}".format(app.file_name)) name_regex = r"{}".format(app.file_name) app_logs = [] for file in log_directory.listdir(names_only=True): log.debug("File name in log directory = {}".format(file)) if re.match(name_regex, file): log.debug("Regex matches file name") app_logs.append(log_directory.child(file)) # Open the log(s) and form them into a single json file json_log = [] # Cycle through all the log files and create a list of the entries for app_log in app_logs: log.debug("Opening log file: {}".format(app_log)) try: