def create_reference_paths_dict(base_path): reference_dict = {} for identity_path in os.list_dir(base_path): image_paths = [] full_identity_dir = os.path.join(base_path, identity_dir) for image_path in os.list_dir(full_identity_dir): image_paths.append(image_path) identity = identity_dir.split('/')[-1] reference_dict[identity] = image_paths assert len(image_paths) > 0 return reference_dict
def get_raw_doc(direcory): files = os.list_dir(direcory) corpus = [] for file in files: doc = [] # parse latex source code, convert to txt cmd_res = os.system(r"detex %s > %s" % (file, file + ".txt")) with open(file + ".txt", 'r', encoding='utf-8') as f: for line in f.readlines(): if len(line.strip()) == 0: continue multi_sents = line.split('.') for sent in multi_sents: doc.append(sent) # tokenize # omit tokenized_doc = [] for sent in doc: words = sent.split(' ') tokenized_doc.append(words) corpus.append(tokenized_doc) return corpus
def get_existing_model(): models = [] files = os.list_dir(".") for filename in files: if filename.split(".")[1] == "json": models.append(filename) return models
def test(): for file_rel_path in list_dir(TEST_DIR): file_path = join_path(TEST_DIR, file_rel_path) file_name = base_name(file_path) if (is_file(file_path) and file_name.startswith('test_') and file_name.endswith('.py')): system('python {}'.format(file_path))
def find_latest_test_build(executable_name): directory = 'target/debug' pattern = compile(rf"^{executable_name}-[0-9a-fA-F]+$") latest = None for entry in list_dir(directory): if pattern.match(entry): entry = f'{directory}/{entry}' if (not latest or modified(latest) < modified(entry)): latest = entry return latest
def recursively_check( input_path ): if is_file( input_path ): try_to_move( input_path ) elif is_dir( input_path ): directory_contents = list_dir( input_path ) for file in directory_contents: file_path = path.join( input_path, file ) recursively_check( file_path )
def create_reference_paths_dict_from_gcp(base_path): reference_dict = {} iterator = bucket.list_blobs(prefix=base_path, delimiter='/') for page in iterator.pages: for prefix in page.prefixes: print("PREFIX: ", prefix) iter2 = bucket.list_blobs(prefix=prefix) for file in iter2: print(file.name) print() for identity_path in os.list_dir(base_path): image_paths = [] full_identity_dir = os.path.join(base_path, identity_dir) for image_path in os.list_dir(full_identity_dir): image_paths.append(image_path) identity = identity_dir.split('/')[-1] reference_dict[identity] = image_paths assert len(image_paths) > 0 return reference_dict
def generate_coverage(): coverage_directory = 'target/coverage/tests' run_command('rm', '-rf', coverage_directory) run_command('mkdir', '-p', coverage_directory) for path, directories, files in walk('.'): # If this is the top-level of a rust project if 'Cargo.toml' in files: for directory in ('src', 'examples', 'target', '.git', 'tests'): try: directories.remove(directory) except ValueError: continue # If `src` is somehow missing from the top, skip this directory source = join(path, 'src') if not exists(source): continue command = ('kcov', '--verify', f'--include-path={source}', coverage_directory) # Get unit tests for executable_name in executable_names(path): latest_build = find_latest_test_build(executable_name) if latest_build: run_command(*command, latest_build) break # Get all the integration tests try: # TODO: Unfortunately Cargo does not use any sort of hierarchy # or namespacing, which means if two members (or in fact # the root) of a workspace have files or directories with # the same name in their `tests` directories then those # are going to be compiled at the same level under the # same name in the shared `target` directory but with # different _hash_ suffixes -- which makes it impossible # to differentiate between them. This means that for now # `rustcov` will associate only one of the binaries with # the correct source all the others will mismatch for test in list_dir(join(path, 'tests')): latest_build = find_latest_test_build(split_ext(test)[0]) run_command(*command, latest_build) except FileNotFoundError: continue run_command('kcov', '--merge', 'target/coverage', coverage_directory) run_command('rm', '-rf', coverage_directory)
def create_list_of_files(abs_path, file_list=None): if file_list is None: file_list = [] directory_files = os.list_dir(abs_path) for file_name in directory_files: file_path = os.path.join(abs_path, file_name) #if file_path is not a directory if not os.is_dir(file_path): file_list.append(file_path) else: #file_path is a directory; recurse create_list_of_files(file_path, file_list) if file_list: return file_list else: return []
def upload_results(mid, model, hist, eval_dict, results_dir = '/tmp/results/'): aws_region = 'us-west-1' s3_bucket_name = 'platt-data' s3_connection = boto.s3.connect_to_region(aws_region) bucket = s3_connection.get_bucket(s3_bucket_name) model_path = os.path.join(results_dir, 'model') save_model_to_path_stub(model, model_path) history_filename = os.path.join(results_dir, 'history.txt') eval_filename = os.path.join(results_dir, 'eval.json') with open(history_filename, 'w') as history_file: history_file.write(hist.history) json.dump(eval_dict, open(eval_filename, 'w')) for file_name in os.list_dir(results_dir): full_file_name = os.path.join(results_dir, file_name) s3_output_key = 'models/{}/{}'.format(mid, file_name) key = Key(s3Bucket) key.key = s3_output_key try: key.set_contents_from_filename(full_file_name) except Exception as e: print(e)
def main(args): parser = argparse.ArgumentParser() parser.add_argument( "--cufflinks-files", nargs='+', required=True, help="List of cufflinks FPKM files to use" ) # parser.add_argument( # "--cufflinks-dir", # help="Directory which contains cufflinks FPKM files" # ) parser.add_argument( "--gep-output", default=".tmp.gep.out", help="Output file for intermediate GEP file" ) parser.add_argument( "--output", required=True, help="Output file final results" ) parser.add_argument( "--cibersort-output", default=".tmp.cibersort.out", help="Output file for intermediate CIBERSORT file" ) parser.add_argument( "--mixture-file", required=True, help="Path to mixture file to use with CIBERSORT (i.e. LM22.txt)" ) parser.add_argument( "--cibersort-jar", required=True, help="Path to CIBERSORT jar (i.e. CIBERSORT.jar)" ) parser.add_argument( "--min-fpkm", type=int, default=5, help="Minimum FPKM for gene in samples (speeds up CIBERSORT)" ) parser.add_argument( "--print-top-n", action='store_true', default=False, help="Print top N cell types per sample" ) parser.add_argument( "--n", type=int, default=3, help="Print top N cell types per sample" ) args = parser.parse_args() if args.cufflinks_files: fpkm_files = args.cufflinks_files elif args.cufflinks_dir: fpkm_files = os.list_dir(args.cufflinks_dir) fpkm_files = [f for f in fpkm_files if 'fpkm' in f] gene_fpkm = load_cufflinks_fpkm(fpkm_files) # pivot gene_fpkm gene_fpkm_pivot = pd.pivot_table(gene_fpkm, index=['SampleId'], values=['FPKM'], columns=['gene_short_name'], fill_value=0) gene_fpkm_pivot.columns = gene_fpkm_pivot.columns.get_level_values(1) # Drop infrequently occurring genes gene_fpkm_pivot.drop( labels=gene_fpkm_pivot.columns[gene_fpkm_pivot.sum() < args.min_fpkm], inplace=True, axis=1 ) gene_fpkm_pivot.T.to_csv(args.gep_output, sep='\t') # Start R Server subprocess.check_call( ["R", "CMD", "Rserve", "--no-save"], #stdout=open("/dev/null"), #stderr="/dev/null", ) # Run CIBERSORT # java -jar CIBERSORT.jar -M Mixture -B Signature_Matrix [Options] subprocess.check_call( ["java", "-jar", args.cibersort_jar, "-B", args.mixture_file, "-M", args.gep_output], stdout=open(args.cibersort_output, 'w') ) cibersort_data = pd.read_csv(args.cibersort_output, sep='\t', comment='>') cibersort_data.drop(labels=['Column'], inplace=True, axis=1) cibersort_data.set_index(pd.Series(data=fpkm_files, name='SampleId'), inplace=True) cibersort_data.to_csv(args.output, index=True, sep='\t') if args.print_top_n: non_cell_type_columns = cibersort_data.columns[-4:] cibersort_cell_types = cibersort_data.drop(labels=non_cell_type_columns, axis=1).reset_index() cibersort_cell_types_melted = \ pd.melt(cibersort_cell_types, id_vars=['SampleId']) print(cibersort_cell_types_melted.sort( ['SampleId', 'value'], ascending=[1, 0] ).groupby(['SampleId']).head(args.n))
assert subset in ["train", "val", "stage1_train", "stage1_test", "stage2_test"] train_dir = 'stage1_train' test_dir = 'stage1_test' final_dir = 'stage2_test_final' if subset == "train": directory = os.path.join(dataset_dir, train_dir) else subset == "test": directory = os.path.join(dataset_dir, test_dir) elif subset == "final": directory = os.path.join(dataset_dir, final_dir) ids = os.list_dir(directory) for i, id in enumerate(ids): image_dir = os.path.join(directory, id) self.add_image("dsb", image_id=i, path=image_dir) def load_image(self, image_id, non_zero=None): info = self.image_info[image_id] path = info['path'] image_name = os.listdir(os.path.join(path, 'images')) image_path = os.path.join(path, 'images', image_name[0]) image = imageio.imread(image_path) if len(image.shape)==2: img = skimage.color.gray2rgb(image) image = img*255.
pdf_name = input("Enter new name of pdf (default is Part)\n") if image_name == '': image_name = 'Cover' if pdf_name == '': pdf_name = 'Part' try: change_dir(parentfolder_path) except Exception: print("ERROR: Path not found: " + parentfolder_path) print("\nPress any key to exit") input() exit(0) subfolders = list_dir() files_in_parentfolder = [] for content in subfolders: if "." in content: files_in_parentfolder.append(content) subfolders = [ folder for folder in subfolders if folder not in files_in_parentfolder ] subfolder_paths = [ parentfolder_path + '\\' + subfolder_name for subfolder_name in subfolders ] folders_with_files_paths = [parentfolder_path] + subfolder_paths
def default_args(args): latest_decoder_path = os.list_dir(MODELS_DIR) if not args.decoder_path:
def ls(self): print(os.list_dir("/"))
def __get_file_set(self): file_set = os.list_dir(self.__doc_path) return file_set pass
iso = 800 minimum_diff = float('inf') for value in speeds: diff = abs(speed - value) if diff <= minimum_diff: minimum_diff = diff iso = value auto_settings['iso'] = iso # Setup camera for capture camera = Camera() camera.set_settings(auto_settings) photo_counter = 0 # Check for previous folders for content in list_dir(frames_dir): if path_exists(frames_dir + '/' + content): folder_number = (int(content) + 1) * 1000 if folder_number > photo_counter: photo_counter = folder_number # Start main loop while 1==1: # Set pins for next stepper motor for index in range(0, 4): pin = step_pins[index] if sequence[step_counter][index] != 0: PinIO.output(pin, True) else: PinIO.output(pin, False)
def __init__(self, base_dir): self.directory = base_dir self.filenames = os.list_dir(base_dir) self.image_data = load_images(directory) self.num_images = len(self.image_data)