def create_reference_paths_dict(base_path):
reference_dict = {}
for identity_path in os.list_dir(base_path):
image_paths = []
full_identity_dir = os.path.join(base_path, identity_dir)
for image_path in os.list_dir(full_identity_dir):
image_paths.append(image_path)
identity = identity_dir.split('/')[-1]
reference_dict[identity] = image_paths
assert len(image_paths) > 0
return reference_dict
def get_raw_doc(direcory):
files = os.list_dir(direcory)
corpus = []
for file in files:
doc = []
# parse latex source code, convert to txt
cmd_res = os.system(r"detex %s > %s" % (file, file + ".txt"))
with open(file + ".txt", 'r', encoding='utf-8') as f:
for line in f.readlines():
if len(line.strip()) == 0:
continue
multi_sents = line.split('.')
for sent in multi_sents:
doc.append(sent)
# tokenize
# omit
tokenized_doc = []
for sent in doc:
words = sent.split(' ')
tokenized_doc.append(words)
corpus.append(tokenized_doc)
return corpus
def get_existing_model():
models = []
files = os.list_dir(".")
for filename in files:
if filename.split(".")[1] == "json":
models.append(filename)
return models
def test():
for file_rel_path in list_dir(TEST_DIR):
file_path = join_path(TEST_DIR, file_rel_path)
file_name = base_name(file_path)
if (is_file(file_path) and file_name.startswith('test_')
and file_name.endswith('.py')):
system('python {}'.format(file_path))
def test():
for file_rel_path in list_dir(TEST_DIR):
file_path = join_path(TEST_DIR, file_rel_path)
file_name = base_name(file_path)
if (is_file(file_path)
and file_name.startswith('test_')
and file_name.endswith('.py')):
system('python {}'.format(file_path))
def find_latest_test_build(executable_name):
directory = 'target/debug'
pattern = compile(rf"^{executable_name}-[0-9a-fA-F]+$")
latest = None
for entry in list_dir(directory):
if pattern.match(entry):
entry = f'{directory}/{entry}'
if (not latest or modified(latest) < modified(entry)):
latest = entry
return latest
def recursively_check( input_path ): if is_file( input_path ): try_to_move( input_path ) elif is_dir( input_path ): directory_contents = list_dir( input_path ) for file in directory_contents: file_path = path.join( input_path, file ) recursively_check( file_path )
def create_reference_paths_dict_from_gcp(base_path):
reference_dict = {}
iterator = bucket.list_blobs(prefix=base_path, delimiter='/')
for page in iterator.pages:
for prefix in page.prefixes:
print("PREFIX: ", prefix)
iter2 = bucket.list_blobs(prefix=prefix)
for file in iter2:
print(file.name)
print()
for identity_path in os.list_dir(base_path):
image_paths = []
full_identity_dir = os.path.join(base_path, identity_dir)
for image_path in os.list_dir(full_identity_dir):
image_paths.append(image_path)
identity = identity_dir.split('/')[-1]
reference_dict[identity] = image_paths
assert len(image_paths) > 0
return reference_dict
def generate_coverage():
coverage_directory = 'target/coverage/tests'
run_command('rm', '-rf', coverage_directory)
run_command('mkdir', '-p', coverage_directory)
for path, directories, files in walk('.'):
# If this is the top-level of a rust project
if 'Cargo.toml' in files:
for directory in ('src', 'examples', 'target', '.git', 'tests'):
try:
directories.remove(directory)
except ValueError:
continue
# If `src` is somehow missing from the top, skip this directory
source = join(path, 'src')
if not exists(source):
continue
command = ('kcov', '--verify', f'--include-path={source}',
coverage_directory)
# Get unit tests
for executable_name in executable_names(path):
latest_build = find_latest_test_build(executable_name)
if latest_build:
run_command(*command, latest_build)
break
# Get all the integration tests
try:
# TODO: Unfortunately Cargo does not use any sort of hierarchy
# or namespacing, which means if two members (or in fact
# the root) of a workspace have files or directories with
# the same name in their `tests` directories then those
# are going to be compiled at the same level under the
# same name in the shared `target` directory but with
# different _hash_ suffixes -- which makes it impossible
# to differentiate between them. This means that for now
# `rustcov` will associate only one of the binaries with
# the correct source all the others will mismatch
for test in list_dir(join(path, 'tests')):
latest_build = find_latest_test_build(split_ext(test)[0])
run_command(*command, latest_build)
except FileNotFoundError:
continue
run_command('kcov', '--merge', 'target/coverage', coverage_directory)
run_command('rm', '-rf', coverage_directory)
def create_list_of_files(abs_path, file_list=None):
if file_list is None:
file_list = []
directory_files = os.list_dir(abs_path)
for file_name in directory_files:
file_path = os.path.join(abs_path, file_name)
#if file_path is not a directory
if not os.is_dir(file_path):
file_list.append(file_path)
else: #file_path is a directory; recurse
create_list_of_files(file_path, file_list)
if file_list:
return file_list
else:
return []
def upload_results(mid, model, hist, eval_dict, results_dir = '/tmp/results/'):
aws_region = 'us-west-1'
s3_bucket_name = 'platt-data'
s3_connection = boto.s3.connect_to_region(aws_region)
bucket = s3_connection.get_bucket(s3_bucket_name)
model_path = os.path.join(results_dir, 'model')
save_model_to_path_stub(model, model_path)
history_filename = os.path.join(results_dir, 'history.txt')
eval_filename = os.path.join(results_dir, 'eval.json')
with open(history_filename, 'w') as history_file:
history_file.write(hist.history)
json.dump(eval_dict, open(eval_filename, 'w'))
for file_name in os.list_dir(results_dir):
full_file_name = os.path.join(results_dir, file_name)
s3_output_key = 'models/{}/{}'.format(mid, file_name)
key = Key(s3Bucket)
key.key = s3_output_key
try:
key.set_contents_from_filename(full_file_name)
except Exception as e:
print(e)
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument(
"--cufflinks-files",
nargs='+',
required=True,
help="List of cufflinks FPKM files to use"
)
# parser.add_argument(
# "--cufflinks-dir",
# help="Directory which contains cufflinks FPKM files"
# )
parser.add_argument(
"--gep-output",
default=".tmp.gep.out",
help="Output file for intermediate GEP file"
)
parser.add_argument(
"--output",
required=True,
help="Output file final results"
)
parser.add_argument(
"--cibersort-output",
default=".tmp.cibersort.out",
help="Output file for intermediate CIBERSORT file"
)
parser.add_argument(
"--mixture-file",
required=True,
help="Path to mixture file to use with CIBERSORT (i.e. LM22.txt)"
)
parser.add_argument(
"--cibersort-jar",
required=True,
help="Path to CIBERSORT jar (i.e. CIBERSORT.jar)"
)
parser.add_argument(
"--min-fpkm",
type=int,
default=5,
help="Minimum FPKM for gene in samples (speeds up CIBERSORT)"
)
parser.add_argument(
"--print-top-n",
action='store_true',
default=False,
help="Print top N cell types per sample"
)
parser.add_argument(
"--n",
type=int,
default=3,
help="Print top N cell types per sample"
)
args = parser.parse_args()
if args.cufflinks_files:
fpkm_files = args.cufflinks_files
elif args.cufflinks_dir:
fpkm_files = os.list_dir(args.cufflinks_dir)
fpkm_files = [f for f in fpkm_files if 'fpkm' in f]
gene_fpkm = load_cufflinks_fpkm(fpkm_files)
# pivot gene_fpkm
gene_fpkm_pivot = pd.pivot_table(gene_fpkm,
index=['SampleId'],
values=['FPKM'],
columns=['gene_short_name'],
fill_value=0)
gene_fpkm_pivot.columns = gene_fpkm_pivot.columns.get_level_values(1)
# Drop infrequently occurring genes
gene_fpkm_pivot.drop(
labels=gene_fpkm_pivot.columns[gene_fpkm_pivot.sum() < args.min_fpkm],
inplace=True,
axis=1
)
gene_fpkm_pivot.T.to_csv(args.gep_output, sep='\t')
# Start R Server
subprocess.check_call(
["R", "CMD", "Rserve", "--no-save"],
#stdout=open("/dev/null"),
#stderr="/dev/null",
)
# Run CIBERSORT
# java -jar CIBERSORT.jar -M Mixture -B Signature_Matrix [Options]
subprocess.check_call(
["java",
"-jar",
args.cibersort_jar,
"-B",
args.mixture_file,
"-M",
args.gep_output],
stdout=open(args.cibersort_output, 'w')
)
cibersort_data = pd.read_csv(args.cibersort_output, sep='\t', comment='>')
cibersort_data.drop(labels=['Column'], inplace=True, axis=1)
cibersort_data.set_index(pd.Series(data=fpkm_files, name='SampleId'), inplace=True)
cibersort_data.to_csv(args.output, index=True, sep='\t')
if args.print_top_n:
non_cell_type_columns = cibersort_data.columns[-4:]
cibersort_cell_types = cibersort_data.drop(labels=non_cell_type_columns, axis=1).reset_index()
cibersort_cell_types_melted = \
pd.melt(cibersort_cell_types, id_vars=['SampleId'])
print(cibersort_cell_types_melted.sort(
['SampleId', 'value'],
ascending=[1, 0]
).groupby(['SampleId']).head(args.n))
assert subset in ["train", "val", "stage1_train", "stage1_test", "stage2_test"]
train_dir = 'stage1_train'
test_dir = 'stage1_test'
final_dir = 'stage2_test_final'
if subset == "train":
directory = os.path.join(dataset_dir, train_dir)
else subset == "test":
directory = os.path.join(dataset_dir, test_dir)
elif subset == "final":
directory = os.path.join(dataset_dir, final_dir)
ids = os.list_dir(directory)
for i, id in enumerate(ids):
image_dir = os.path.join(directory, id)
self.add_image("dsb", image_id=i, path=image_dir)
def load_image(self, image_id, non_zero=None):
info = self.image_info[image_id]
path = info['path']
image_name = os.listdir(os.path.join(path, 'images'))
image_path = os.path.join(path, 'images', image_name[0])
image = imageio.imread(image_path)
if len(image.shape)==2:
img = skimage.color.gray2rgb(image)
image = img*255.
pdf_name = input("Enter new name of pdf (default is Part)\n")
if image_name == '':
image_name = 'Cover'
if pdf_name == '':
pdf_name = 'Part'
try:
change_dir(parentfolder_path)
except Exception:
print("ERROR: Path not found: " + parentfolder_path)
print("\nPress any key to exit")
input()
exit(0)
subfolders = list_dir()
files_in_parentfolder = []
for content in subfolders:
if "." in content:
files_in_parentfolder.append(content)
subfolders = [
folder for folder in subfolders if folder not in files_in_parentfolder
]
subfolder_paths = [
parentfolder_path + '\\' + subfolder_name for subfolder_name in subfolders
]
folders_with_files_paths = [parentfolder_path] + subfolder_paths
def default_args(args):
latest_decoder_path = os.list_dir(MODELS_DIR)
if not args.decoder_path:
def ls(self):
print(os.list_dir("/"))
def __get_file_set(self): file_set = os.list_dir(self.__doc_path) return file_set pass
iso = 800
minimum_diff = float('inf')
for value in speeds:
diff = abs(speed - value)
if diff <= minimum_diff:
minimum_diff = diff
iso = value
auto_settings['iso'] = iso
# Setup camera for capture
camera = Camera()
camera.set_settings(auto_settings)
photo_counter = 0
# Check for previous folders
for content in list_dir(frames_dir):
if path_exists(frames_dir + '/' + content):
folder_number = (int(content) + 1) * 1000
if folder_number > photo_counter:
photo_counter = folder_number
# Start main loop
while 1==1:
# Set pins for next stepper motor
for index in range(0, 4):
pin = step_pins[index]
if sequence[step_counter][index] != 0:
PinIO.output(pin, True)
else:
PinIO.output(pin, False)
def __init__(self, base_dir):
self.directory = base_dir
self.filenames = os.list_dir(base_dir)
self.image_data = load_images(directory)
self.num_images = len(self.image_data)
