def generate_for_grimm():
if len(sys.argv) < 6:
print 'USAGE:', sys.argv[
0], 'grimm', 'homology_file', 'first_specie_gff', 'second_specie_gff', 'output_directory'
exit()
print utils.get_time()
print 'parsing homology file...'
homology = parse_homology(sys.argv[2])
print utils.get_time()
print 'parsing genes for the first species...'
first_specie_genes = parse_genes(sys.argv[3])
print utils.get_time()
print 'parsing genes for the second species...'
second_specie_genes = parse_genes(sys.argv[4])
#homology = reduce_homology(first_specie_genes, second_specie_genes, homology)
directory = sys.argv[5]
utils.create_dir_if_not_exists(directory)
file = os.path.join(directory, 'grimm.input')
print utils.get_time()
print 'writing genes to file...'
coding_table = output_genes('genome1', first_specie_genes, homology, file)
output_genes_and_rename('genome2', second_specie_genes, homology,
coding_table, file)
coding_table_file = os.path.join(directory, 'coding_table.txt')
print utils.get_time()
print 'saving coding table...'
save_coding_table(coding_table, coding_table_file)
print utils.get_time()
print 'done.'
def __init__(self, model, time_series_list, checkpoint_path, output_dir):
model.load_state_dict(torch.load(checkpoint_path)['model'])
self.model = model
self.time_series_list = time_series_list
self.output_dir = output_dir
create_dir_if_not_exists(self.output_dir)
def run(dir, maf, sizes_folder, species_file):
grimm_synt_input_file = os.path.join(dir, utils.get_name(maf)+'.grimm_synt')
print utils.get_time()
print 'converting maf to input for grimm...'
params = ['./generate_input_grimm_synt', maf, sizes_folder, grimm_synt_input_file, species_file]
subprocess.check_call(params)
print utils.get_time()
print 'generating anchors for grimm_synt...'
anchors_folder = os.path.join(dir,'grimm_synt_anchors')
utils.create_dir_if_not_exists(anchors_folder)
params = ['grimm_synt', '-A', '-f', grimm_synt_input_file, '-d', anchors_folder]
subprocess.check_call(params)
print utils.get_time()
print 'running grimm_synt...'
grimm_synt_output = os.path.join(dir,'grimm_synt_output')
utils.create_dir_if_not_exists(grimm_synt_output)
#used this for comparing mhc regions
#params = ['grimm_synt','-f', os.path.join(anchors_folder, 'unique_coords.txt'),'-d',grimm_synt_output, '-m 1000 -g 1000 -c']
#params = ['grimm_synt','-f', os.path.join(anchors_folder, 'unique_coords.txt'),'-d',grimm_synt_output,'-m', '300000', '-g', '300000', '-c']
params = ['grimm_synt','-f', os.path.join(anchors_folder, 'unique_coords.txt'),'-d',grimm_synt_output,'-m', '100000', '-g', '100000', '-c']
subprocess.check_call(params)
print 'synteny blocks are at',os.path.join(grimm_synt_output,'blocks.txt')
print utils.get_time()
print 'creating bigBed files...'
create_bigBed(grimm_synt_output, sizes_folder, species_file)
print utils.get_time()
print 'running grimm...'
params = ['grimm', '-f', os.path.join(grimm_synt_output,'mgr_macro.txt'), '-o', os.path.join(dir,'grimm.output')]
subprocess.call(" ".join(params), shell=True)
print 'grimm output is saved to', os.path.join(dir,'grimm.output')
print utils.get_time()
print 'done.'
def generate_for_grimm() :
if len(sys.argv) < 6:
print 'USAGE:', sys.argv[0], 'grimm', 'homology_file', 'first_specie_gff', 'second_specie_gff', 'output_directory'
exit()
print utils.get_time()
print 'parsing homology file...'
homology = parse_homology(sys.argv[2])
print utils.get_time()
print 'parsing genes for the first species...'
first_specie_genes = parse_genes(sys.argv[3])
print utils.get_time()
print 'parsing genes for the second species...'
second_specie_genes = parse_genes(sys.argv[4])
#homology = reduce_homology(first_specie_genes, second_specie_genes, homology)
directory = sys.argv[5]
utils.create_dir_if_not_exists(directory)
file = os.path.join(directory,'grimm.input')
print utils.get_time()
print 'writing genes to file...'
coding_table = output_genes('genome1', first_specie_genes, homology, file)
output_genes_and_rename('genome2', second_specie_genes, homology, coding_table, file)
coding_table_file = os.path.join(directory,'coding_table.txt')
print utils.get_time()
print 'saving coding table...'
save_coding_table(coding_table, coding_table_file)
print utils.get_time()
print 'done.'
def create_bigBed(grimm_synt_output, sizes_folder, species_file):
blocks = os.path.join(grimm_synt_output, 'blocks.txt')
bed = os.path.join(grimm_synt_output, 'bed')
utils.create_dir_if_not_exists(bed)
params = ['./blocks_to_bed.py', blocks, bed]
subprocess.check_call(params)
with open(species_file) as species:
for line in species:
line = line.strip().split(' ')
if len(line) < 2:
print 'species file format: genome_id genome_name'
print 'no bigBeds generated'
raise Exception('wrong species format')
params = [
'sort -k1,1 -k2,2n',
os.path.join(bed, line[0] + '.bed'), '>',
os.path.join(bed, line[0] + '.sorted.bed')
]
subprocess.check_call(" ".join(params), shell=True)
params = [
'bedToBigBed', '-type=bed6',
os.path.join(bed, line[0] + '.sorted.bed'),
os.path.join(sizes_folder, line[1] + '.sizes'),
os.path.join(bed, line[1] + '.bb')
]
subprocess.check_call(params)
print 'path to bigBed folder is', bed
def run(fasta, ncrna, results) :
print utils.get_time()
print 'check that folder for results exists...'
utils.create_dir_if_not_exists(results)
print utils.get_time()
print 'running blat...'
blat_psl = run_blat(fasta, ncrna, results)
#blat_psl = '/hive/groups/recon/projs/pipeline_data/assemblies/FelisCatus/6.2/tracks/ncrna/2014-10-18/FelisCatus.psl'
#blat_psl = '/hive/groups/recon/projs/pipeline_data/assemblies/FelisCatus/6.2/tracks/ncrna/FelisCatus.sorted.only_entries.filtered.psl'
print utils.get_time()
print 'sorting blat output and adjusting file to future processing...'
sorted_psl = run_sorting(blat_psl, results)
print utils.get_time()
print 'filtering blat results...'
filtered_psl = run_filtering(sorted_psl, results)
print utils.get_time()
print 'calculating sizes of fasta sequences...'
sizes = utils.run_faSize(fasta, results)
#sizes = '/hive/groups/recon/projs/pipeline_data/assemblies/FelisCatus/6.2/tracks/ncrna/2014-10-18/FelisCatus.sizes'
print utils.get_time()
print 'converting psl to bigBed'
run_pslToBigBed(filtered_psl, sizes, results)
print utils.get_time()
print 'done.'
def __init__(self,
model,
criterion,
optimizer,
scheduler,
time_series_list,
train_valid_split=0.8,
num_epochs=30,
early_stopping=5,
batch_size=64,
checkpoint_path="../checkpoints/seq2seq.h5",
plot=True,
offet_for_plot=50):
self.model = model
self.criterion = criterion
self.optimizer = optimizer
self.scheduler = scheduler
self.num_epochs = num_epochs
self.early_stopping = early_stopping
self.train_valid_split = train_valid_split
self.batch_size = batch_size
self.checkpoint_path = checkpoint_path
self.time_series_list = time_series_list
self.plot_ts = []
self.plot = plot
self.offset = offet_for_plot
create_dir_if_not_exists(self.checkpoint_path)
self._make_loader()
def get_gs_ds_mat(gs1, gs2, dist_sim_calculator, tvt1, tvt2,
dataset, dist_metric, dist_algo, norm, dec_gsize, return_neg1=False):
mat_str = '{}({})_{}({})'.format(tvt1, len(gs1), tvt2, len(gs2))
dir = '{}/ds_mat'.format(get_save_path())
create_dir_if_not_exists(dir)
sfn = '{}/{}_{}_ds_mat_{}{}_{}'.format(
dir, dataset, mat_str, dist_metric,
get_norm_str(norm), dist_algo)
l = load(sfn)
if l is not None:
print('Loaded from {}'.format(sfn))
return l
m = len(gs1)
n = len(gs2)
dist_mat = np.zeros((m, n))
for i in range(m):
for j in range(n):
g1 = gs1[i]
g2 = gs2[j]
d, normed_d = dist_sim_calculator.calculate_dist_sim(
g1, g2, dec_gsize=dec_gsize, return_neg1=return_neg1)
if norm:
dist_mat[i][j] = normed_d
else:
dist_mat[i][j] = d
save(sfn, dist_mat)
print('Saved to {}'.format(sfn))
return dist_mat
def main(opts):
if not opts.do_train and not opts.do_eval:
raise ValueError('At least one of `do_train` or `do_eval` must be True.')
if os.path.exists(opts.output_dir) and os.listdir(opts.output_dir) and opts.do_train:
raise ValueError('Output directory ({}) already exists and is not empty.'.format(opts.output_dir))
# Create the output directory (if not exists)
create_dir_if_not_exists(opts.output_dir)
# Device device type
opts.device = torch.device('cuda' if torch.cuda.is_available() and not opts.no_cuda else 'cpu')
opts.n_gpus = torch.cuda.device_count() if str(opts.device) == 'cuda' else 0
print('Device Type: {} | Number of GPUs: {}'.format(opts.device, opts.n_gpus), flush=True)
# Load Datasets and Ontology
dataset = MyDataset()
if opts.do_train:
# Load model from scratch
model = Model.from_scratch('bert-base-chinese')
model.move_to_device(opts)
print('Number of model parameters is: {}'.format(get_n_params(model)))
# Start Training
print('Start Training', flush=True)
model.run_train(dataset, opts)
# Free up all memory pytorch is taken from gpu memory
del model
torch.cuda.empty_cache()
def setup_temp_data_folder(gp, append_str):
dir = gp + '/data'
create_dir_if_not_exists(dir)
tp = dir + '/temp_{}'.format(append_str)
exec_cmd('rm -rf {} && mkdir {}'.format(tp, tp))
src = get_root_path() + '/src/gmt_files'
exec_cmd('cp {}/temp.xml {}/temp_{}.xml'.format(src, tp, append_str))
return src, tp
def exp5():
""" Query visualization. """
dataset = 'imdbmulti'
model = 'astar'
concise = True
norms = [True, False]
dir = get_result_path() + '/{}/query_vis/{}'.format(dataset, model)
create_dir_if_not_exists(dir)
info_dict = {
# draw node config
'draw_node_size': 150 if dataset != 'linux' else 10,
'draw_node_label_enable': True,
'node_label_name': None if dataset == 'linux' else 'type',
'draw_node_label_font_size': 6,
'draw_node_color_map': TYPE_COLOR_MAP,
# draw edge config
'draw_edge_label_enable': False,
'edge_label_name': 'valence',
'draw_edge_label_font_size': 6,
# graph text info config
'each_graph_text_list': [],
'each_graph_text_font_size': 8,
'each_graph_text_pos': [0.5, 1.05],
# graph padding: value range: [0, 1]
'top_space': 0.20 if concise else 0.26, # out of whole graph
'bottom_space': 0.05,
'hbetween_space': 0.6 if concise else 1, # out of the subgraph
'wbetween_space': 0,
# plot config
'plot_dpi': 200,
'plot_save_path_eps': '',
'plot_save_path_png': ''
}
train_data = load_data(dataset, train=True)
test_data = load_data(dataset, train=False)
row_graphs = test_data.graphs
col_graphs = train_data.graphs
r = load_result(dataset, model, row_graphs=row_graphs, col_graphs=col_graphs)
tr = load_result(dataset, TRUE_MODEL, row_graphs=row_graphs, col_graphs=col_graphs)
for norm in norms:
ids = r.get_sort_id_mat(norm)
m, n = r.m_n()
num_vis = 10
for i in range(num_vis):
q = test_data.graphs[i]
gids = np.concatenate([ids[i][:3], [ids[i][int(n / 2)]], ids[i][-3:]])
gs = [train_data.graphs[j] for j in gids]
info_dict['each_graph_text_list'] = \
[get_text_label(dataset, r, tr, i, i, q, model, norm, True, concise)] + \
[get_text_label(dataset, r, tr, i, j,
train_data.graphs[j], model, norm, False, concise) \
for j in gids]
# print(info_dict['each_graph_text_list'])
info_dict['plot_save_path_png'] = '{}/query_vis_{}_{}_{}{}.{}'.format(
dir, dataset, model, i, get_norm_str(norm), 'png')
info_dict['plot_save_path_eps'] = '{}/query_vis_{}_{}_{}{}.{}'.format(
dir, dataset, model, i, get_norm_str(norm), 'eps')
vis(q, gs, info_dict)
def __init__(self):
model_str = self.get_model_str()
self.logdir = join(get_root_path(), 'logs',
'{}_{}'.format(model_str, get_ts()))
create_dir_if_not_exists(self.logdir)
self.model_info_f = self._open('model_info.txt')
self._log_model_info()
self._save_conf_code()
print('Logging to {}'.format(self.logdir))
def gen_dataset(graphs):
random.Random(123).shuffle(graphs)
dirout_train = get_data_path() + '/IMDBMulti/train'
dirout_test = get_data_path() + '/IMDBMulti/test'
create_dir_if_not_exists(dirout_train)
create_dir_if_not_exists(dirout_test)
for g in graphs[0:1200]:
nx.write_gexf(g, dirout_train + '/{}.gexf'.format(g.graph['gid']))
for g in graphs[1200:]:
nx.write_gexf(g, dirout_test + '/{}.gexf'.format(g.graph['gid']))
def __init__(self, sess):
if FLAGS.log:
model_str = self._get_model_str()
logdir = '{}/logs/{}_{}'.format(get_siamese_dir(), model_str,
get_ts())
create_dir_if_not_exists(logdir)
self.tw = tf.summary.FileWriter(logdir + '/train', sess.graph)
self.vw = tf.summary.FileWriter(logdir + '/val', sess.graph)
self.merged = tf.summary.merge_all()
self._log_model_info(logdir, sess)
print('Logging to {}'.format(logdir))
def gen_dataset(graphs):
random.Random(123).shuffle(graphs)
dirout_train = get_data_path() + '/{}/train'.format(conf.outfolder)
dirout_test = get_data_path() + '/{}/test'.format(conf.outfolder)
create_dir_if_not_exists(dirout_train)
create_dir_if_not_exists(dirout_test)
sp = int(len(graphs) * conf.train_perc_)
for g in graphs[0:sp]:
nx.write_gexf(g, dirout_train + '/{}.gexf'.format(g.graph['gid']))
for g in graphs[sp:]:
nx.write_gexf(g, dirout_test + '/{}.gexf'.format(g.graph['gid']))
def run(config, dir, outputhal):
utils.check_existence_or_raise(config)
utils.create_dir_if_not_exists(dir)
command = ['runProgressiveCactus.sh', '--batchSystem', 'parasol', '--bigBatchSystem', 'singleMachine',
'--defaultMemory', '8589934593', '--bigMemoryThreshold', '8589934592', '--bigMaxMemory', '893353197568',
'--bigMaxCpus', '25', '--maxThreads', '25', '--parasolCommand=/cluster/home/jcarmstr/bin/parasol -host=ku',
'--retryCount', '3']
params = command + [config, dir, outputhal]
print params
subprocess.check_call(params)
return outputhal
def get_gs_ds_mat(gs1,
gs2,
dist_sim_calculator,
tvt1,
tvt2,
dataset,
dist_metric,
dist_algo,
norm,
dec_gsize,
return_neg1=False):
if logging_enabled == True:
print("- Entered dist_sim_calculator::get_gs_ds_mat Global Method")
mat_str = '{}({})_{}({})'.format(tvt1, len(gs1), tvt2, len(gs2))
dir = '{}\\ds_mat'.format(get_save_path())
create_dir_if_not_exists(dir)
sfn = '{}\\{}_{}_ds_mat_{}{}_{}'.format(dir, dataset, mat_str, dist_metric,
get_norm_str(norm), dist_algo)
l = load(sfn)
if l is not None:
print('Loaded from {}'.format(sfn))
return l
if not dist_sim_calculator.gidpair_ds_map:
# dist_sim_calculator.initial_calculate_dist_sim(gs1, gs2)
dist_sim_calculator.initial_dist_sim_pairs_with_netcomp(gs1, gs2)
m = len(gs1)
n = len(gs2)
dist_mat = np.zeros((m, n))
for i in range(m):
for j in range(n):
g1 = gs1[i]
g2 = gs2[j]
d, normed_d = dist_sim_calculator.calculate_dist_sim(
g1, g2, dec_gsize=dec_gsize, return_neg1=return_neg1)
if norm:
dist_mat[i][j] = normed_d
print("i: ", i, ", j: ", j, ", d: ", d, ", normed_d: ",
normed_d)
else:
dist_mat[i][j] = d
save(sfn, dist_mat)
print('Saved to {}'.format(sfn))
return dist_mat
def run_filtering(sorted_psl, results) :
utils.check_existence_or_raise(sorted_psl)
stats = os.path.join(results,'filtering_stats')
utils.create_dir_if_not_exists(stats)
filtered_psl = os.path.join(results,utils.get_name(sorted_psl)+'.filtered.psl')
tmp = utils.atomic_tmp_file(filtered_psl)
params = ['pslCDnaFilter','-localNearBest=0.1 -ignoreNs -minCover=0.6 -repsAsMatch',
' -dropped='+os.path.join(stats,'dropped.out'), '-weirdOverlapped='+os.path.join(stats,'weird_overlapped.out'),
'-alignStats='+os.path.join(stats,'align_stats.out'), '-statsOut='+os.path.join(stats,'overall_stats.out'),
sorted_psl, tmp]
subprocess.call(" ".join(params), shell=True)
utils.atomic_install(tmp, filtered_psl)
return filtered_psl
def __init__(self, sess=None):
model_str = self._get_model_str()
self.logdir = '{}/logs/{}_{}'.format(get_siamese_dir(), model_str,
get_ts())
create_dir_if_not_exists(self.logdir)
if sess is not None:
self.tw = tf.summary.FileWriter(self.logdir + '/train', sess.graph)
self.all_merged = tf.summary.merge_all()
self.loss_merged = tf.summary.merge(
self._extract_loss_related_summaries_as_list())
self._log_model_info(self.logdir, sess)
self.f = open('{}/results_{}.txt'.format(self.logdir, get_ts()), 'w')
print('Logging to {}'.format(self.logdir))
def main():
person_group_id = FACE_GROUP_ID
display_image = False
face_api_wrapper = FaceAPIWrapper(FACE_API_KEY, FACE_BASE_URL)
create_dir_if_not_exists('captured_images/' + CAMERA_NAME)
print("Capturing Image every ", CAPTURE_INTERVAL, " seconds...")
i = 0
while 1:
try:
image_filename = 'captured_images/' + CAMERA_NAME + "/" + current_time_to_string() + ".jpg"
image = Camera().capture_image()
cv2.imwrite(image_filename, image)
if display_image:
cv2.imshow("Camera Image", image)
cv2.waitKey(1)
image_link = upload_to_s3(image_filename)
image_url = 'https://mybuckfucket.s3-eu-west-1.amazonaws.com/public_folder/'+image_filename
face_ids = face_api_wrapper.detect_faces(image=image_filename)
i += 1
print(i, "Captured at ", current_time_to_string())
print("S3 url",image_url)
if face_ids:
person_ids = \
face_api_wrapper.identify_faces(face_ids=face_ids,
large_person_group=person_group_id)
req_ids = [{id} for id in person_ids]
print("Detected Faces...", req_ids)
contents=load_dict_from_file('json.txt')
#my_data=(contents[0]['faceAttributes'])
requests.post('http://127.0.0.1:8000/students/', data={
'attributes' : contents,
'name' : "Darragh",
'time_date' : current_time_to_string(),
'face_ids' : person_ids,
'image_link' : image_url,
})
print("#####",data)
#print(contents[0]['faceAttributes']['emotion'])
#requests.post('http://127.0.0.1:8000/students/', data=contents)
time.sleep(CAPTURE_INTERVAL)
except Exception as e:
print(e)
def run(dir, maf, sizes_folder, species_file):
grimm_synt_input_file = os.path.join(dir,
utils.get_name(maf) + '.grimm_synt')
print utils.get_time()
print 'converting maf to input for grimm...'
params = [
'./generate_input_grimm_synt', maf, sizes_folder,
grimm_synt_input_file, species_file
]
subprocess.check_call(params)
print utils.get_time()
print 'generating anchors for grimm_synt...'
anchors_folder = os.path.join(dir, 'grimm_synt_anchors')
utils.create_dir_if_not_exists(anchors_folder)
params = [
'grimm_synt', '-A', '-f', grimm_synt_input_file, '-d', anchors_folder
]
subprocess.check_call(params)
print utils.get_time()
print 'running grimm_synt...'
grimm_synt_output = os.path.join(dir, 'grimm_synt_output')
utils.create_dir_if_not_exists(grimm_synt_output)
#used this for comparing mhc regions
#params = ['grimm_synt','-f', os.path.join(anchors_folder, 'unique_coords.txt'),'-d',grimm_synt_output, '-m 1000 -g 1000 -c']
#params = ['grimm_synt','-f', os.path.join(anchors_folder, 'unique_coords.txt'),'-d',grimm_synt_output,'-m', '300000', '-g', '300000', '-c']
params = [
'grimm_synt', '-f',
os.path.join(anchors_folder, 'unique_coords.txt'), '-d',
grimm_synt_output, '-m', '100000', '-g', '100000', '-c'
]
subprocess.check_call(params)
print 'synteny blocks are at', os.path.join(grimm_synt_output,
'blocks.txt')
print utils.get_time()
print 'creating bigBed files...'
create_bigBed(grimm_synt_output, sizes_folder, species_file)
print utils.get_time()
print 'running grimm...'
params = [
'grimm', '-f',
os.path.join(grimm_synt_output, 'mgr_macro.txt'), '-o',
os.path.join(dir, 'grimm.output')
]
subprocess.call(" ".join(params), shell=True)
print 'grimm output is saved to', os.path.join(dir, 'grimm.output')
print utils.get_time()
print 'done.'
def plot_preck(dataset, dsmetric, models, rs, true_result, metric, norms,
plot_results=True, extra_dir=None):
""" Plot prec@k. """
create_dir_if_not_exists('{}/{}/{}'.format(
get_result_path(), dataset, metric))
rtn = {}
for norm in norms:
_, n = true_result.m_n()
ks = range(1, n)
d = plot_preck_helper(
dataset, dsmetric, models, rs, true_result, metric, norm, ks,
False, plot_results, extra_dir)
rtn.update(d)
return rtn
def run(config, dir, outputhal):
utils.check_existence_or_raise(config)
utils.create_dir_if_not_exists(dir)
command = [
'runProgressiveCactus.sh', '--batchSystem', 'parasol',
'--bigBatchSystem', 'singleMachine', '--defaultMemory', '8589934593',
'--bigMemoryThreshold', '8589934592', '--bigMaxMemory', '893353197568',
'--bigMaxCpus', '25', '--maxThreads', '25',
'--parasolCommand=/cluster/home/jcarmstr/bin/parasol -host=ku',
'--retryCount', '3'
]
params = command + [config, dir, outputhal]
print params
subprocess.check_call(params)
return outputhal
def __init__(self, sess=None):
if logging_enabled == True:
print("- Entered saver::Saver::__init__ Constructor Method")
model_str = self._get_model_str()
self.logdir = '{}\\logs\\{}'.format(get_siamese_dir(), model_str)
create_dir_if_not_exists(self.logdir)
if sess is not None:
self.tw = tf.compat.v1.summary.FileWriter(self.logdir + '\\train', sess.graph)
self.all_merged = tf.compat.v1.summary.merge_all()
self.loss_merged = tf.compat.v1.summary.merge(self._extract_loss_related_summaries_as_list())
self.f = open('{}\\results.txt'.format(self.logdir), 'w')
print('Logging to {}'.format(self.logdir))
def save_fig(plt, dir, fn, print_path=False):
plt_cnt = 0
if dir is None or fn is None:
return plt_cnt
final_path_without_ext = '{}/{}'.format(dir, fn)
for ext in ['png', 'eps']:
full_path = final_path_without_ext + '.' + ext
create_dir_if_not_exists(dirname(full_path))
try:
plt.savefig(full_path, bbox_inches='tight')
except:
warn('savefig')
if print_path:
print('Saved to {}'.format(full_path))
plt_cnt += 1
return plt_cnt
def select_dump_graphs(graphs, less_than_eq_10_ids):
remove_type(graphs)
random.Random(123).shuffle(less_than_eq_10_ids)
tr = 800
te = 200
assert (tr + te <= len(graphs))
dirout = get_data_path() + '/linux'
train_dir = dirout + '/train'
create_dir_if_not_exists(train_dir)
for i in range(tr):
gid = less_than_eq_10_ids[i]
nx.write_gexf(graphs[gid], train_dir + '/{}.gexf'.format(gid))
test_dir = dirout + '/test'
create_dir_if_not_exists(test_dir)
for i in range(tr, tr + te):
gid = less_than_eq_10_ids[i]
nx.write_gexf(graphs[gid], test_dir + '/{}.gexf'.format(gid))
def save_figs(info_dict):
save_path = info_dict['plot_save_path_png']
print(save_path)
if not save_path:
# print('plt.show')
plt.show()
else:
for full_path in [
info_dict['plot_save_path_png'],
info_dict['plot_save_path_eps']
]:
if not full_path:
continue
# print('Saving query vis plot to {}'.format(sp))
create_dir_if_not_exists(dirname(full_path))
plt.savefig(full_path, dpi=info_dict['plot_dpi'])
plt.close()
def main():
person_group_id = FACE_GROUP_ID
display_image = False
face_api_wrapper = FaceAPIWrapper(FACE_API_KEY, FACE_BASE_URL)
create_dir_if_not_exists('captured_images/' + CAMERA_NAME)
print("Capturing Image every ", CAPTURE_INTERVAL, " seconds...")
i = 0
while 1:
try:
image_filename = 'captured_images/' + CAMERA_NAME + "/" + current_time_to_string(
) + ".jpg"
image = Camera().capture_image()
cv2.imwrite(image_filename, image)
if display_image:
cv2.imshow("Camera Image", image)
cv2.waitKey(1)
image_link = upload_to_s3(image_filename)
face_ids = face_api_wrapper.detect_faces(image=image_filename)
i += 1
print(i, "Captured at ", time.time())
if face_ids:
person_ids = \
face_api_wrapper.identify_faces(face_ids=face_ids,
large_person_group=person_group_id)
req_ids = [{id} for id in person_ids]
print("Detected Faces...", req_ids)
requests.post(REST_SERVER_URL + 'time-face-id',
data={
'lecture_number': get_lecture_number(),
'face_ids': req_ids,
'image-link': image_link,
'camera-name': CAMERA_NAME,
})
print("Present IDs:", req_ids)
time.sleep(CAPTURE_INTERVAL)
except Exception as e:
print(e)
def create_bigBed(grimm_synt_output, sizes_folder, species_file):
blocks = os.path.join(grimm_synt_output,'blocks.txt')
bed = os.path.join(grimm_synt_output,'bed')
utils.create_dir_if_not_exists(bed)
params = ['./blocks_to_bed.py', blocks, bed]
subprocess.check_call(params)
with open(species_file) as species:
for line in species :
line = line.strip().split(' ')
if len(line) < 2:
print 'species file format: genome_id genome_name'
print 'no bigBeds generated'
raise Exception('wrong species format')
params = ['sort -k1,1 -k2,2n', os.path.join(bed,line[0]+'.bed'), '>', os.path.join(bed,line[0]+'.sorted.bed')]
subprocess.check_call(" ".join(params), shell=True)
params = ['bedToBigBed','-type=bed6', os.path.join(bed,line[0]+'.sorted.bed'), os.path.join(sizes_folder,line[1]+'.sizes'), os.path.join(bed,line[1]+'.bb')]
subprocess.check_call(params)
print 'path to bigBed folder is',bed
def create_necessary_dirs(FLAGS, batches, class_per_frame):
create_dir_if_not_exists(FLAGS.output_labels_dir)
if FLAGS.copy_dir:
create_dir_if_not_exists(FLAGS.copy_dir)
classes_per_video = [class_per_frame[video].keys() for video in batches]
for i in range(len(classes_per_video)):
for klass in classes_per_video[i]:
class_path = os.path.join(FLAGS.copy_dir, klass)
video_path = os.path.join(class_path, batches[i])
create_dir_if_not_exists(class_path)
create_dir_if_not_exists(video_path)
def _save_fig(plt, dir, fn, need_eps=False, print_path=False):
plt_cnt = 0
if dir is None or fn is None:
return plt_cnt
final_path_without_ext = join(dir, fn)
exts = ['.png']
if need_eps:
exts += '.eps'
for ext in exts:
final_path = append_ext_to_filepath(ext, final_path_without_ext)
create_dir_if_not_exists(dirname(final_path))
try:
plt.savefig(final_path, bbox_inches='tight')
except:
warn('savefig')
if print_path:
print('Saved to {}'.format(final_path))
plt_cnt += 1
return plt_cnt
def _load_create_fake_pairs_if_needed(self, data):
if FLAGS.fake_generation:
assert ('fake_' in FLAGS.fake_generation)
fake_num = int(FLAGS.fake_generation.split('_')[1])
dir = get_save_path() + '/siamese_regession_fake_pairs'
if FLAGS.ds_metric == 'mcs':
dir += '_mcs'
create_dir_if_not_exists(dir)
filepath = dir + '/{}_fake_{}'.format(FLAGS.dataset_train,
fake_num)
ld = load(filepath)
if ld:
print('Loaded from {} with {} fake triples'.format(
filepath, len(ld)))
return ld
rtn = self._create_fake_pairs(data, fake_num)
save(filepath, rtn)
return rtn
else:
return []
def add_kml_file(request):
if request.method == 'POST':
form = KMLFilesForm(request.POST, request.FILES)
if form.is_valid():
kml_name = form.cleaned_data['name']
uploaded_file = request.FILES['filename']
file_name = uploaded_file.name
file_name = file_name.replace(" ", "_")
app_dir = _get_app_dir(__file__)
destination_name = app_dir + '/static/kml/' + file_name
create_dir_if_not_exists(destination_name)
destination = open(destination_name, 'wb+')
for chunk in uploaded_file.chunks():
destination.write(chunk)
destination.close()
kml_file = KMLFile(name=kml_name, filename=('static/kml/' + file_name))
kml_file.save()
return HttpResponseRedirect('/')
else:
return _render_response(form, request)
else:
form = KMLFilesForm()
return _render_response(form, request)
def run(path_to_fasta, path_to_results):
print utils.get_time()
print 'check that folder for results exists...'
utils.create_dir_if_not_exists(path_to_results)
print utils.get_time()
print 'running faToBit to convert fasta file to 2bit format...'
twobit = run_faToBit(path_to_fasta, path_to_results)
print utils.get_time()
print 'calculating gc content...'
wig = run_gcPercent(path_to_fasta, twobit, path_to_results)
print utils.get_time()
print 'cacluclating sizes of fasta sequences...'
sizes = utils.run_faSize(path_to_fasta, path_to_results)
print utils.get_time()
print 'converting wig to bigWig...'
bigWig = run_wigToBigWig(path_to_fasta, wig, sizes, path_to_results)
print utils.get_time()
print 'done.'
def run(path_to_fasta, path_to_results) : print utils.get_time() print 'check that folder for results exists...' utils.create_dir_if_not_exists(path_to_results) print utils.get_time() print 'running faToBit to convert fasta file to 2bit format...' twobit = run_faToBit(path_to_fasta, path_to_results) print utils.get_time() print 'calculating gc content...' wig = run_gcPercent(path_to_fasta, twobit, path_to_results) print utils.get_time() print 'cacluclating sizes of fasta sequences...' sizes = utils.run_faSize(path_to_fasta, path_to_results) print utils.get_time() print 'converting wig to bigWig...' bigWig = run_wigToBigWig(path_to_fasta,wig, sizes, path_to_results) print utils.get_time() print 'done.'
def plot_single_number_metric(dataset, dsmetric, models, rs, true_result, metric, norms,
ds_kernel=None,
thresh_poss=None, thresh_negs=None,
thresh_poss_sim=None, thresh_negs_sim=None,
plot_results=True,
extra_dir=None):
""" Plot mrr or mse. """
create_dir_if_not_exists('{}/{}/{}'.format(
get_result_path(), dataset, metric))
rtn = {}
if norms and thresh_poss and thresh_negs:
assert (len(norms) == len(thresh_poss) == len(thresh_negs))
for i, norm in enumerate(norms):
thresh_pos = thresh_poss[i] if thresh_poss else None
thresh_neg = thresh_negs[i] if thresh_negs else None
thresh_pos_sim = thresh_poss_sim[i] if thresh_poss_sim else None
thresh_neg_sim = thresh_negs_sim[i] if thresh_negs_sim else None
d = plot_single_number_metric_helper(
dataset, dsmetric, models, rs, true_result, metric, norm, ds_kernel,
thresh_pos, thresh_neg, thresh_pos_sim, thresh_neg_sim,
plot_results, extra_dir)
rtn.update(d)
return rtn
