def generate_for_grimm(): if len(sys.argv) < 6: print 'USAGE:', sys.argv[ 0], 'grimm', 'homology_file', 'first_specie_gff', 'second_specie_gff', 'output_directory' exit() print utils.get_time() print 'parsing homology file...' homology = parse_homology(sys.argv[2]) print utils.get_time() print 'parsing genes for the first species...' first_specie_genes = parse_genes(sys.argv[3]) print utils.get_time() print 'parsing genes for the second species...' second_specie_genes = parse_genes(sys.argv[4]) #homology = reduce_homology(first_specie_genes, second_specie_genes, homology) directory = sys.argv[5] utils.create_dir_if_not_exists(directory) file = os.path.join(directory, 'grimm.input') print utils.get_time() print 'writing genes to file...' coding_table = output_genes('genome1', first_specie_genes, homology, file) output_genes_and_rename('genome2', second_specie_genes, homology, coding_table, file) coding_table_file = os.path.join(directory, 'coding_table.txt') print utils.get_time() print 'saving coding table...' save_coding_table(coding_table, coding_table_file) print utils.get_time() print 'done.'
def __init__(self, model, time_series_list, checkpoint_path, output_dir): model.load_state_dict(torch.load(checkpoint_path)['model']) self.model = model self.time_series_list = time_series_list self.output_dir = output_dir create_dir_if_not_exists(self.output_dir)
def run(dir, maf, sizes_folder, species_file): grimm_synt_input_file = os.path.join(dir, utils.get_name(maf)+'.grimm_synt') print utils.get_time() print 'converting maf to input for grimm...' params = ['./generate_input_grimm_synt', maf, sizes_folder, grimm_synt_input_file, species_file] subprocess.check_call(params) print utils.get_time() print 'generating anchors for grimm_synt...' anchors_folder = os.path.join(dir,'grimm_synt_anchors') utils.create_dir_if_not_exists(anchors_folder) params = ['grimm_synt', '-A', '-f', grimm_synt_input_file, '-d', anchors_folder] subprocess.check_call(params) print utils.get_time() print 'running grimm_synt...' grimm_synt_output = os.path.join(dir,'grimm_synt_output') utils.create_dir_if_not_exists(grimm_synt_output) #used this for comparing mhc regions #params = ['grimm_synt','-f', os.path.join(anchors_folder, 'unique_coords.txt'),'-d',grimm_synt_output, '-m 1000 -g 1000 -c'] #params = ['grimm_synt','-f', os.path.join(anchors_folder, 'unique_coords.txt'),'-d',grimm_synt_output,'-m', '300000', '-g', '300000', '-c'] params = ['grimm_synt','-f', os.path.join(anchors_folder, 'unique_coords.txt'),'-d',grimm_synt_output,'-m', '100000', '-g', '100000', '-c'] subprocess.check_call(params) print 'synteny blocks are at',os.path.join(grimm_synt_output,'blocks.txt') print utils.get_time() print 'creating bigBed files...' create_bigBed(grimm_synt_output, sizes_folder, species_file) print utils.get_time() print 'running grimm...' params = ['grimm', '-f', os.path.join(grimm_synt_output,'mgr_macro.txt'), '-o', os.path.join(dir,'grimm.output')] subprocess.call(" ".join(params), shell=True) print 'grimm output is saved to', os.path.join(dir,'grimm.output') print utils.get_time() print 'done.'
def generate_for_grimm() : if len(sys.argv) < 6: print 'USAGE:', sys.argv[0], 'grimm', 'homology_file', 'first_specie_gff', 'second_specie_gff', 'output_directory' exit() print utils.get_time() print 'parsing homology file...' homology = parse_homology(sys.argv[2]) print utils.get_time() print 'parsing genes for the first species...' first_specie_genes = parse_genes(sys.argv[3]) print utils.get_time() print 'parsing genes for the second species...' second_specie_genes = parse_genes(sys.argv[4]) #homology = reduce_homology(first_specie_genes, second_specie_genes, homology) directory = sys.argv[5] utils.create_dir_if_not_exists(directory) file = os.path.join(directory,'grimm.input') print utils.get_time() print 'writing genes to file...' coding_table = output_genes('genome1', first_specie_genes, homology, file) output_genes_and_rename('genome2', second_specie_genes, homology, coding_table, file) coding_table_file = os.path.join(directory,'coding_table.txt') print utils.get_time() print 'saving coding table...' save_coding_table(coding_table, coding_table_file) print utils.get_time() print 'done.'
def create_bigBed(grimm_synt_output, sizes_folder, species_file): blocks = os.path.join(grimm_synt_output, 'blocks.txt') bed = os.path.join(grimm_synt_output, 'bed') utils.create_dir_if_not_exists(bed) params = ['./blocks_to_bed.py', blocks, bed] subprocess.check_call(params) with open(species_file) as species: for line in species: line = line.strip().split(' ') if len(line) < 2: print 'species file format: genome_id genome_name' print 'no bigBeds generated' raise Exception('wrong species format') params = [ 'sort -k1,1 -k2,2n', os.path.join(bed, line[0] + '.bed'), '>', os.path.join(bed, line[0] + '.sorted.bed') ] subprocess.check_call(" ".join(params), shell=True) params = [ 'bedToBigBed', '-type=bed6', os.path.join(bed, line[0] + '.sorted.bed'), os.path.join(sizes_folder, line[1] + '.sizes'), os.path.join(bed, line[1] + '.bb') ] subprocess.check_call(params) print 'path to bigBed folder is', bed
def run(fasta, ncrna, results) : print utils.get_time() print 'check that folder for results exists...' utils.create_dir_if_not_exists(results) print utils.get_time() print 'running blat...' blat_psl = run_blat(fasta, ncrna, results) #blat_psl = '/hive/groups/recon/projs/pipeline_data/assemblies/FelisCatus/6.2/tracks/ncrna/2014-10-18/FelisCatus.psl' #blat_psl = '/hive/groups/recon/projs/pipeline_data/assemblies/FelisCatus/6.2/tracks/ncrna/FelisCatus.sorted.only_entries.filtered.psl' print utils.get_time() print 'sorting blat output and adjusting file to future processing...' sorted_psl = run_sorting(blat_psl, results) print utils.get_time() print 'filtering blat results...' filtered_psl = run_filtering(sorted_psl, results) print utils.get_time() print 'calculating sizes of fasta sequences...' sizes = utils.run_faSize(fasta, results) #sizes = '/hive/groups/recon/projs/pipeline_data/assemblies/FelisCatus/6.2/tracks/ncrna/2014-10-18/FelisCatus.sizes' print utils.get_time() print 'converting psl to bigBed' run_pslToBigBed(filtered_psl, sizes, results) print utils.get_time() print 'done.'
def __init__(self, model, criterion, optimizer, scheduler, time_series_list, train_valid_split=0.8, num_epochs=30, early_stopping=5, batch_size=64, checkpoint_path="../checkpoints/seq2seq.h5", plot=True, offet_for_plot=50): self.model = model self.criterion = criterion self.optimizer = optimizer self.scheduler = scheduler self.num_epochs = num_epochs self.early_stopping = early_stopping self.train_valid_split = train_valid_split self.batch_size = batch_size self.checkpoint_path = checkpoint_path self.time_series_list = time_series_list self.plot_ts = [] self.plot = plot self.offset = offet_for_plot create_dir_if_not_exists(self.checkpoint_path) self._make_loader()
def get_gs_ds_mat(gs1, gs2, dist_sim_calculator, tvt1, tvt2, dataset, dist_metric, dist_algo, norm, dec_gsize, return_neg1=False): mat_str = '{}({})_{}({})'.format(tvt1, len(gs1), tvt2, len(gs2)) dir = '{}/ds_mat'.format(get_save_path()) create_dir_if_not_exists(dir) sfn = '{}/{}_{}_ds_mat_{}{}_{}'.format( dir, dataset, mat_str, dist_metric, get_norm_str(norm), dist_algo) l = load(sfn) if l is not None: print('Loaded from {}'.format(sfn)) return l m = len(gs1) n = len(gs2) dist_mat = np.zeros((m, n)) for i in range(m): for j in range(n): g1 = gs1[i] g2 = gs2[j] d, normed_d = dist_sim_calculator.calculate_dist_sim( g1, g2, dec_gsize=dec_gsize, return_neg1=return_neg1) if norm: dist_mat[i][j] = normed_d else: dist_mat[i][j] = d save(sfn, dist_mat) print('Saved to {}'.format(sfn)) return dist_mat
def main(opts): if not opts.do_train and not opts.do_eval: raise ValueError('At least one of `do_train` or `do_eval` must be True.') if os.path.exists(opts.output_dir) and os.listdir(opts.output_dir) and opts.do_train: raise ValueError('Output directory ({}) already exists and is not empty.'.format(opts.output_dir)) # Create the output directory (if not exists) create_dir_if_not_exists(opts.output_dir) # Device device type opts.device = torch.device('cuda' if torch.cuda.is_available() and not opts.no_cuda else 'cpu') opts.n_gpus = torch.cuda.device_count() if str(opts.device) == 'cuda' else 0 print('Device Type: {} | Number of GPUs: {}'.format(opts.device, opts.n_gpus), flush=True) # Load Datasets and Ontology dataset = MyDataset() if opts.do_train: # Load model from scratch model = Model.from_scratch('bert-base-chinese') model.move_to_device(opts) print('Number of model parameters is: {}'.format(get_n_params(model))) # Start Training print('Start Training', flush=True) model.run_train(dataset, opts) # Free up all memory pytorch is taken from gpu memory del model torch.cuda.empty_cache()
def setup_temp_data_folder(gp, append_str): dir = gp + '/data' create_dir_if_not_exists(dir) tp = dir + '/temp_{}'.format(append_str) exec_cmd('rm -rf {} && mkdir {}'.format(tp, tp)) src = get_root_path() + '/src/gmt_files' exec_cmd('cp {}/temp.xml {}/temp_{}.xml'.format(src, tp, append_str)) return src, tp
def exp5(): """ Query visualization. """ dataset = 'imdbmulti' model = 'astar' concise = True norms = [True, False] dir = get_result_path() + '/{}/query_vis/{}'.format(dataset, model) create_dir_if_not_exists(dir) info_dict = { # draw node config 'draw_node_size': 150 if dataset != 'linux' else 10, 'draw_node_label_enable': True, 'node_label_name': None if dataset == 'linux' else 'type', 'draw_node_label_font_size': 6, 'draw_node_color_map': TYPE_COLOR_MAP, # draw edge config 'draw_edge_label_enable': False, 'edge_label_name': 'valence', 'draw_edge_label_font_size': 6, # graph text info config 'each_graph_text_list': [], 'each_graph_text_font_size': 8, 'each_graph_text_pos': [0.5, 1.05], # graph padding: value range: [0, 1] 'top_space': 0.20 if concise else 0.26, # out of whole graph 'bottom_space': 0.05, 'hbetween_space': 0.6 if concise else 1, # out of the subgraph 'wbetween_space': 0, # plot config 'plot_dpi': 200, 'plot_save_path_eps': '', 'plot_save_path_png': '' } train_data = load_data(dataset, train=True) test_data = load_data(dataset, train=False) row_graphs = test_data.graphs col_graphs = train_data.graphs r = load_result(dataset, model, row_graphs=row_graphs, col_graphs=col_graphs) tr = load_result(dataset, TRUE_MODEL, row_graphs=row_graphs, col_graphs=col_graphs) for norm in norms: ids = r.get_sort_id_mat(norm) m, n = r.m_n() num_vis = 10 for i in range(num_vis): q = test_data.graphs[i] gids = np.concatenate([ids[i][:3], [ids[i][int(n / 2)]], ids[i][-3:]]) gs = [train_data.graphs[j] for j in gids] info_dict['each_graph_text_list'] = \ [get_text_label(dataset, r, tr, i, i, q, model, norm, True, concise)] + \ [get_text_label(dataset, r, tr, i, j, train_data.graphs[j], model, norm, False, concise) \ for j in gids] # print(info_dict['each_graph_text_list']) info_dict['plot_save_path_png'] = '{}/query_vis_{}_{}_{}{}.{}'.format( dir, dataset, model, i, get_norm_str(norm), 'png') info_dict['plot_save_path_eps'] = '{}/query_vis_{}_{}_{}{}.{}'.format( dir, dataset, model, i, get_norm_str(norm), 'eps') vis(q, gs, info_dict)
def __init__(self): model_str = self.get_model_str() self.logdir = join(get_root_path(), 'logs', '{}_{}'.format(model_str, get_ts())) create_dir_if_not_exists(self.logdir) self.model_info_f = self._open('model_info.txt') self._log_model_info() self._save_conf_code() print('Logging to {}'.format(self.logdir))
def gen_dataset(graphs): random.Random(123).shuffle(graphs) dirout_train = get_data_path() + '/IMDBMulti/train' dirout_test = get_data_path() + '/IMDBMulti/test' create_dir_if_not_exists(dirout_train) create_dir_if_not_exists(dirout_test) for g in graphs[0:1200]: nx.write_gexf(g, dirout_train + '/{}.gexf'.format(g.graph['gid'])) for g in graphs[1200:]: nx.write_gexf(g, dirout_test + '/{}.gexf'.format(g.graph['gid']))
def __init__(self, sess): if FLAGS.log: model_str = self._get_model_str() logdir = '{}/logs/{}_{}'.format(get_siamese_dir(), model_str, get_ts()) create_dir_if_not_exists(logdir) self.tw = tf.summary.FileWriter(logdir + '/train', sess.graph) self.vw = tf.summary.FileWriter(logdir + '/val', sess.graph) self.merged = tf.summary.merge_all() self._log_model_info(logdir, sess) print('Logging to {}'.format(logdir))
def gen_dataset(graphs): random.Random(123).shuffle(graphs) dirout_train = get_data_path() + '/{}/train'.format(conf.outfolder) dirout_test = get_data_path() + '/{}/test'.format(conf.outfolder) create_dir_if_not_exists(dirout_train) create_dir_if_not_exists(dirout_test) sp = int(len(graphs) * conf.train_perc_) for g in graphs[0:sp]: nx.write_gexf(g, dirout_train + '/{}.gexf'.format(g.graph['gid'])) for g in graphs[sp:]: nx.write_gexf(g, dirout_test + '/{}.gexf'.format(g.graph['gid']))
def run(config, dir, outputhal): utils.check_existence_or_raise(config) utils.create_dir_if_not_exists(dir) command = ['runProgressiveCactus.sh', '--batchSystem', 'parasol', '--bigBatchSystem', 'singleMachine', '--defaultMemory', '8589934593', '--bigMemoryThreshold', '8589934592', '--bigMaxMemory', '893353197568', '--bigMaxCpus', '25', '--maxThreads', '25', '--parasolCommand=/cluster/home/jcarmstr/bin/parasol -host=ku', '--retryCount', '3'] params = command + [config, dir, outputhal] print params subprocess.check_call(params) return outputhal
def get_gs_ds_mat(gs1, gs2, dist_sim_calculator, tvt1, tvt2, dataset, dist_metric, dist_algo, norm, dec_gsize, return_neg1=False): if logging_enabled == True: print("- Entered dist_sim_calculator::get_gs_ds_mat Global Method") mat_str = '{}({})_{}({})'.format(tvt1, len(gs1), tvt2, len(gs2)) dir = '{}\\ds_mat'.format(get_save_path()) create_dir_if_not_exists(dir) sfn = '{}\\{}_{}_ds_mat_{}{}_{}'.format(dir, dataset, mat_str, dist_metric, get_norm_str(norm), dist_algo) l = load(sfn) if l is not None: print('Loaded from {}'.format(sfn)) return l if not dist_sim_calculator.gidpair_ds_map: # dist_sim_calculator.initial_calculate_dist_sim(gs1, gs2) dist_sim_calculator.initial_dist_sim_pairs_with_netcomp(gs1, gs2) m = len(gs1) n = len(gs2) dist_mat = np.zeros((m, n)) for i in range(m): for j in range(n): g1 = gs1[i] g2 = gs2[j] d, normed_d = dist_sim_calculator.calculate_dist_sim( g1, g2, dec_gsize=dec_gsize, return_neg1=return_neg1) if norm: dist_mat[i][j] = normed_d print("i: ", i, ", j: ", j, ", d: ", d, ", normed_d: ", normed_d) else: dist_mat[i][j] = d save(sfn, dist_mat) print('Saved to {}'.format(sfn)) return dist_mat
def run_filtering(sorted_psl, results) : utils.check_existence_or_raise(sorted_psl) stats = os.path.join(results,'filtering_stats') utils.create_dir_if_not_exists(stats) filtered_psl = os.path.join(results,utils.get_name(sorted_psl)+'.filtered.psl') tmp = utils.atomic_tmp_file(filtered_psl) params = ['pslCDnaFilter','-localNearBest=0.1 -ignoreNs -minCover=0.6 -repsAsMatch', ' -dropped='+os.path.join(stats,'dropped.out'), '-weirdOverlapped='+os.path.join(stats,'weird_overlapped.out'), '-alignStats='+os.path.join(stats,'align_stats.out'), '-statsOut='+os.path.join(stats,'overall_stats.out'), sorted_psl, tmp] subprocess.call(" ".join(params), shell=True) utils.atomic_install(tmp, filtered_psl) return filtered_psl
def __init__(self, sess=None): model_str = self._get_model_str() self.logdir = '{}/logs/{}_{}'.format(get_siamese_dir(), model_str, get_ts()) create_dir_if_not_exists(self.logdir) if sess is not None: self.tw = tf.summary.FileWriter(self.logdir + '/train', sess.graph) self.all_merged = tf.summary.merge_all() self.loss_merged = tf.summary.merge( self._extract_loss_related_summaries_as_list()) self._log_model_info(self.logdir, sess) self.f = open('{}/results_{}.txt'.format(self.logdir, get_ts()), 'w') print('Logging to {}'.format(self.logdir))
def main(): person_group_id = FACE_GROUP_ID display_image = False face_api_wrapper = FaceAPIWrapper(FACE_API_KEY, FACE_BASE_URL) create_dir_if_not_exists('captured_images/' + CAMERA_NAME) print("Capturing Image every ", CAPTURE_INTERVAL, " seconds...") i = 0 while 1: try: image_filename = 'captured_images/' + CAMERA_NAME + "/" + current_time_to_string() + ".jpg" image = Camera().capture_image() cv2.imwrite(image_filename, image) if display_image: cv2.imshow("Camera Image", image) cv2.waitKey(1) image_link = upload_to_s3(image_filename) image_url = 'https://mybuckfucket.s3-eu-west-1.amazonaws.com/public_folder/'+image_filename face_ids = face_api_wrapper.detect_faces(image=image_filename) i += 1 print(i, "Captured at ", current_time_to_string()) print("S3 url",image_url) if face_ids: person_ids = \ face_api_wrapper.identify_faces(face_ids=face_ids, large_person_group=person_group_id) req_ids = [{id} for id in person_ids] print("Detected Faces...", req_ids) contents=load_dict_from_file('json.txt') #my_data=(contents[0]['faceAttributes']) requests.post('http://127.0.0.1:8000/students/', data={ 'attributes' : contents, 'name' : "Darragh", 'time_date' : current_time_to_string(), 'face_ids' : person_ids, 'image_link' : image_url, }) print("#####",data) #print(contents[0]['faceAttributes']['emotion']) #requests.post('http://127.0.0.1:8000/students/', data=contents) time.sleep(CAPTURE_INTERVAL) except Exception as e: print(e)
def run(dir, maf, sizes_folder, species_file): grimm_synt_input_file = os.path.join(dir, utils.get_name(maf) + '.grimm_synt') print utils.get_time() print 'converting maf to input for grimm...' params = [ './generate_input_grimm_synt', maf, sizes_folder, grimm_synt_input_file, species_file ] subprocess.check_call(params) print utils.get_time() print 'generating anchors for grimm_synt...' anchors_folder = os.path.join(dir, 'grimm_synt_anchors') utils.create_dir_if_not_exists(anchors_folder) params = [ 'grimm_synt', '-A', '-f', grimm_synt_input_file, '-d', anchors_folder ] subprocess.check_call(params) print utils.get_time() print 'running grimm_synt...' grimm_synt_output = os.path.join(dir, 'grimm_synt_output') utils.create_dir_if_not_exists(grimm_synt_output) #used this for comparing mhc regions #params = ['grimm_synt','-f', os.path.join(anchors_folder, 'unique_coords.txt'),'-d',grimm_synt_output, '-m 1000 -g 1000 -c'] #params = ['grimm_synt','-f', os.path.join(anchors_folder, 'unique_coords.txt'),'-d',grimm_synt_output,'-m', '300000', '-g', '300000', '-c'] params = [ 'grimm_synt', '-f', os.path.join(anchors_folder, 'unique_coords.txt'), '-d', grimm_synt_output, '-m', '100000', '-g', '100000', '-c' ] subprocess.check_call(params) print 'synteny blocks are at', os.path.join(grimm_synt_output, 'blocks.txt') print utils.get_time() print 'creating bigBed files...' create_bigBed(grimm_synt_output, sizes_folder, species_file) print utils.get_time() print 'running grimm...' params = [ 'grimm', '-f', os.path.join(grimm_synt_output, 'mgr_macro.txt'), '-o', os.path.join(dir, 'grimm.output') ] subprocess.call(" ".join(params), shell=True) print 'grimm output is saved to', os.path.join(dir, 'grimm.output') print utils.get_time() print 'done.'
def plot_preck(dataset, dsmetric, models, rs, true_result, metric, norms, plot_results=True, extra_dir=None): """ Plot prec@k. """ create_dir_if_not_exists('{}/{}/{}'.format( get_result_path(), dataset, metric)) rtn = {} for norm in norms: _, n = true_result.m_n() ks = range(1, n) d = plot_preck_helper( dataset, dsmetric, models, rs, true_result, metric, norm, ks, False, plot_results, extra_dir) rtn.update(d) return rtn
def run(config, dir, outputhal): utils.check_existence_or_raise(config) utils.create_dir_if_not_exists(dir) command = [ 'runProgressiveCactus.sh', '--batchSystem', 'parasol', '--bigBatchSystem', 'singleMachine', '--defaultMemory', '8589934593', '--bigMemoryThreshold', '8589934592', '--bigMaxMemory', '893353197568', '--bigMaxCpus', '25', '--maxThreads', '25', '--parasolCommand=/cluster/home/jcarmstr/bin/parasol -host=ku', '--retryCount', '3' ] params = command + [config, dir, outputhal] print params subprocess.check_call(params) return outputhal
def __init__(self, sess=None): if logging_enabled == True: print("- Entered saver::Saver::__init__ Constructor Method") model_str = self._get_model_str() self.logdir = '{}\\logs\\{}'.format(get_siamese_dir(), model_str) create_dir_if_not_exists(self.logdir) if sess is not None: self.tw = tf.compat.v1.summary.FileWriter(self.logdir + '\\train', sess.graph) self.all_merged = tf.compat.v1.summary.merge_all() self.loss_merged = tf.compat.v1.summary.merge(self._extract_loss_related_summaries_as_list()) self.f = open('{}\\results.txt'.format(self.logdir), 'w') print('Logging to {}'.format(self.logdir))
def save_fig(plt, dir, fn, print_path=False): plt_cnt = 0 if dir is None or fn is None: return plt_cnt final_path_without_ext = '{}/{}'.format(dir, fn) for ext in ['png', 'eps']: full_path = final_path_without_ext + '.' + ext create_dir_if_not_exists(dirname(full_path)) try: plt.savefig(full_path, bbox_inches='tight') except: warn('savefig') if print_path: print('Saved to {}'.format(full_path)) plt_cnt += 1 return plt_cnt
def select_dump_graphs(graphs, less_than_eq_10_ids): remove_type(graphs) random.Random(123).shuffle(less_than_eq_10_ids) tr = 800 te = 200 assert (tr + te <= len(graphs)) dirout = get_data_path() + '/linux' train_dir = dirout + '/train' create_dir_if_not_exists(train_dir) for i in range(tr): gid = less_than_eq_10_ids[i] nx.write_gexf(graphs[gid], train_dir + '/{}.gexf'.format(gid)) test_dir = dirout + '/test' create_dir_if_not_exists(test_dir) for i in range(tr, tr + te): gid = less_than_eq_10_ids[i] nx.write_gexf(graphs[gid], test_dir + '/{}.gexf'.format(gid))
def save_figs(info_dict): save_path = info_dict['plot_save_path_png'] print(save_path) if not save_path: # print('plt.show') plt.show() else: for full_path in [ info_dict['plot_save_path_png'], info_dict['plot_save_path_eps'] ]: if not full_path: continue # print('Saving query vis plot to {}'.format(sp)) create_dir_if_not_exists(dirname(full_path)) plt.savefig(full_path, dpi=info_dict['plot_dpi']) plt.close()
def main(): person_group_id = FACE_GROUP_ID display_image = False face_api_wrapper = FaceAPIWrapper(FACE_API_KEY, FACE_BASE_URL) create_dir_if_not_exists('captured_images/' + CAMERA_NAME) print("Capturing Image every ", CAPTURE_INTERVAL, " seconds...") i = 0 while 1: try: image_filename = 'captured_images/' + CAMERA_NAME + "/" + current_time_to_string( ) + ".jpg" image = Camera().capture_image() cv2.imwrite(image_filename, image) if display_image: cv2.imshow("Camera Image", image) cv2.waitKey(1) image_link = upload_to_s3(image_filename) face_ids = face_api_wrapper.detect_faces(image=image_filename) i += 1 print(i, "Captured at ", time.time()) if face_ids: person_ids = \ face_api_wrapper.identify_faces(face_ids=face_ids, large_person_group=person_group_id) req_ids = [{id} for id in person_ids] print("Detected Faces...", req_ids) requests.post(REST_SERVER_URL + 'time-face-id', data={ 'lecture_number': get_lecture_number(), 'face_ids': req_ids, 'image-link': image_link, 'camera-name': CAMERA_NAME, }) print("Present IDs:", req_ids) time.sleep(CAPTURE_INTERVAL) except Exception as e: print(e)
def create_bigBed(grimm_synt_output, sizes_folder, species_file): blocks = os.path.join(grimm_synt_output,'blocks.txt') bed = os.path.join(grimm_synt_output,'bed') utils.create_dir_if_not_exists(bed) params = ['./blocks_to_bed.py', blocks, bed] subprocess.check_call(params) with open(species_file) as species: for line in species : line = line.strip().split(' ') if len(line) < 2: print 'species file format: genome_id genome_name' print 'no bigBeds generated' raise Exception('wrong species format') params = ['sort -k1,1 -k2,2n', os.path.join(bed,line[0]+'.bed'), '>', os.path.join(bed,line[0]+'.sorted.bed')] subprocess.check_call(" ".join(params), shell=True) params = ['bedToBigBed','-type=bed6', os.path.join(bed,line[0]+'.sorted.bed'), os.path.join(sizes_folder,line[1]+'.sizes'), os.path.join(bed,line[1]+'.bb')] subprocess.check_call(params) print 'path to bigBed folder is',bed
def create_necessary_dirs(FLAGS, batches, class_per_frame): create_dir_if_not_exists(FLAGS.output_labels_dir) if FLAGS.copy_dir: create_dir_if_not_exists(FLAGS.copy_dir) classes_per_video = [class_per_frame[video].keys() for video in batches] for i in range(len(classes_per_video)): for klass in classes_per_video[i]: class_path = os.path.join(FLAGS.copy_dir, klass) video_path = os.path.join(class_path, batches[i]) create_dir_if_not_exists(class_path) create_dir_if_not_exists(video_path)
def _save_fig(plt, dir, fn, need_eps=False, print_path=False): plt_cnt = 0 if dir is None or fn is None: return plt_cnt final_path_without_ext = join(dir, fn) exts = ['.png'] if need_eps: exts += '.eps' for ext in exts: final_path = append_ext_to_filepath(ext, final_path_without_ext) create_dir_if_not_exists(dirname(final_path)) try: plt.savefig(final_path, bbox_inches='tight') except: warn('savefig') if print_path: print('Saved to {}'.format(final_path)) plt_cnt += 1 return plt_cnt
def _load_create_fake_pairs_if_needed(self, data): if FLAGS.fake_generation: assert ('fake_' in FLAGS.fake_generation) fake_num = int(FLAGS.fake_generation.split('_')[1]) dir = get_save_path() + '/siamese_regession_fake_pairs' if FLAGS.ds_metric == 'mcs': dir += '_mcs' create_dir_if_not_exists(dir) filepath = dir + '/{}_fake_{}'.format(FLAGS.dataset_train, fake_num) ld = load(filepath) if ld: print('Loaded from {} with {} fake triples'.format( filepath, len(ld))) return ld rtn = self._create_fake_pairs(data, fake_num) save(filepath, rtn) return rtn else: return []
def add_kml_file(request): if request.method == 'POST': form = KMLFilesForm(request.POST, request.FILES) if form.is_valid(): kml_name = form.cleaned_data['name'] uploaded_file = request.FILES['filename'] file_name = uploaded_file.name file_name = file_name.replace(" ", "_") app_dir = _get_app_dir(__file__) destination_name = app_dir + '/static/kml/' + file_name create_dir_if_not_exists(destination_name) destination = open(destination_name, 'wb+') for chunk in uploaded_file.chunks(): destination.write(chunk) destination.close() kml_file = KMLFile(name=kml_name, filename=('static/kml/' + file_name)) kml_file.save() return HttpResponseRedirect('/') else: return _render_response(form, request) else: form = KMLFilesForm() return _render_response(form, request)
def run(path_to_fasta, path_to_results): print utils.get_time() print 'check that folder for results exists...' utils.create_dir_if_not_exists(path_to_results) print utils.get_time() print 'running faToBit to convert fasta file to 2bit format...' twobit = run_faToBit(path_to_fasta, path_to_results) print utils.get_time() print 'calculating gc content...' wig = run_gcPercent(path_to_fasta, twobit, path_to_results) print utils.get_time() print 'cacluclating sizes of fasta sequences...' sizes = utils.run_faSize(path_to_fasta, path_to_results) print utils.get_time() print 'converting wig to bigWig...' bigWig = run_wigToBigWig(path_to_fasta, wig, sizes, path_to_results) print utils.get_time() print 'done.'
def run(path_to_fasta, path_to_results) : print utils.get_time() print 'check that folder for results exists...' utils.create_dir_if_not_exists(path_to_results) print utils.get_time() print 'running faToBit to convert fasta file to 2bit format...' twobit = run_faToBit(path_to_fasta, path_to_results) print utils.get_time() print 'calculating gc content...' wig = run_gcPercent(path_to_fasta, twobit, path_to_results) print utils.get_time() print 'cacluclating sizes of fasta sequences...' sizes = utils.run_faSize(path_to_fasta, path_to_results) print utils.get_time() print 'converting wig to bigWig...' bigWig = run_wigToBigWig(path_to_fasta,wig, sizes, path_to_results) print utils.get_time() print 'done.'
def plot_single_number_metric(dataset, dsmetric, models, rs, true_result, metric, norms, ds_kernel=None, thresh_poss=None, thresh_negs=None, thresh_poss_sim=None, thresh_negs_sim=None, plot_results=True, extra_dir=None): """ Plot mrr or mse. """ create_dir_if_not_exists('{}/{}/{}'.format( get_result_path(), dataset, metric)) rtn = {} if norms and thresh_poss and thresh_negs: assert (len(norms) == len(thresh_poss) == len(thresh_negs)) for i, norm in enumerate(norms): thresh_pos = thresh_poss[i] if thresh_poss else None thresh_neg = thresh_negs[i] if thresh_negs else None thresh_pos_sim = thresh_poss_sim[i] if thresh_poss_sim else None thresh_neg_sim = thresh_negs_sim[i] if thresh_negs_sim else None d = plot_single_number_metric_helper( dataset, dsmetric, models, rs, true_result, metric, norm, ds_kernel, thresh_pos, thresh_neg, thresh_pos_sim, thresh_neg_sim, plot_results, extra_dir) rtn.update(d) return rtn