def generate_for_grimm():
    if len(sys.argv) < 6:
        print 'USAGE:', sys.argv[
            0], 'grimm', 'homology_file', 'first_specie_gff', 'second_specie_gff', 'output_directory'
        exit()
    print utils.get_time()
    print 'parsing homology file...'
    homology = parse_homology(sys.argv[2])
    print utils.get_time()
    print 'parsing genes for the first species...'
    first_specie_genes = parse_genes(sys.argv[3])
    print utils.get_time()
    print 'parsing genes for the second species...'
    second_specie_genes = parse_genes(sys.argv[4])
    #homology = reduce_homology(first_specie_genes, second_specie_genes, homology)
    directory = sys.argv[5]
    utils.create_dir_if_not_exists(directory)
    file = os.path.join(directory, 'grimm.input')
    print utils.get_time()
    print 'writing genes to file...'
    coding_table = output_genes('genome1', first_specie_genes, homology, file)
    output_genes_and_rename('genome2', second_specie_genes, homology,
                            coding_table, file)
    coding_table_file = os.path.join(directory, 'coding_table.txt')
    print utils.get_time()
    print 'saving coding table...'
    save_coding_table(coding_table, coding_table_file)
    print utils.get_time()
    print 'done.'
    def __init__(self, model, time_series_list, checkpoint_path, output_dir):
        model.load_state_dict(torch.load(checkpoint_path)['model'])
        self.model = model
        self.time_series_list = time_series_list
        self.output_dir = output_dir

        create_dir_if_not_exists(self.output_dir)
def run(dir, maf, sizes_folder, species_file):
    grimm_synt_input_file = os.path.join(dir, utils.get_name(maf)+'.grimm_synt')
   
    print utils.get_time()
    print 'converting maf to input for grimm...'
    params = ['./generate_input_grimm_synt', maf, sizes_folder, grimm_synt_input_file, species_file]
    subprocess.check_call(params)
    
    print utils.get_time()
    print 'generating anchors for grimm_synt...'
    anchors_folder = os.path.join(dir,'grimm_synt_anchors')
    utils.create_dir_if_not_exists(anchors_folder)
    params = ['grimm_synt', '-A', '-f', grimm_synt_input_file, '-d', anchors_folder]
    subprocess.check_call(params)
    print utils.get_time()
    print 'running grimm_synt...'
    grimm_synt_output = os.path.join(dir,'grimm_synt_output')
    utils.create_dir_if_not_exists(grimm_synt_output)
    #used this for comparing mhc regions
    #params = ['grimm_synt','-f', os.path.join(anchors_folder, 'unique_coords.txt'),'-d',grimm_synt_output, '-m 1000 -g 1000 -c']
    #params = ['grimm_synt','-f', os.path.join(anchors_folder, 'unique_coords.txt'),'-d',grimm_synt_output,'-m', '300000', '-g', '300000', '-c']
    params = ['grimm_synt','-f', os.path.join(anchors_folder, 'unique_coords.txt'),'-d',grimm_synt_output,'-m', '100000', '-g', '100000', '-c']
    subprocess.check_call(params)
    print 'synteny blocks are at',os.path.join(grimm_synt_output,'blocks.txt')
    print utils.get_time()
    print 'creating bigBed files...'
    create_bigBed(grimm_synt_output, sizes_folder, species_file)
    print utils.get_time()
    print 'running grimm...'
    params = ['grimm', '-f', os.path.join(grimm_synt_output,'mgr_macro.txt'), '-o', os.path.join(dir,'grimm.output')]
    subprocess.call(" ".join(params), shell=True)
    print 'grimm output is saved to', os.path.join(dir,'grimm.output')
    print utils.get_time()
    
    print 'done.'
def generate_for_grimm() :
    if len(sys.argv) < 6:
        print 'USAGE:', sys.argv[0], 'grimm', 'homology_file', 'first_specie_gff', 'second_specie_gff', 'output_directory'
        exit()
    print utils.get_time()
    print 'parsing homology file...'
    homology = parse_homology(sys.argv[2])
    print utils.get_time()
    print 'parsing genes for the first species...'
    first_specie_genes = parse_genes(sys.argv[3])
    print utils.get_time()
    print 'parsing genes for the second species...'
    second_specie_genes = parse_genes(sys.argv[4])
    #homology = reduce_homology(first_specie_genes, second_specie_genes, homology)
    directory = sys.argv[5]
    utils.create_dir_if_not_exists(directory)
    file = os.path.join(directory,'grimm.input')
    print utils.get_time()
    print 'writing genes to file...'
    coding_table = output_genes('genome1', first_specie_genes, homology, file)
    output_genes_and_rename('genome2', second_specie_genes, homology, coding_table, file)
    coding_table_file = os.path.join(directory,'coding_table.txt')
    print utils.get_time()
    print 'saving coding table...'
    save_coding_table(coding_table, coding_table_file)
    print utils.get_time()
    print 'done.'
Beispiel #5
0
def create_bigBed(grimm_synt_output, sizes_folder, species_file):
    blocks = os.path.join(grimm_synt_output, 'blocks.txt')
    bed = os.path.join(grimm_synt_output, 'bed')
    utils.create_dir_if_not_exists(bed)
    params = ['./blocks_to_bed.py', blocks, bed]
    subprocess.check_call(params)
    with open(species_file) as species:
        for line in species:
            line = line.strip().split(' ')
            if len(line) < 2:
                print 'species file format: genome_id genome_name'
                print 'no bigBeds generated'
                raise Exception('wrong species format')
            params = [
                'sort -k1,1 -k2,2n',
                os.path.join(bed, line[0] + '.bed'), '>',
                os.path.join(bed, line[0] + '.sorted.bed')
            ]
            subprocess.check_call(" ".join(params), shell=True)
            params = [
                'bedToBigBed', '-type=bed6',
                os.path.join(bed, line[0] + '.sorted.bed'),
                os.path.join(sizes_folder, line[1] + '.sizes'),
                os.path.join(bed, line[1] + '.bb')
            ]
            subprocess.check_call(params)
    print 'path to bigBed folder is', bed
Beispiel #6
0
def run(fasta, ncrna, results) :
    print utils.get_time()
    print 'check that folder for results exists...'
    utils.create_dir_if_not_exists(results)
    
    print utils.get_time()
    print 'running blat...'
    blat_psl = run_blat(fasta, ncrna, results)
    
    #blat_psl = '/hive/groups/recon/projs/pipeline_data/assemblies/FelisCatus/6.2/tracks/ncrna/2014-10-18/FelisCatus.psl'
    #blat_psl = '/hive/groups/recon/projs/pipeline_data/assemblies/FelisCatus/6.2/tracks/ncrna/FelisCatus.sorted.only_entries.filtered.psl'
    print utils.get_time()
    print 'sorting blat output and adjusting file to future processing...'
    sorted_psl = run_sorting(blat_psl, results)	
    
    print utils.get_time()
    print 'filtering blat results...'
    filtered_psl = run_filtering(sorted_psl, results)
    
    print utils.get_time()
    print 'calculating sizes of fasta sequences...'
    sizes = utils.run_faSize(fasta, results)
    
    #sizes = '/hive/groups/recon/projs/pipeline_data/assemblies/FelisCatus/6.2/tracks/ncrna/2014-10-18/FelisCatus.sizes'
    print utils.get_time()
    print 'converting psl to bigBed'
    run_pslToBigBed(filtered_psl, sizes, results)
    
    print utils.get_time()
    print 'done.'
Beispiel #7
0
    def __init__(self,
                 model,
                 criterion,
                 optimizer,
                 scheduler,
                 time_series_list,
                 train_valid_split=0.8,
                 num_epochs=30,
                 early_stopping=5,
                 batch_size=64,
                 checkpoint_path="../checkpoints/seq2seq.h5",
                 plot=True,
                 offet_for_plot=50):
        self.model = model
        self.criterion = criterion
        self.optimizer = optimizer
        self.scheduler = scheduler
        self.num_epochs = num_epochs
        self.early_stopping = early_stopping
        self.train_valid_split = train_valid_split
        self.batch_size = batch_size
        self.checkpoint_path = checkpoint_path
        self.time_series_list = time_series_list
        self.plot_ts = []
        self.plot = plot
        self.offset = offet_for_plot

        create_dir_if_not_exists(self.checkpoint_path)

        self._make_loader()
def get_gs_ds_mat(gs1, gs2, dist_sim_calculator, tvt1, tvt2,
                  dataset, dist_metric, dist_algo, norm, dec_gsize, return_neg1=False):
    mat_str = '{}({})_{}({})'.format(tvt1, len(gs1), tvt2, len(gs2))
    dir = '{}/ds_mat'.format(get_save_path())
    create_dir_if_not_exists(dir)
    sfn = '{}/{}_{}_ds_mat_{}{}_{}'.format(
        dir, dataset, mat_str, dist_metric,
        get_norm_str(norm), dist_algo)
    l = load(sfn)
    if l is not None:
        print('Loaded from {}'.format(sfn))
        return l
    m = len(gs1)
    n = len(gs2)
    dist_mat = np.zeros((m, n))
    for i in range(m):
        for j in range(n):
            g1 = gs1[i]
            g2 = gs2[j]
            d, normed_d = dist_sim_calculator.calculate_dist_sim(
                g1, g2, dec_gsize=dec_gsize, return_neg1=return_neg1)
            if norm:
                dist_mat[i][j] = normed_d
            else:
                dist_mat[i][j] = d
    save(sfn, dist_mat)
    print('Saved to {}'.format(sfn))
    return dist_mat
Beispiel #9
0
def main(opts):
    if not opts.do_train and not opts.do_eval:
        raise ValueError('At least one of `do_train` or `do_eval` must be True.')
    if os.path.exists(opts.output_dir) and os.listdir(opts.output_dir) and opts.do_train:
        raise ValueError('Output directory ({}) already exists and is not empty.'.format(opts.output_dir))

    # Create the output directory (if not exists)
    create_dir_if_not_exists(opts.output_dir)

    # Device device type
    opts.device = torch.device('cuda' if torch.cuda.is_available() and not opts.no_cuda else 'cpu')
    opts.n_gpus = torch.cuda.device_count() if str(opts.device) == 'cuda' else 0
    print('Device Type: {} | Number of GPUs: {}'.format(opts.device, opts.n_gpus), flush=True)

    # Load Datasets and Ontology
    dataset = MyDataset()

    if opts.do_train:
        # Load model from scratch
        model = Model.from_scratch('bert-base-chinese')
        model.move_to_device(opts)
        print('Number of model parameters is: {}'.format(get_n_params(model)))

        # Start Training
        print('Start Training', flush=True)
        model.run_train(dataset, opts)

        # Free up all memory pytorch is taken from gpu memory
        del model
        torch.cuda.empty_cache()
Beispiel #10
0
def setup_temp_data_folder(gp, append_str):
    dir = gp + '/data'
    create_dir_if_not_exists(dir)
    tp = dir + '/temp_{}'.format(append_str)
    exec_cmd('rm -rf {} && mkdir {}'.format(tp, tp))
    src = get_root_path() + '/src/gmt_files'
    exec_cmd('cp {}/temp.xml {}/temp_{}.xml'.format(src, tp, append_str))
    return src, tp
Beispiel #11
0
def exp5():
    """ Query visualization. """
    dataset = 'imdbmulti'
    model = 'astar'
    concise = True
    norms = [True, False]
    dir = get_result_path() + '/{}/query_vis/{}'.format(dataset, model)
    create_dir_if_not_exists(dir)
    info_dict = {
        # draw node config
        'draw_node_size': 150 if dataset != 'linux' else 10,
        'draw_node_label_enable': True,
        'node_label_name': None if dataset == 'linux' else 'type',
        'draw_node_label_font_size': 6,
        'draw_node_color_map': TYPE_COLOR_MAP,
        # draw edge config
        'draw_edge_label_enable': False,
        'edge_label_name': 'valence',
        'draw_edge_label_font_size': 6,
        # graph text info config
        'each_graph_text_list': [],
        'each_graph_text_font_size': 8,
        'each_graph_text_pos': [0.5, 1.05],
        # graph padding: value range: [0, 1]
        'top_space': 0.20 if concise else 0.26,  # out of whole graph
        'bottom_space': 0.05,
        'hbetween_space': 0.6 if concise else 1,  # out of the subgraph
        'wbetween_space': 0,
        # plot config
        'plot_dpi': 200,
        'plot_save_path_eps': '',
        'plot_save_path_png': ''
    }
    train_data = load_data(dataset, train=True)
    test_data = load_data(dataset, train=False)
    row_graphs = test_data.graphs
    col_graphs = train_data.graphs
    r = load_result(dataset, model, row_graphs=row_graphs, col_graphs=col_graphs)
    tr = load_result(dataset, TRUE_MODEL, row_graphs=row_graphs, col_graphs=col_graphs)
    for norm in norms:
        ids = r.get_sort_id_mat(norm)
        m, n = r.m_n()
        num_vis = 10
        for i in range(num_vis):
            q = test_data.graphs[i]
            gids = np.concatenate([ids[i][:3], [ids[i][int(n / 2)]], ids[i][-3:]])
            gs = [train_data.graphs[j] for j in gids]
            info_dict['each_graph_text_list'] = \
                [get_text_label(dataset, r, tr, i, i, q, model, norm, True, concise)] + \
                [get_text_label(dataset, r, tr, i, j,
                                train_data.graphs[j], model, norm, False, concise) \
                 for j in gids]
            # print(info_dict['each_graph_text_list'])
            info_dict['plot_save_path_png'] = '{}/query_vis_{}_{}_{}{}.{}'.format(
                dir, dataset, model, i, get_norm_str(norm), 'png')
            info_dict['plot_save_path_eps'] = '{}/query_vis_{}_{}_{}{}.{}'.format(
                dir, dataset, model, i, get_norm_str(norm), 'eps')
            vis(q, gs, info_dict)
Beispiel #12
0
 def __init__(self):
     model_str = self.get_model_str()
     self.logdir = join(get_root_path(), 'logs',
                        '{}_{}'.format(model_str, get_ts()))
     create_dir_if_not_exists(self.logdir)
     self.model_info_f = self._open('model_info.txt')
     self._log_model_info()
     self._save_conf_code()
     print('Logging to {}'.format(self.logdir))
Beispiel #13
0
def gen_dataset(graphs):
    random.Random(123).shuffle(graphs)
    dirout_train = get_data_path() + '/IMDBMulti/train'
    dirout_test = get_data_path() + '/IMDBMulti/test'
    create_dir_if_not_exists(dirout_train)
    create_dir_if_not_exists(dirout_test)
    for g in graphs[0:1200]:
        nx.write_gexf(g, dirout_train + '/{}.gexf'.format(g.graph['gid']))
    for g in graphs[1200:]:
        nx.write_gexf(g, dirout_test + '/{}.gexf'.format(g.graph['gid']))
Beispiel #14
0
 def __init__(self, sess):
     if FLAGS.log:
         model_str = self._get_model_str()
         logdir = '{}/logs/{}_{}'.format(get_siamese_dir(), model_str,
                                         get_ts())
         create_dir_if_not_exists(logdir)
         self.tw = tf.summary.FileWriter(logdir + '/train', sess.graph)
         self.vw = tf.summary.FileWriter(logdir + '/val', sess.graph)
         self.merged = tf.summary.merge_all()
         self._log_model_info(logdir, sess)
         print('Logging to {}'.format(logdir))
def gen_dataset(graphs):
    random.Random(123).shuffle(graphs)
    dirout_train = get_data_path() + '/{}/train'.format(conf.outfolder)
    dirout_test = get_data_path() + '/{}/test'.format(conf.outfolder)
    create_dir_if_not_exists(dirout_train)
    create_dir_if_not_exists(dirout_test)
    sp = int(len(graphs) * conf.train_perc_)
    for g in graphs[0:sp]:
        nx.write_gexf(g, dirout_train + '/{}.gexf'.format(g.graph['gid']))
    for g in graphs[sp:]:
        nx.write_gexf(g, dirout_test + '/{}.gexf'.format(g.graph['gid']))
def run(config, dir, outputhal):
    utils.check_existence_or_raise(config)
    utils.create_dir_if_not_exists(dir)
    command = ['runProgressiveCactus.sh', '--batchSystem', 'parasol', '--bigBatchSystem', 'singleMachine',
    '--defaultMemory', '8589934593', '--bigMemoryThreshold', '8589934592', '--bigMaxMemory', '893353197568',
    '--bigMaxCpus', '25', '--maxThreads', '25', '--parasolCommand=/cluster/home/jcarmstr/bin/parasol -host=ku',
    '--retryCount', '3']
    params = command + [config, dir, outputhal]
    print params
    subprocess.check_call(params)
    return outputhal
def get_gs_ds_mat(gs1,
                  gs2,
                  dist_sim_calculator,
                  tvt1,
                  tvt2,
                  dataset,
                  dist_metric,
                  dist_algo,
                  norm,
                  dec_gsize,
                  return_neg1=False):

    if logging_enabled == True:
        print("- Entered dist_sim_calculator::get_gs_ds_mat Global Method")

    mat_str = '{}({})_{}({})'.format(tvt1, len(gs1), tvt2, len(gs2))
    dir = '{}\\ds_mat'.format(get_save_path())
    create_dir_if_not_exists(dir)
    sfn = '{}\\{}_{}_ds_mat_{}{}_{}'.format(dir, dataset, mat_str, dist_metric,
                                            get_norm_str(norm), dist_algo)

    l = load(sfn)

    if l is not None:
        print('Loaded from {}'.format(sfn))
        return l

    if not dist_sim_calculator.gidpair_ds_map:
        # dist_sim_calculator.initial_calculate_dist_sim(gs1, gs2)
        dist_sim_calculator.initial_dist_sim_pairs_with_netcomp(gs1, gs2)

    m = len(gs1)
    n = len(gs2)

    dist_mat = np.zeros((m, n))
    for i in range(m):
        for j in range(n):
            g1 = gs1[i]
            g2 = gs2[j]
            d, normed_d = dist_sim_calculator.calculate_dist_sim(
                g1, g2, dec_gsize=dec_gsize, return_neg1=return_neg1)
            if norm:
                dist_mat[i][j] = normed_d
                print("i: ", i, ", j: ", j, ", d: ", d, ", normed_d: ",
                      normed_d)

            else:
                dist_mat[i][j] = d

    save(sfn, dist_mat)
    print('Saved to {}'.format(sfn))

    return dist_mat
Beispiel #18
0
def run_filtering(sorted_psl, results) :	
    utils.check_existence_or_raise(sorted_psl)
    stats = os.path.join(results,'filtering_stats')
    utils.create_dir_if_not_exists(stats)
    filtered_psl = os.path.join(results,utils.get_name(sorted_psl)+'.filtered.psl')    
    tmp = utils.atomic_tmp_file(filtered_psl)
    params = ['pslCDnaFilter','-localNearBest=0.1 -ignoreNs -minCover=0.6 -repsAsMatch',
			' -dropped='+os.path.join(stats,'dropped.out'), '-weirdOverlapped='+os.path.join(stats,'weird_overlapped.out'),
				'-alignStats='+os.path.join(stats,'align_stats.out'), '-statsOut='+os.path.join(stats,'overall_stats.out'),
					sorted_psl, tmp]
    subprocess.call(" ".join(params), shell=True)
    utils.atomic_install(tmp, filtered_psl)
    return filtered_psl
Beispiel #19
0
 def __init__(self, sess=None):
     model_str = self._get_model_str()
     self.logdir = '{}/logs/{}_{}'.format(get_siamese_dir(), model_str,
                                          get_ts())
     create_dir_if_not_exists(self.logdir)
     if sess is not None:
         self.tw = tf.summary.FileWriter(self.logdir + '/train', sess.graph)
         self.all_merged = tf.summary.merge_all()
         self.loss_merged = tf.summary.merge(
             self._extract_loss_related_summaries_as_list())
     self._log_model_info(self.logdir, sess)
     self.f = open('{}/results_{}.txt'.format(self.logdir, get_ts()), 'w')
     print('Logging to {}'.format(self.logdir))
def main():
    person_group_id = FACE_GROUP_ID
    display_image = False
    
    face_api_wrapper = FaceAPIWrapper(FACE_API_KEY, FACE_BASE_URL)
    create_dir_if_not_exists('captured_images/' + CAMERA_NAME)
   
    print("Capturing Image every ", CAPTURE_INTERVAL, " seconds...")

    i = 0

    while 1:
        try:
            image_filename = 'captured_images/' + CAMERA_NAME + "/" + current_time_to_string() + ".jpg"
            image = Camera().capture_image()
            cv2.imwrite(image_filename, image)

            if display_image:
                cv2.imshow("Camera Image", image)
                cv2.waitKey(1)

            image_link = upload_to_s3(image_filename)
            image_url = 'https://mybuckfucket.s3-eu-west-1.amazonaws.com/public_folder/'+image_filename
            face_ids = face_api_wrapper.detect_faces(image=image_filename)
            i += 1
            print(i, "Captured at ", current_time_to_string())
            print("S3 url",image_url)
            if face_ids:
                person_ids = \
                    face_api_wrapper.identify_faces(face_ids=face_ids,
                                                    large_person_group=person_group_id)
                req_ids = [{id} for id in person_ids]
                print("Detected Faces...", req_ids)
                contents=load_dict_from_file('json.txt')
                
                #my_data=(contents[0]['faceAttributes'])
                requests.post('http://127.0.0.1:8000/students/', data={
                    'attributes' : contents,
                    'name' : "Darragh",
                    'time_date' : current_time_to_string(),
                    'face_ids' : person_ids,
                    'image_link' : image_url,
                })
                print("#####",data)
                #print(contents[0]['faceAttributes']['emotion'])
                #requests.post('http://127.0.0.1:8000/students/', data=contents)
                

            time.sleep(CAPTURE_INTERVAL)
        except Exception as e:
            print(e)
Beispiel #21
0
def run(dir, maf, sizes_folder, species_file):
    grimm_synt_input_file = os.path.join(dir,
                                         utils.get_name(maf) + '.grimm_synt')

    print utils.get_time()
    print 'converting maf to input for grimm...'
    params = [
        './generate_input_grimm_synt', maf, sizes_folder,
        grimm_synt_input_file, species_file
    ]
    subprocess.check_call(params)

    print utils.get_time()
    print 'generating anchors for grimm_synt...'
    anchors_folder = os.path.join(dir, 'grimm_synt_anchors')
    utils.create_dir_if_not_exists(anchors_folder)
    params = [
        'grimm_synt', '-A', '-f', grimm_synt_input_file, '-d', anchors_folder
    ]
    subprocess.check_call(params)
    print utils.get_time()
    print 'running grimm_synt...'
    grimm_synt_output = os.path.join(dir, 'grimm_synt_output')
    utils.create_dir_if_not_exists(grimm_synt_output)
    #used this for comparing mhc regions
    #params = ['grimm_synt','-f', os.path.join(anchors_folder, 'unique_coords.txt'),'-d',grimm_synt_output, '-m 1000 -g 1000 -c']
    #params = ['grimm_synt','-f', os.path.join(anchors_folder, 'unique_coords.txt'),'-d',grimm_synt_output,'-m', '300000', '-g', '300000', '-c']
    params = [
        'grimm_synt', '-f',
        os.path.join(anchors_folder, 'unique_coords.txt'), '-d',
        grimm_synt_output, '-m', '100000', '-g', '100000', '-c'
    ]
    subprocess.check_call(params)
    print 'synteny blocks are at', os.path.join(grimm_synt_output,
                                                'blocks.txt')
    print utils.get_time()
    print 'creating bigBed files...'
    create_bigBed(grimm_synt_output, sizes_folder, species_file)
    print utils.get_time()
    print 'running grimm...'
    params = [
        'grimm', '-f',
        os.path.join(grimm_synt_output, 'mgr_macro.txt'), '-o',
        os.path.join(dir, 'grimm.output')
    ]
    subprocess.call(" ".join(params), shell=True)
    print 'grimm output is saved to', os.path.join(dir, 'grimm.output')
    print utils.get_time()

    print 'done.'
Beispiel #22
0
def plot_preck(dataset, dsmetric, models, rs, true_result, metric, norms,
               plot_results=True, extra_dir=None):
    """ Plot prec@k. """
    create_dir_if_not_exists('{}/{}/{}'.format(
        get_result_path(), dataset, metric))
    rtn = {}
    for norm in norms:
        _, n = true_result.m_n()
        ks = range(1, n)
        d = plot_preck_helper(
            dataset, dsmetric, models, rs, true_result, metric, norm, ks,
            False, plot_results, extra_dir)
        rtn.update(d)
    return rtn
Beispiel #23
0
def run(config, dir, outputhal):
    utils.check_existence_or_raise(config)
    utils.create_dir_if_not_exists(dir)
    command = [
        'runProgressiveCactus.sh', '--batchSystem', 'parasol',
        '--bigBatchSystem', 'singleMachine', '--defaultMemory', '8589934593',
        '--bigMemoryThreshold', '8589934592', '--bigMaxMemory', '893353197568',
        '--bigMaxCpus', '25', '--maxThreads', '25',
        '--parasolCommand=/cluster/home/jcarmstr/bin/parasol -host=ku',
        '--retryCount', '3'
    ]
    params = command + [config, dir, outputhal]
    print params
    subprocess.check_call(params)
    return outputhal
Beispiel #24
0
    def __init__(self, sess=None):
        if logging_enabled == True:
            print("- Entered saver::Saver::__init__ Constructor Method")

        model_str = self._get_model_str()
        self.logdir = '{}\\logs\\{}'.format(get_siamese_dir(), model_str)
        create_dir_if_not_exists(self.logdir)
        
        if sess is not None:
            self.tw = tf.compat.v1.summary.FileWriter(self.logdir + '\\train', sess.graph)
            self.all_merged = tf.compat.v1.summary.merge_all()
            self.loss_merged = tf.compat.v1.summary.merge(self._extract_loss_related_summaries_as_list())
        
        self.f = open('{}\\results.txt'.format(self.logdir), 'w')
        print('Logging to {}'.format(self.logdir))
Beispiel #25
0
def save_fig(plt, dir, fn, print_path=False):
    plt_cnt = 0
    if dir is None or fn is None:
        return plt_cnt
    final_path_without_ext = '{}/{}'.format(dir, fn)
    for ext in ['png', 'eps']:
        full_path = final_path_without_ext + '.' + ext
        create_dir_if_not_exists(dirname(full_path))
        try:
            plt.savefig(full_path, bbox_inches='tight')
        except:
            warn('savefig')
        if print_path:
            print('Saved to {}'.format(full_path))
        plt_cnt += 1
    return plt_cnt
Beispiel #26
0
def select_dump_graphs(graphs, less_than_eq_10_ids):
    remove_type(graphs)
    random.Random(123).shuffle(less_than_eq_10_ids)
    tr = 800
    te = 200
    assert (tr + te <= len(graphs))
    dirout = get_data_path() + '/linux'
    train_dir = dirout + '/train'
    create_dir_if_not_exists(train_dir)
    for i in range(tr):
        gid = less_than_eq_10_ids[i]
        nx.write_gexf(graphs[gid], train_dir + '/{}.gexf'.format(gid))
    test_dir = dirout + '/test'
    create_dir_if_not_exists(test_dir)
    for i in range(tr, tr + te):
        gid = less_than_eq_10_ids[i]
        nx.write_gexf(graphs[gid], test_dir + '/{}.gexf'.format(gid))
Beispiel #27
0
def save_figs(info_dict):
    save_path = info_dict['plot_save_path_png']
    print(save_path)
    if not save_path:
        # print('plt.show')
        plt.show()
    else:
        for full_path in [
                info_dict['plot_save_path_png'],
                info_dict['plot_save_path_eps']
        ]:
            if not full_path:
                continue
            # print('Saving query vis plot to {}'.format(sp))
            create_dir_if_not_exists(dirname(full_path))
            plt.savefig(full_path, dpi=info_dict['plot_dpi'])
    plt.close()
Beispiel #28
0
def main():
    person_group_id = FACE_GROUP_ID
    display_image = False

    face_api_wrapper = FaceAPIWrapper(FACE_API_KEY, FACE_BASE_URL)
    create_dir_if_not_exists('captured_images/' + CAMERA_NAME)

    print("Capturing Image every ", CAPTURE_INTERVAL, " seconds...")

    i = 0

    while 1:
        try:
            image_filename = 'captured_images/' + CAMERA_NAME + "/" + current_time_to_string(
            ) + ".jpg"
            image = Camera().capture_image()
            cv2.imwrite(image_filename, image)

            if display_image:
                cv2.imshow("Camera Image", image)
                cv2.waitKey(1)

            image_link = upload_to_s3(image_filename)
            face_ids = face_api_wrapper.detect_faces(image=image_filename)
            i += 1
            print(i, "Captured at ", time.time())
            if face_ids:
                person_ids = \
                    face_api_wrapper.identify_faces(face_ids=face_ids,
                                                    large_person_group=person_group_id)
                req_ids = [{id} for id in person_ids]
                print("Detected Faces...", req_ids)

                requests.post(REST_SERVER_URL + 'time-face-id',
                              data={
                                  'lecture_number': get_lecture_number(),
                                  'face_ids': req_ids,
                                  'image-link': image_link,
                                  'camera-name': CAMERA_NAME,
                              })
                print("Present IDs:", req_ids)

            time.sleep(CAPTURE_INTERVAL)
        except Exception as e:
            print(e)
def create_bigBed(grimm_synt_output, sizes_folder, species_file):
    blocks = os.path.join(grimm_synt_output,'blocks.txt')
    bed = os.path.join(grimm_synt_output,'bed')
    utils.create_dir_if_not_exists(bed)
    params = ['./blocks_to_bed.py', blocks, bed]
    subprocess.check_call(params)
    with open(species_file) as species:
        for line in species :
            line = line.strip().split(' ')
            if len(line) < 2:
                print 'species file format: genome_id genome_name'
                print 'no bigBeds generated'
                raise Exception('wrong species format')
            params = ['sort -k1,1 -k2,2n', os.path.join(bed,line[0]+'.bed'), '>', os.path.join(bed,line[0]+'.sorted.bed')]
            subprocess.check_call(" ".join(params), shell=True)
            params = ['bedToBigBed','-type=bed6', os.path.join(bed,line[0]+'.sorted.bed'), os.path.join(sizes_folder,line[1]+'.sizes'), os.path.join(bed,line[1]+'.bb')]
            subprocess.check_call(params)
    print 'path to bigBed folder is',bed 
Beispiel #30
0
def create_necessary_dirs(FLAGS, batches, class_per_frame):
    create_dir_if_not_exists(FLAGS.output_labels_dir)
    if FLAGS.copy_dir:
        create_dir_if_not_exists(FLAGS.copy_dir)
        classes_per_video = [class_per_frame[video].keys() for video in batches]
        for i in range(len(classes_per_video)):
            for klass in classes_per_video[i]:
                class_path = os.path.join(FLAGS.copy_dir, klass)
                video_path = os.path.join(class_path, batches[i])
                create_dir_if_not_exists(class_path)
                create_dir_if_not_exists(video_path)
Beispiel #31
0
def _save_fig(plt, dir, fn, need_eps=False, print_path=False):
    plt_cnt = 0
    if dir is None or fn is None:
        return plt_cnt
    final_path_without_ext = join(dir, fn)
    exts = ['.png']
    if need_eps:
        exts += '.eps'
    for ext in exts:
        final_path = append_ext_to_filepath(ext, final_path_without_ext)
        create_dir_if_not_exists(dirname(final_path))
        try:
            plt.savefig(final_path, bbox_inches='tight')
        except:
            warn('savefig')
        if print_path:
            print('Saved to {}'.format(final_path))
        plt_cnt += 1
    return plt_cnt
Beispiel #32
0
 def _load_create_fake_pairs_if_needed(self, data):
     if FLAGS.fake_generation:
         assert ('fake_' in FLAGS.fake_generation)
         fake_num = int(FLAGS.fake_generation.split('_')[1])
         dir = get_save_path() + '/siamese_regession_fake_pairs'
         if FLAGS.ds_metric == 'mcs':
             dir += '_mcs'
         create_dir_if_not_exists(dir)
         filepath = dir + '/{}_fake_{}'.format(FLAGS.dataset_train,
                                               fake_num)
         ld = load(filepath)
         if ld:
             print('Loaded from {} with {} fake triples'.format(
                 filepath, len(ld)))
             return ld
         rtn = self._create_fake_pairs(data, fake_num)
         save(filepath, rtn)
         return rtn
     else:
         return []
def add_kml_file(request):
    if request.method == 'POST':
        form = KMLFilesForm(request.POST, request.FILES)
        if form.is_valid():
            kml_name = form.cleaned_data['name']
            uploaded_file = request.FILES['filename']
            file_name = uploaded_file.name
            file_name = file_name.replace(" ", "_")
            app_dir = _get_app_dir(__file__)
            destination_name = app_dir + '/static/kml/' + file_name
            create_dir_if_not_exists(destination_name)
            destination = open(destination_name, 'wb+')
            for chunk in uploaded_file.chunks(): 
                destination.write(chunk) 
                destination.close()
            kml_file = KMLFile(name=kml_name, filename=('static/kml/' + file_name))
            kml_file.save()
            return HttpResponseRedirect('/')
        else:
            return _render_response(form, request)
    else:
        form = KMLFilesForm()
        return _render_response(form, request)
Beispiel #34
0
def run(path_to_fasta, path_to_results):
    print utils.get_time()
    print 'check that folder for results exists...'
    utils.create_dir_if_not_exists(path_to_results)

    print utils.get_time()
    print 'running faToBit to convert fasta file to 2bit format...'
    twobit = run_faToBit(path_to_fasta, path_to_results)

    print utils.get_time()
    print 'calculating gc content...'
    wig = run_gcPercent(path_to_fasta, twobit, path_to_results)

    print utils.get_time()
    print 'cacluclating sizes of fasta sequences...'
    sizes = utils.run_faSize(path_to_fasta, path_to_results)

    print utils.get_time()
    print 'converting wig to bigWig...'
    bigWig = run_wigToBigWig(path_to_fasta, wig, sizes, path_to_results)

    print utils.get_time()
    print 'done.'
def run(path_to_fasta, path_to_results) :
	print utils.get_time()
	print 'check that folder for results exists...'
	utils.create_dir_if_not_exists(path_to_results)

	print utils.get_time()
	print 'running faToBit to convert fasta file to 2bit format...'
	twobit = run_faToBit(path_to_fasta, path_to_results)

	print utils.get_time()
	print 'calculating gc content...'
	wig = run_gcPercent(path_to_fasta, twobit, path_to_results)	

	print utils.get_time()
	print 'cacluclating sizes of fasta sequences...'
	sizes = utils.run_faSize(path_to_fasta, path_to_results)

	print utils.get_time()
	print 'converting wig to bigWig...'
	bigWig = run_wigToBigWig(path_to_fasta,wig, sizes, path_to_results)

	print utils.get_time()
	print 'done.'
Beispiel #36
0
def plot_single_number_metric(dataset, dsmetric, models, rs, true_result, metric, norms,
                              ds_kernel=None,
                              thresh_poss=None, thresh_negs=None,
                              thresh_poss_sim=None, thresh_negs_sim=None,
                              plot_results=True,
                              extra_dir=None):
    """ Plot mrr or mse. """
    create_dir_if_not_exists('{}/{}/{}'.format(
        get_result_path(), dataset, metric))
    rtn = {}
    if norms and thresh_poss and thresh_negs:
        assert (len(norms) == len(thresh_poss) == len(thresh_negs))
    for i, norm in enumerate(norms):
        thresh_pos = thresh_poss[i] if thresh_poss else None
        thresh_neg = thresh_negs[i] if thresh_negs else None
        thresh_pos_sim = thresh_poss_sim[i] if thresh_poss_sim else None
        thresh_neg_sim = thresh_negs_sim[i] if thresh_negs_sim else None
        d = plot_single_number_metric_helper(
            dataset, dsmetric, models, rs, true_result, metric, norm, ds_kernel,
            thresh_pos, thresh_neg, thresh_pos_sim, thresh_neg_sim,
            plot_results, extra_dir)
        rtn.update(d)
    return rtn