def find_binary():
    """ Returns the path to the generated binary and pdb file """

    source_file = None
    pdb_file = None
    possible_files = []

    if is_windows():

        # Check the different Configurations
        configurations = ["RelWithDebInfo", "Release"]
        target_file = MODULE_NAME + ".pyd"

        for config in configurations:
            possible_files.append(
                join(get_output_dir(), config, MODULE_NAME + ".dll"))

    elif is_linux():
        target_file = MODULE_NAME + ".so"
        possible_files.append(join(get_output_dir(), target_file))

    for file in possible_files:
        if isfile(file):
            source_file = file

            pdb_name = file.replace(".so", ".pdb").replace(".dll", ".pdb")
            if isfile(pdb_name):
                pdb_file = pdb_name

    return source_file, pdb_file, target_file
Ejemplo n.º 2
0
def find_binary():
    """ Returns the path to the generated binary and pdb file """

    source_file = None
    pdb_file = None
    possible_files = []

    if is_windows():

        # Check the different Configurations
        configurations = ["RelWithDebInfo", "Release"]
        target_file = MODULE_NAME + ".pyd"

        for config in configurations:
            possible_files.append(join(get_output_dir(), config, MODULE_NAME + ".dll"))

    elif is_linux():
        target_file = MODULE_NAME + ".so"
        possible_files.append(join(get_output_dir(), target_file))

    for file in possible_files:
        if isfile(file):
            source_file = file

            pdb_name = file.replace(".so", ".pdb").replace(".dll", ".pdb")
            if isfile(pdb_name):
                pdb_file = pdb_name

    return source_file, pdb_file, target_file
Ejemplo n.º 3
0
    def get_evidences(self):
        GoogleBucketResource.has_valid_auth_key(self.args.google_credential_key)
        output_dir_evidence = get_output_dir(None, PIS_OUTPUT_EVIDENCES)
        list_files_evidence = {}
        for entry in self.yaml.evidences.downloads:
            file_from_bucket = self.get_file_from_bucket(entry, output_dir_evidence, self.yaml.evidences.gs_output_dir)
            list_files_evidence.update(file_from_bucket)


        self.get_stats_files(list_files_evidence)
        output_dir_subset = get_output_dir(None, PIS_OUTPUT_SUBSET_EVIDENCES)
        subsetEvidence = EvidenceSubset(ROOT_DIR+'/minimal_ensembl.txt',output_dir_subset,self.yaml.evidences.gs_output_dir)
        list_files_subsets = subsetEvidence.execute_subset(self.list_files_downloaded)
        subsetEvidence.create_stats_file()
        self.list_files_downloaded.update(list_files_subsets)
Ejemplo n.º 4
0
def main():
    kwargs = common.get_default_kwargs(args)
    job_gen_type = "direct"
    template_dir = common.get_template_dir(job_gen_type)
    kwargs["command"] = common.get_command(args.unique_id)
    output_dir = common.get_output_dir("multi-level", args)
    assert os.path.isdir(template_dir), template_dir
    assert os.path.isdir(output_dir), output_dir

    if args.small_scale:
        num_nodes = 2
        topologies = [[1], [4], [2, 4]]
    elif args.medium_scale:
        num_nodes = 8
        topologies = [[1], [4], [2, 4]]
    else:
        num_nodes = 32
        topologies = [[1], [1, num_nodes], [1, num_nodes, num_cores_per_node]]

    repetitions = range(1) if args.small_scale or args.medium_scale else range(3)
    if args.repetitions:
        repetitions = range(args.repetitions)
    for repetition in repetitions:
        variable_num_jobs(
            template_dir,
            output_dir,
            job_gen_type,
            topologies,
            num_nodes,
            repetition,
            **kwargs,
        )
def main():
    kwargs = common.get_default_kwargs(args)
    job_gen_type = "direct"
    template_dir = common.get_template_dir(job_gen_type)
    kwargs["command"] = common.get_command(args.unique_id)
    output_dir = common.get_output_dir("model", args)
    assert os.path.isdir(template_dir), template_dir
    assert os.path.isdir(output_dir), output_dir

    num_nodes = 32
    topologies = [[1], [1, num_nodes], [1, num_nodes, num_cores_per_node]]

    repetitions = range(1) if args.small_scale or args.medium_scale else range(
        3)
    if args.repetitions:
        repetitions = range(args.repetitions)
    for repetition in repetitions:
        build_model(template_dir, output_dir, job_gen_type, repetition,
                    **kwargs)
        just_hierarchy_setup(
            template_dir,
            output_dir,
            job_gen_type,
            topologies,
            num_nodes,
            repetition,
            **kwargs,
        )
Ejemplo n.º 6
0
 def get_known_target_safety(self):
     # config.yaml known target safety file : download spreadsheets + generate file for data_pipeline
     safety_output_dir = get_output_dir(None, PIS_OUTPUT_KNOWN_TARGET_SAFETY)
     known_target_safety_resource = KnownTargetSafetyResource(self.output_dir)
     known_target_safety_resource.download_spreadsheet(self.yaml.known_target_safety,safety_output_dir)
     ksafety_filename = known_target_safety_resource.generate_known_safety_json(self.yaml.known_target_safety)
     self.list_files_downloaded[ksafety_filename] = {'resource': self.yaml.known_target_safety.resource,
                                                      'gs_output_dir': self.yaml.known_target_safety.gs_output_dir}
Ejemplo n.º 7
0
 def get_chemical_probes(self):
     # config.yaml chemical probes file : downnload spreadsheets + generate file for data_pipeline
     chemical_output_dir = get_output_dir(None, PIS_OUTPUT_CHEMICAL_PROBES)
     chemical_probes_resource = ChemicalProbesResource(self.output_dir)
     chemical_probes_resource.download_spreadsheet(self.yaml.chemical_probes,chemical_output_dir)
     chemical_filename = chemical_probes_resource.generate_probes(self.yaml.chemical_probes)
     self.list_files_downloaded[chemical_filename] = {'resource': self.yaml.chemical_probes.resource,
                                                      'gs_output_dir': self.yaml.chemical_probes.gs_output_dir}
Ejemplo n.º 8
0
 def __init__(self,args, yaml, yaml_data_pipeline_schema):
     self.args = args
     self.output_dir = get_output_dir(args.output_dir, PIS_OUTPUT_ANNOTATIONS)
     self.yaml = yaml
     self.yaml_data_pipeline_schema = yaml_data_pipeline_schema
     self.list_files_downloaded = {}
     self.list_google_storage_files = {}
     self.list_files_downladed_failed = {}
Ejemplo n.º 9
0
def file_size(url):
    try:
        size = float(os.path.getsize(os.path.join(get_output_dir(), 'data', url)))
    except OSError:
        size = 0
    for x in ['bytes','KB','MB','GB','TB']:
        if size < 1024.0:
            return "%3.1f %s" % (size, x)
        size /= 1024.0
Ejemplo n.º 10
0
 def get_ChEMBL(self):
     list_files_ChEMBL ={}
     output_dir_ChEMBL = get_output_dir(None, PIS_OUTPUT_CHEMBL_API)
     chembl_handler = ChEMBLLookup(self.yaml.ChEMBL)
     list_files_ChEMBL_unzipped = chembl_handler.download_chEMBL_files()
     for file_with_path in list_files_ChEMBL_unzipped:
         filename_zip = make_gzip(file_with_path)
         list_files_ChEMBL[filename_zip] = {'resource': list_files_ChEMBL_unzipped[file_with_path]['resource'],
                                            'gs_output_dir': self.yaml.ChEMBL.gs_output_dir }
     self.list_files_downloaded.update(list_files_ChEMBL)
Ejemplo n.º 11
0
def freeze_request(req_path):
    print "Freezing %s..." % req_path
    path = os.path.join(get_output_dir(), req_path.lstrip('/'))
    dirname = os.path.dirname(path)
    if not os.path.exists(dirname):
        os.makedirs(dirname)
    fh = open(path, 'w')
    res = client.get(req_path)
    fh.write(res.data)
    fh.close()
Ejemplo n.º 12
0
    def run(self):
        output_dir_annotations = get_output_dir(self.args.output_dir, PIS_OUTPUT_ANNOTATIONS)
        google_opts = GoogleBucketResource.has_google_parameters(self.args.google_credential_key, self.args.google_bucket)
        if google_opts:
            GoogleBucketResource.has_valid_auth_key(self.args.google_credential_key)

        if self.has_step("annotations") : self.annotations_downloaded_by_uri()
        if self.has_step("ensembl"): self.get_ensembl()
        if self.has_step("chemical_probes"): self.get_chemical_probes()
        if self.has_step("known_target_safety"): self.get_known_target_safety()
        if self.has_step("ChEMBL"): self.get_ChEMBL()
        if self.has_step("annotations_from_buckets"): self.get_annotations_from_bucket()
        if self.has_step("evidences"): self.get_evidences()

        # At this point the auth key is already valid.
        if google_opts: self.copy_files_to_google_storage()

        self.create_yaml_config_file()

        logging.info("Done.")
        return True
Ejemplo n.º 13
0
def store_csv(q, filename):
    q = engine.query(q)
    prefix = os.path.join(get_output_dir(), 'data')
    freeze(q, filename=filename, prefix=prefix)
Ejemplo n.º 14
0
def store_csv(q, filename):
    q = engine.query(q)
    prefix = os.path.join(get_output_dir(), 'data')
    freeze(q, filename=filename, prefix=prefix)