def find_binary():
""" Returns the path to the generated binary and pdb file """
source_file = None
pdb_file = None
possible_files = []
if is_windows():
# Check the different Configurations
configurations = ["RelWithDebInfo", "Release"]
target_file = MODULE_NAME + ".pyd"
for config in configurations:
possible_files.append(
join(get_output_dir(), config, MODULE_NAME + ".dll"))
elif is_linux():
target_file = MODULE_NAME + ".so"
possible_files.append(join(get_output_dir(), target_file))
for file in possible_files:
if isfile(file):
source_file = file
pdb_name = file.replace(".so", ".pdb").replace(".dll", ".pdb")
if isfile(pdb_name):
pdb_file = pdb_name
return source_file, pdb_file, target_file
def find_binary():
""" Returns the path to the generated binary and pdb file """
source_file = None
pdb_file = None
possible_files = []
if is_windows():
# Check the different Configurations
configurations = ["RelWithDebInfo", "Release"]
target_file = MODULE_NAME + ".pyd"
for config in configurations:
possible_files.append(join(get_output_dir(), config, MODULE_NAME + ".dll"))
elif is_linux():
target_file = MODULE_NAME + ".so"
possible_files.append(join(get_output_dir(), target_file))
for file in possible_files:
if isfile(file):
source_file = file
pdb_name = file.replace(".so", ".pdb").replace(".dll", ".pdb")
if isfile(pdb_name):
pdb_file = pdb_name
return source_file, pdb_file, target_file
def get_evidences(self):
GoogleBucketResource.has_valid_auth_key(self.args.google_credential_key)
output_dir_evidence = get_output_dir(None, PIS_OUTPUT_EVIDENCES)
list_files_evidence = {}
for entry in self.yaml.evidences.downloads:
file_from_bucket = self.get_file_from_bucket(entry, output_dir_evidence, self.yaml.evidences.gs_output_dir)
list_files_evidence.update(file_from_bucket)
self.get_stats_files(list_files_evidence)
output_dir_subset = get_output_dir(None, PIS_OUTPUT_SUBSET_EVIDENCES)
subsetEvidence = EvidenceSubset(ROOT_DIR+'/minimal_ensembl.txt',output_dir_subset,self.yaml.evidences.gs_output_dir)
list_files_subsets = subsetEvidence.execute_subset(self.list_files_downloaded)
subsetEvidence.create_stats_file()
self.list_files_downloaded.update(list_files_subsets)
def main():
kwargs = common.get_default_kwargs(args)
job_gen_type = "direct"
template_dir = common.get_template_dir(job_gen_type)
kwargs["command"] = common.get_command(args.unique_id)
output_dir = common.get_output_dir("multi-level", args)
assert os.path.isdir(template_dir), template_dir
assert os.path.isdir(output_dir), output_dir
if args.small_scale:
num_nodes = 2
topologies = [[1], [4], [2, 4]]
elif args.medium_scale:
num_nodes = 8
topologies = [[1], [4], [2, 4]]
else:
num_nodes = 32
topologies = [[1], [1, num_nodes], [1, num_nodes, num_cores_per_node]]
repetitions = range(1) if args.small_scale or args.medium_scale else range(3)
if args.repetitions:
repetitions = range(args.repetitions)
for repetition in repetitions:
variable_num_jobs(
template_dir,
output_dir,
job_gen_type,
topologies,
num_nodes,
repetition,
**kwargs,
)
def main():
kwargs = common.get_default_kwargs(args)
job_gen_type = "direct"
template_dir = common.get_template_dir(job_gen_type)
kwargs["command"] = common.get_command(args.unique_id)
output_dir = common.get_output_dir("model", args)
assert os.path.isdir(template_dir), template_dir
assert os.path.isdir(output_dir), output_dir
num_nodes = 32
topologies = [[1], [1, num_nodes], [1, num_nodes, num_cores_per_node]]
repetitions = range(1) if args.small_scale or args.medium_scale else range(
3)
if args.repetitions:
repetitions = range(args.repetitions)
for repetition in repetitions:
build_model(template_dir, output_dir, job_gen_type, repetition,
**kwargs)
just_hierarchy_setup(
template_dir,
output_dir,
job_gen_type,
topologies,
num_nodes,
repetition,
**kwargs,
)
def get_known_target_safety(self):
# config.yaml known target safety file : download spreadsheets + generate file for data_pipeline
safety_output_dir = get_output_dir(None, PIS_OUTPUT_KNOWN_TARGET_SAFETY)
known_target_safety_resource = KnownTargetSafetyResource(self.output_dir)
known_target_safety_resource.download_spreadsheet(self.yaml.known_target_safety,safety_output_dir)
ksafety_filename = known_target_safety_resource.generate_known_safety_json(self.yaml.known_target_safety)
self.list_files_downloaded[ksafety_filename] = {'resource': self.yaml.known_target_safety.resource,
'gs_output_dir': self.yaml.known_target_safety.gs_output_dir}
def get_chemical_probes(self):
# config.yaml chemical probes file : downnload spreadsheets + generate file for data_pipeline
chemical_output_dir = get_output_dir(None, PIS_OUTPUT_CHEMICAL_PROBES)
chemical_probes_resource = ChemicalProbesResource(self.output_dir)
chemical_probes_resource.download_spreadsheet(self.yaml.chemical_probes,chemical_output_dir)
chemical_filename = chemical_probes_resource.generate_probes(self.yaml.chemical_probes)
self.list_files_downloaded[chemical_filename] = {'resource': self.yaml.chemical_probes.resource,
'gs_output_dir': self.yaml.chemical_probes.gs_output_dir}
def __init__(self,args, yaml, yaml_data_pipeline_schema):
self.args = args
self.output_dir = get_output_dir(args.output_dir, PIS_OUTPUT_ANNOTATIONS)
self.yaml = yaml
self.yaml_data_pipeline_schema = yaml_data_pipeline_schema
self.list_files_downloaded = {}
self.list_google_storage_files = {}
self.list_files_downladed_failed = {}
def file_size(url):
try:
size = float(os.path.getsize(os.path.join(get_output_dir(), 'data', url)))
except OSError:
size = 0
for x in ['bytes','KB','MB','GB','TB']:
if size < 1024.0:
return "%3.1f %s" % (size, x)
size /= 1024.0
def get_ChEMBL(self):
list_files_ChEMBL ={}
output_dir_ChEMBL = get_output_dir(None, PIS_OUTPUT_CHEMBL_API)
chembl_handler = ChEMBLLookup(self.yaml.ChEMBL)
list_files_ChEMBL_unzipped = chembl_handler.download_chEMBL_files()
for file_with_path in list_files_ChEMBL_unzipped:
filename_zip = make_gzip(file_with_path)
list_files_ChEMBL[filename_zip] = {'resource': list_files_ChEMBL_unzipped[file_with_path]['resource'],
'gs_output_dir': self.yaml.ChEMBL.gs_output_dir }
self.list_files_downloaded.update(list_files_ChEMBL)
def freeze_request(req_path):
print "Freezing %s..." % req_path
path = os.path.join(get_output_dir(), req_path.lstrip('/'))
dirname = os.path.dirname(path)
if not os.path.exists(dirname):
os.makedirs(dirname)
fh = open(path, 'w')
res = client.get(req_path)
fh.write(res.data)
fh.close()
def run(self):
output_dir_annotations = get_output_dir(self.args.output_dir, PIS_OUTPUT_ANNOTATIONS)
google_opts = GoogleBucketResource.has_google_parameters(self.args.google_credential_key, self.args.google_bucket)
if google_opts:
GoogleBucketResource.has_valid_auth_key(self.args.google_credential_key)
if self.has_step("annotations") : self.annotations_downloaded_by_uri()
if self.has_step("ensembl"): self.get_ensembl()
if self.has_step("chemical_probes"): self.get_chemical_probes()
if self.has_step("known_target_safety"): self.get_known_target_safety()
if self.has_step("ChEMBL"): self.get_ChEMBL()
if self.has_step("annotations_from_buckets"): self.get_annotations_from_bucket()
if self.has_step("evidences"): self.get_evidences()
# At this point the auth key is already valid.
if google_opts: self.copy_files_to_google_storage()
self.create_yaml_config_file()
logging.info("Done.")
return True
def store_csv(q, filename):
q = engine.query(q)
prefix = os.path.join(get_output_dir(), 'data')
freeze(q, filename=filename, prefix=prefix)
def store_csv(q, filename):
q = engine.query(q)
prefix = os.path.join(get_output_dir(), 'data')
freeze(q, filename=filename, prefix=prefix)