def execute(self):
# compile submission arguments
kwargs = self.build_args[1]
save_id = kwargs.pop("save_id")
hardy_type = kwargs.pop("hardy_type")
# make server directory
directory = "{root}/{proc}".format(root=root, proc=self.identity)
if not os.path.exists(directory):
os.makedirs(directory)
else:
shutil.rmtree(directory)
os.makedirs(directory)
calc_params = ["--hardy_type", hardy_type, "--out_dir", directory,
"--other_args", json.dumps(kwargs), "--year_id"]
q = self.get_qmaster() # start up monitor
# parallelize by year
year_list = [{YEAR IDS}]
for i in year_list:
remote_job = job.Job(mon_dir=log_dir,
runfile=("{root}/hardy_weinberg/calc.py"
.format(root=root)),
name="{proc}_{loc}".format(proc=self.identity,
loc=i),
job_args=calc_params + [str(i)])
q.queue_job(
remote_job,
slots=4,
memory=8,
project={PROJECT},
stderr=log_dir,
stdout=log_dir)
q.block_till_done(poll_interval=60) # monitor them
# now save the results
save_params = [
save_id, description, directory, "--best", "--file_pattern",
"{year_id}.h5", "--h5_tablename", "draws"]
remote_job = job.Job(mon_dir=log_dir,
name="save_" + str(save_id),
runfile=("{FILEPATH}/save_custom_results"),
job_args=save_params)
q.queue_job(remote_job,
slots=20,
memory=40,
project="{PROJECT}",
stderr=log_dir,
stdout=log_dir)
q.block_till_done(poll_interval=60)
def execute(self):
# compile submission arguments
kwargs = self.build_args[1]
me_map = kwargs.pop("me_map")
# get qmaster
q = self.get_qmaster()
# command line args for adjuster.py
# parallelize by year
for i in [1990, 1995, 2000, 2005, 2010, 2016]:
ex_params = ["--me_map", json.dumps(me_map),
"--out_dir", data_dir, "--year_id", str(i)]
remote_job = job.Job(
mon_dir=log_dir,
runfile=os.path.join(code_dir, "scripts", "run_ex_adjust.py"),
name="{proc}_{year}".format(proc=self.identity, year=i),
job_args=ex_params
)
q.queue_job(
remote_job,
slots=20,
memory=40,
project="proj_epic")
q.block_till_done(poll_interval=60)
outputs_tuples = self.builder.get_process_outputs(self.identity)
result_meids = [task_tuple[0] for task_tuple in outputs_tuples]
for meid in result_meids:
save_params = [
meid,
description,
os.path.join(data_dir, str(meid)),
"--best",
"--file_pattern", "{year_id}.h5",
"--h5_tablename", "draws"]
remote_job = job.Job(
mon_dir=log_dir,
name="save_" + str(meid),
runfile=sge.true_path(executable="save_custom_results"),
job_args=save_params)
q.queue_job(
remote_job,
slots=40,
memory=80,
project="proj_epic")
q.block_till_done(poll_interval=60)
def squeezer(q, log_dir):
locations = dbtrees.loctree(None, 35)
root_path = os.path.dirname(os.path.abspath(__file__))
runfile = "%s/squeeze_em_all.py" % root_path
for location_id in [l.id for l in locations.leaves()]:
for year_id in [1990, 1995, 2000, 2005, 2010, 2016]:
for sex_id in [1, 2]:
params = [
'--location_id',
str(location_id), '--year_id',
str(year_id), '--sex_id',
str(sex_id)
]
remote_job = job.Job(mon_dir=log_dir,
runfile=runfile,
name='squeeze_%s_%s_%s' %
(location_id, year_id, sex_id),
job_args=params)
q.queue_job(remote_job,
slots=30,
memory=60,
project='proj_epic',
stderr='/FILEPATH',
stdout='/FILEPATH')
q.block_till_done(poll_interval=60)
def execute(self):
# compile submission arguments
kwargs = self.build_args[1]
source_me_id = kwargs.pop("source_me_id")
target_me_id = kwargs.pop("target_me_id")
# make server directory
directory = "{root}/{proc}".format(root=root, proc=self.identity)
if not os.path.exists(directory):
os.makedirs(directory)
else:
shutil.rmtree(directory)
os.makedirs(directory)
west_params = [
"--source_me_id", source_me_id,
"--target_me_id", target_me_id
]
q = self.get_qmaster()
remote_job = job.Job(
mon_dir=log_dir,
name=self.identity,
runfile="{root}/infertility/westrom.py".format(root=root),
job_args=west_params)
q.queue_job(
remote_job,
slots=10,
memory=20,
project={PROJECT},
stderr=log_dir,
stdout=log_dir)
q.block_till_done(poll_interval=60) # monitor them
def __init__(self, mvid, cv_iter_id):
self.mvid = mvid
self.cv_iter_id = cv_iter_id
try:
j = job.Job('%s/%s' % (settings['cascade_ode_out_dir'], mvid))
j.start()
except IOError as e:
logging.exception(e)
except Exception as e:
logging.exception(e)
self.cascade = Cascade(mvid, reimport=True, cv_iter=cv_iter_id)
self.meid = (
self.cascade.model_version_meta.modelable_entity_id.values[0])
if self.meid in [9422, 7695, 1175, 10352, 9309]:
self.project = "proj_tb"
self.is_tb = True
else:
self.project = "proj_dismod"
self.is_tb = False
self.logdir = '{}/{}'.format(settings['log_dir'], self.mvid)
self.finished = False
self.has_csmr = 'mtspecific' in self.cascade.data.integrand.unique()
if cv_iter_id == 0:
self.jt_dir = "{}/".format(self.cascade.root_dir)
else:
self.jt_dir = "{}/{}".format(self.cascade.root_dir, cv_iter_id)
self.rerun_num = self.get_rerun_num()
ijs = self.incomplete_jobs()
# Run global once
self.run_jobtree_global()
# Check if retry limit has been exceeded
if self.rerun_num > 3:
elog_file = '{}/error{}.log'.format(self.cascade.root_dir,
self.cv_iter_id)
with open(elog_file, 'w') as log:
log.write('Model is incomplete after two attempted relaunches')
for ij in ijs:
log.write(str(ij))
sys.exit()
# Submit all jobs and checking job, making sure varnish waits
# for the checking job to complete
varn_jobname = 'dm_%s_varnish' % (mvid)
varn_job = sge.qstat(pattern=varn_jobname)
varn_jid = int(varn_job.job_id.values[0])
if len(ijs) > 0:
jids = self.submit_cascade_jobs(ijs)
pjid = self.resubmit_self_check(jids)
sge.add_holds(varn_jid, pjid)
def execute(self):
# get args
kwargs = self.build_args[1]
parent_meid = kwargs["parent_meid"]
env = "prod"
# get qmaster
q = self.get_qmaster()
# submit split job
remote_job = job.Job(
mon_dir=log_dir,
name="split_" + (str(parent_meid)),
runfile=os.path.join(code_dir, "scripts", "FILEPATH.py"),
job_args=[str(parent_meid), env])
q.queue_job(
remote_job,
slots=49,
memory=98,
project="proj_epic")
q.block_till_done(poll_interval=60)
# submit aggregation/save jobs
outputs_tuples = self.builder.get_process_outputs(self.identity)
children_meids = [task_tuple[0] for task_tuple in outputs_tuples]
for meid in children_meids:
mvid = self._get_latest_mvid(meid)
remote_job = job.Job(
mon_dir=log_dir,
name="save_" + str(mvid),
runfile=sge.true_path(executable="aggregate_mvid"),
job_args=[str(mvid), '--env', env, '--mark_best'])
q.queue_job(
remote_job,
slots=40,
memory=80,
project="proj_epic")
q.block_till_done(poll_interval=60)
def saver(q, dirs, save_func, log_dir):
runfile = sge.true_path(executable=save_func)
for d in dirs:
meid = os.path.basename(d)
params = [
meid, "super-squeeze result", d, '--env', 'prod', '--file_pattern',
'{location_id}_{year_id}_{sex_id}.h5', '--h5_tablename', 'draws',
'--best'
]
remote_job = job.Job(mon_dir=log_dir,
runfile=runfile,
name='ss_save_%s' % meid,
job_args=params)
q.queue_job(remote_job,
slots=20,
memory=40,
project='proj_epic',
stderr='/FILEPATH',
stdout='/FILEPATH')
q.block_till_done(poll_interval=60)
import sys
import os
import upload
import fit_stats
import drill
import subprocess
from jobmon import sge, job
from adding_machine.agg_locations import aggregate_mvid
# Set default file mask to readable-for all users
os.umask(0o0002)
if __name__ == "__main__":
mvid = sys.argv[1]
j = job.Job('%s/%s' % (drill.settings['cascade_ode_out_dir'], mvid))
j.start()
try:
commit_hash = sge.get_commit_hash(dir='%s/..' % drill.this_path)
upload.set_commit_hash(mvid, commit_hash)
upload.upload_model(mvid)
outdir = "%s/%s/full" % (drill.settings['cascade_ode_out_dir'],
str(mvid))
joutdir = "%s/%s" % (drill.settings['diag_out_dir'], mvid)
logdir = '%s/%s' % (drill.settings['log_dir'], mvid)
fit_stats.write_fit_stats(mvid, outdir, joutdir)
upload.upload_fit_stat(mvid)
# Write effect PDFs
if sys.argv[6] == "debug":
debug = True
else:
debug = False
except:
debug = False
if sex == 'male':
sex_id = 0.5
elif sex == 'female':
sex_id = -0.5
c = Cascade(mvid, reimport=False, cv_iter=cv_iter)
try:
j = job.Job(os.path.normpath(os.path.join(c.root_dir, '..')))
j.start()
except IOError as e:
logging.exception(e)
except Exception as e:
logging.exception(e)
year_split_lvl = c.model_version_meta.fix_year.values[0] - 1
lt = c.loctree
this_lvl = lt.get_nodelvl_by_id(location_id)
if location_id == 1:
cl_parent = Cascade_loc(location_id, 0, 2000, c, reimport=False)
else:
cl_parent = Cascade_loc(location_id, sex_id, y, c, reimport=False)
num_children = len(lt.get_node_by_id(location_id).children)
import sys
import os
import upload
import json
import settings
from jobmon import job
# Set default file mask to readable-for all users
os.umask(0o0002)
if __name__ == "__main__":
mvid = sys.argv[1]
# Get configuration options
jm_path = os.path.dirname(job.__file__)
sett = settings.load()
j = job.Job('%s/%s' % (sett['cascade_ode_out_dir'], mvid))
j.start()
upload.upload_final(mvid)
upload.update_model_status(mvid, 1)
j.finish()
j.send_request('stop')
def execute(self):
# compile submission arguments
kwargs = self.build_args[1]
male_prop_id = kwargs.pop("male_prop_id")
female_prop_id = kwargs.pop("female_prop_id")
exp_id = kwargs.pop("exp_id")
env_id = kwargs.pop("env_id")
male_env_id = kwargs.pop("male_env_id")
female_env_id = kwargs.pop("female_env_id")
# make server directory
directory = "{root}/{proc}".format(root=root, proc=self.identity)
if not os.path.exists(directory):
os.makedirs(directory)
else:
shutil.rmtree(directory)
os.makedirs(directory)
# make output directories
for _id in [male_env_id, female_env_id]:
sub_dir = os.path.join(directory, str(_id))
if not os.path.exists(sub_dir):
os.makedirs(sub_dir)
else:
shutil.rmtree(sub_dir)
os.makedirs(sub_dir)
env_params = ["--male_prop_id", str(male_prop_id), "--female_prop_id",
str(female_prop_id), "--exp_id", str(exp_id), "--env_id", str(env_id),
"--male_env_id", str(male_env_id), "--female_env_id",
str(female_env_id), "--out_dir", str(directory), "--year_id"]
q = self.get_qmaster()
# try:
# parallelize by year
for i in {YEAR IDS}:
remote_job=job.Job(
mon_dir=log_dir,
name="{proc}_{loc}".format(proc=self.identity,
loc=i),
runfile="{root}/infertility/calc_env.py".format(root=root),
job_args=env_params + [str(i)])
q.queue_job(
remote_job,
slots=7,
memory=14,
project={PROJECT},
stderr=log_dir,
stdout=log_dir)
q.block_till_done(poll_interval=60) # monitor them
# save the results
for save_id in [male_env_id, female_env_id]:
save_params = [
str(save_id), env_description,
os.path.join(directory, str(save_id)), "--best",
"--file_pattern", "{year_id}.h5", "--h5_tablename", "data"]
remote_job=job.Job(
mon_dir=log_dir,
name="save_" + str(save_id),
runfile=("{FILEPATH}/save_custom_results"),
job_args=save_params)
q.queue_job(
remote_job,
slots=20,
memory=40,
project={PROJECT},
stderr=log_dir,
stdout=log_dir)
q.block_till_done(poll_interval=60) # monitor them
def execute(self):
kwargs = self.build_args[1]
excess_id = kwargs.pop("excess")
redist_map = kwargs.pop("redist")
# make server directory
directory = "{root}/{proc}".format(root=root, proc=self.identity)
if not os.path.exists(directory):
os.makedirs(directory)
else:
shutil.rmtree(directory)
os.makedirs(directory)
# make output directories
for me_id in redist_map.values():
sub_dir = os.path.join(directory, str(me_id))
if not os.path.exists(sub_dir):
os.makedirs(sub_dir)
else:
shutil.rmtree(sub_dir)
os.makedirs(sub_dir)
exs_params = ["--excess_id", str(excess_id),
"--redist_map", json.dumps(redist_map),
"--out_dir", directory,
"--year_id"]
q = self.get_qmaster() # monitor
# attribution jobs by location_id
for i in [{YEAR IDS}]:
remote_job=job.Job(
mon_dir=log_dir,
name="{proc}_{loc}".format(proc=self.identity,
loc=i),
runfile="{root}/infertility/excess.py".format(
root=root),
job_args=exs_params + [str(i)])
q.queue_job(
remote_job,
slots=5,
memory=10,
project={PROJECT},
stderr=log_dir,
stdout=log_dir)
time.sleep(1.5)
q.block_till_done(poll_interval=60) # monitor them
# save the results
for save_id in redist_map.values():
save_params = [
str(save_id), excess_description,
os.path.join(directory, str(save_id)), "--best",
"--sexes", "{SEX ID}", "--file_pattern", "{year_id}.h5",
"--h5_tablename", "data"]
remote_job=job.Job(
mon_dir=log_dir,
name="save_" + str(save_id),
runfile=("{FILEPATH}/save_custom_results"),
job_args=save_params)
q.queue_job(
remote_job,
slots=20,
memory=40,
project={PROJECT},
stderr=log_dir,
stdout=log_dir)
q.block_till_done(poll_interval=60) # monitor them
def execute(self):
# compile submission arguments
me_map = self.build_args[0][0]
# make server directory
directory = "{root}/{proc}".format(root=root, proc=self.identity)
if not os.path.exists(directory):
os.makedirs(directory)
else:
shutil.rmtree(directory)
os.makedirs(directory)
# make output directories
save_ids = []
for mapper in me_map.values():
outputs = mapper.get("trgs", {})
for me_id in outputs.values():
os.makedirs(os.path.join(directory, str(me_id)))
save_ids.append(me_id)
attr_params = ["--me_map", json.dumps(me_map),
"--out_dir", directory,
"--year_id"]
q = self.get_qmaster() # monitor
# attribution jobs by year_id
for i in [{YEAR IDS}]:
remote_job=job.Job(
mon_dir=log_dir,
name="{proc}_{year}".format(proc=self.identity,
year=i),
runfile="{root}/infertility/male_attr.py".format(
root=root),
job_args=attr_params + [str(i)])
q.queue_job(
remote_job,
slots=3,
memory=6,
project={PROJECT},
stderr=log_dir,
stdout=log_dir)
q.block_till_done(poll_interval=60) # monitor them
# save the results
for save_id in save_ids:
save_params = [
str(save_id), male_attr_description,
os.path.join(directory, str(save_id)), "--best",
"--sexes", "{SEX ID}", "--file_pattern", "{year_id}.h5",
"--h5_tablename", "data"]
remote_job=job.Job(
mon_dir=log_dir,
name="save_" + str(save_id),
runfile=("{FILEPATH}/save_custom_results"),
job_args=save_params)
q.queue_job(
remote_job,
slots=20,
memory=40,
project={PROJECT},
stderr=log_dir,
stdout=log_dir)
q.block_till_done(poll_interval=60) # monitor them
def execute(self):
# compile submission arguments
kwargs = self.build_args[1]
pid_env_id = kwargs.pop("pid_env_id")
chlam_prop_id = kwargs.pop("chlam_prop_id")
gono_prop_id = kwargs.pop("gono_prop_id")
other_prop_id = kwargs.pop("other_prop_id")
chlam_id = kwargs.pop("chlam_id")
gono_id = kwargs.pop("gono_id")
other_id = kwargs.pop("other_id")
# make server directory
directory = "{root}/{proc}".format(root=root, proc=self.identity)
if not os.path.exists(directory):
os.makedirs(directory)
else:
shutil.rmtree(directory)
os.makedirs(directory)
split_params = [
str(pid_env_id),
"--target_meids", str(chlam_id), str(gono_id), str(other_id),
"--prop_meids", str(chlam_prop_id), str(gono_prop_id), str(other_prop_id),
"--split_meas_ids", "{MEASURE ID}", "{MEASURE ID}",
"--prop_meas_id", "{MEASURE ID}",
"--output_dir", directory]
q = self.get_qmaster()
# try:
remote_job = job.Job(
mon_dir=log_dir,
name=self.identity,
runfile=("{FILEPATH}/epi_splits"),
job_args=split_params)
q.queue_job(
remote_job,
slots=40,
memory=60,
project={PROJECT},
stderr=log_dir,
stdout=log_dir)
q.block_till_done(poll_interval=60) # monitor them
# save the results
for save_id in [chlam_id, gono_id, other_id]:
save_params = [
str(save_id), pid_split_description,
os.path.join(directory, str(save_id)), "--best",
"--sexes", "{SEX ID}", "--file_pattern", "{location_id}.h5",
"--h5_tablename", "draws"]
remote_job = job.Job(
mon_dir=log_dir,
name="save_" + str(save_id),
runfile=("{FILEPATH}/save_custom_results"),
job_args=save_params)
q.queue_job(
remote_job,
slots=20,
memory=40,
project={PROJECT},
stderr=log_dir,
stdout=log_dir)
q.block_till_done(poll_interval=60) # monitor them