def submit_job_location_draws(indicator_type, location_id):
"""Submit a cluser job to process a location year."""
log_dir = "FILEPATH"
if not os.path.exists(log_dir):
os.mkdir(log_dir)
os.mkdir(os.path.join(log_dir, 'errors'))
os.mkdir(os.path.join(log_dir, 'output'))
jobname = "sdg_loc_{it}_{loc}".format(it=indicator_type, loc=location_id)
worker = "{}/data_prep/prep_draw_files_parallel.py".format(SDG_REPO)
shell = "{}/sdg_utils/run_on_cluster.sh".format(SDG_REPO)
args = {
'--process': "run_location",
'--indicator_type': indicator_type,
'--location_id': location_id
}
if indicator_type in ['codcorrect', 'risk_exposure']:
jobslots = 3
else:
jobslots = 18
job_id = ch.qsub(worker,
shell,
'proj_sdg',
custom_args=args,
name=jobname,
log_dir=log_dir,
slots=jobslots,
verbose=True)
return job_id
def submit_job_collect(job_ids, indicator_type):
"""Submit a cluster job to collect processed draws."""
log_dir = "FILEPATH"
if not os.path.exists(log_dir):
os.mkdir(log_dir)
os.mkdir(os.path.join(log_dir, 'errors'))
os.mkdir(os.path.join(log_dir, 'output'))
jobname = "sdg_collect_{it}".format(it=indicator_type)
worker = "{}/data_prep/prep_draw_files_parallel.py".format(SDG_REPO)
shell = "{}/sdg_utils/run_on_cluster.sh".format(SDG_REPO)
args = {
'--process': "collect",
'--indicator_type': indicator_type,
}
holds = job_ids
job_id = ch.qsub(worker,
shell,
'proj_sdg',
custom_args=args,
name=jobname,
log_dir=log_dir,
slots=20,
holds=holds,
verbose=True)
return job_id
def submit_collect(measure_id):
"""Submit a cluster job to collect processed draws."""
log_dir = "/share/temp/sgeoutput/sdg/"
if not os.path.exists(log_dir):
os.mkdir(log_dir)
jobname = "sdg_risk_burden_{m}".format(m=measure_id)
worker = "{}/data_prep/risk_burden.py".format(SDG_REPO)
shell = "{}/sdg_utils/run_on_cluster.sh".format(SDG_REPO)
args = ["run_one", measure_id]
job_id = ch.qsub(worker,
shell,
'proj_sdg',
custom_args=args,
name=jobname,
log_dir=log_dir,
slots=10,
verbose=True)
return job_id