def get_locations():
'''
Description: get list of locations to iterate through for every part of the
maternal custom process, down to one level of subnationals
Args: None
Output: (list) location_ids
'''
enginer = dbapis.engine_factory()
loc_set_version_query = '''
SELECT location_set_version_id FROM shared.location_set_version
WHERE location_set_id = 35 AND end_date IS NULL'''
location_set_version = query_tools.query_2_df(
loc_set_version_query,
engine=enginer.engines["cod_prod"]).ix[
0, 'location_set_version_id']
query = ('call shared.view_location_hierarchy_history(%s)'
% location_set_version)
locations_df = query_tools.query_2_df(query,
engine=enginer.engines["cod_prod"])
locations = (locations_df[locations_df['most_detailed'] == 1][
'location_id'].tolist())
return locations
def check_dependencies(step):
'''
Description: Checks dependencies of the step given, using the dependency
map.
Args: specify which step for which you want to check dependencies
Options: 1, 2, 3, or '4'
Output: True or False, which turns the step specified to 'On' or 'Off'
'''
dep_map = pd.read_csv(
"dependency_map.csv", header=0).dropna(axis='columns', how='all')
step_df = dep_map.ix[dep_map.step == step]
enginer = dbapis.engine_factory()
if len(step_df) != 0:
bool_list = []
for idx in step_df.index:
src_date = get_best_date(
enginer, step, step_df.ix[idx, "source_type"],
step_df.ix[idx, "source_id"])
trg_date = get_best_date(
enginer, step, step_df.ix[idx, "target_type"],
step_df.ix[idx, "target_id"])
boolean = src_date > trg_date
bool_list.append(boolean)
if any(bool_list):
return True
else:
return False
else:
raise ValueError("Must specify 1, 2, 3, or 4")
from db_tools import dbapis, query_tools
import subprocess
import time
# create enginer
enginer = dbapis.engine_factory()
query = ('''
SELECT
*
FROM
cod.model_version
WHERE
cause_id = 366 AND model_version_type_id = 3 AND is_best = 1''')
while True:
codem_df = query_tools.query_2_df(query,
engine=enginer.engines["cod_prod"])
if len(codem_df) == 1:
break
else:
print "CODEm model still running"
time.sleep(600)
call = ('qsub -cwd -P proj_custom_models -o FILEPATH '
'-e FILEPATH -pe multi_slot 10 '
'-N master_maternal cluster_shell.sh 00_master_maternal.py')
subprocess.call(call, shell=True)
def get_model_vers(process, model_id=None, step=None):
'''
Description: Queries the database for the best model_version for the given
model_id. Can do this for Dismod, Codem, or Codcorrect outputs.
Args:
1. (str) process ('dismod', 'codem', or 'codcorrect')
2. id (model_id for dismod, cause_id for codem, or none for codcorrect)
Output: (int) best model_version
'''
enginer = dbapis.engine_factory()
enginer.servers["gbd"] = {"prod": "modeling-gbd-db.ihme.washington.edu"}
enginer.define_engine(strConnection)
if model_id is not None:
model_id = int(model_id)
if process == 'dismod':
if model_id is None:
raise ValueError('Must specify a me_id')
else:
query = '''SELECT model_version_id from epi.model_version WHERE
is_best = 1 AND best_end IS NULL AND modelable_entity_id =
%d''' % model_id
model_vers_df = query_tools.query_2_df(
query, engine=enginer.engines['epi_prod'])
if len(model_vers_df) > 0:
model_vers = model_vers_df.ix[0, 'model_version_id']
else:
model_vers = None
elif process == 'codem':
if model_id is None:
raise ValueError('Must specify a cause_id')
if step == 2:
query = '''SELECT MAX(model_version_id) as model_version_id from
cod.model_version where best_start IS NOT NULL AND
best_start > '2015-01-01 00:00:01'
AND cause_id = %d and model_version_type_id = 3''' % model_id
else:
query = '''SELECT model_version_id from cod.model_version where
best_end IS NULL AND best_start IS NOT NULL AND
best_start > '2015-01-01 00:00:01'
AND cause_id = %d''' % model_id
model_vers = query_tools.query_2_df(query, engine=enginer.engines[
"cod_prod"]).ix[0, 'model_version_id']
else:
query = ('SELECT '
'distinct(val) AS daly_id '
'FROM '
'gbd.gbd_process_version_metadata gpvm '
'JOIN '
'gbd.gbd_process_version USING (gbd_process_version_id) '
'JOIN '
'gbd.compare_version_output USING (compare_version_id) '
'WHERE '
'compare_version_id = (SELECT '
'compare_version_id '
'FROM '
'gbd.compare_version '
'WHERE '
'compare_version_status_id = 1 '
'AND gbd_round_id = 3) '
'AND gpvm.metadata_type_id = 5')
model_vers = query_tools.query_2_df(
query, engine=enginer.engines["gbd_prod"]).loc[0, 'daly_id']
return model_vers
def __init__(self, cluster_dir, year_id, input_me, output_me):
self.cluster_dir = cluster_dir
self.year_id = year_id
self.input_me = input_me
self.output_me = output_me
self.enginer = dbapis.engine_factory()