def save_custom_results(
meid,
description,
input_dir,
years=[1990, 1995, 2000, 2005, 2010, 2015],
sexes=[1, 2],
mark_best=False,
in_counts=False,
env='dev',
custom_file_pattern=None,
h5_tablename=None):
if in_counts:
operator = 'sum'
else:
operator = 'wtd_sum'
print "========================================="
print "%s Beginning save_results" % pretty_now()
try:
if env == 'dev':
db = EpiDB('epi-dev-custom')
elif env == 'prod':
db = EpiDB('epi')
else:
raise Exception("Only dev or prod environments are supported")
except Exception, e:
print """Could not initialize ODBC connection. Are you sure you
have properly configured you ~/.odbc.ini file?"""
raise e
def get_age_spans():
query = """
SELECT age_group_id, age_group_years_start, age_group_years_end
FROM shared.age_group"""
db = EpiDB('epi')
eng = db.get_engine(db.dsn_name)
ags = pd.read_sql(query, eng)
return ags
def get_age_weights():
query = """
SELECT age_group_id, age_group_weight_value
FROM shared.age_group_weight
WHERE gbd_round_id = 3"""
db = EpiDB('epi')
eng = db.get_engine(db.dsn_name)
aws = pd.read_sql(query, eng)
return aws
def get_pop():
query = """
SELECT age_group_id, year_id, location_id, sex_id, pop_scaled
FROM mortality.output o
LEFT JOIN mortality.output_version ov using (output_version_id)
WHERE ov.is_best = 1
AND year_id >= 1980 AND year_id <= 2015"""
db = EpiDB('cod')
eng = db.get_engine(db.dsn_name)
pop = pd.read_sql(query, eng)
return pop
def get_pop(filters={}):
query = """
SELECT o.age_group_id, year_id, o.location_id, o.sex_id, pop_scaled
FROM mortality.output o
LEFT JOIN mortality.output_version ov using (output_version_id)
LEFT JOIN shared.age_group a using (age_group_id)
LEFT JOIN shared.location l using (location_id)
LEFT JOIN shared.sex s using (sex_id)
WHERE ov.is_best = 1
AND year_id >= 1980 AND year_id <= 2015"""
for k, v in filters.iteritems():
v = np.atleast_1d(v)
v = [str(i) for i in v]
query = query + " AND {k} IN ({vlist})".format(
k=k, vlist=",".join(v))
db = EpiDB('cod')
eng = db.get_engine(db.dsn_name)
pop = pd.read_sql(query, eng)
return pop
def aggregate_mvid(
model_version_id, env='dev', custom_file_pattern='all_draws.h5',
h5_tablename='draws', single_file=True, odbc_filepath='~/.odbc.ini',
mark_best=False, best_description="auto-marked best"):
try:
if env == 'dev':
db = EpiDB('epi-dev-custom', odbc_filepath=odbc_filepath)
elif env == 'prod':
db = EpiDB('epi', odbc_filepath=odbc_filepath)
elif 'cascade' in env:
db = EpiDB('epi-cascade', odbc_filepath=odbc_filepath)
env = env.split("-")[1]
else:
raise Exception(
"Only dev/prod/cascade-dev/cascade-prod "
"environments are supported")
except Exception, e:
print """Could not initialize ODBC connection. Are you sure you
have properly configured you ~/.odbc.ini file?"""
raise e
def write_me(nwait, nmeids, outqueue, mvid_q):
if envi == 'dev':
db = EpiDB('epi-dev-custom')
elif envi == 'prod':
db = EpiDB('epi')
get_count = 0
me_mv = {}
while get_count < nwait:
try:
df = outqueue.get()
meid = df.modelable_entity_id.unique()[0]
if meid not in me_mv:
print 'Creating mvid for meid %s' % (meid)
mvid = db.create_model_version(
meid, 'Central severity split: parent %s' % ss.parent_meid,
75)
print 'Created mvid %s for meid %s' % (mvid, meid)
me_mv[meid] = mvid
else:
mvid = me_mv[meid]
outfile = (
'/ihme/epi/panda_cascade/%s/%s/full/draws/all_draws.h5' %
(envi, mvid))
for col in [
'location_id', 'year_id', 'age_group_id', 'sex_id',
'measure_id'
]:
df[col] = df[col].astype(int)
df = df[[
'location_id', 'year_id', 'age_group_id', 'sex_id',
'measure_id'
] + ['draw_%s' % i for i in range(1000)]]
store = pd.HDFStore(outfile)
store.append('draws',
df,
data_columns=[
'measure_id', 'location_id', 'year_id',
'age_group_id', 'sex_id'
],
index=False)
store.close()
get_count += 1
except Exception as e:
print str(e)
get_count += 1
try:
for meid, mvid in me_mv.iteritems():
outfile = (
'/ihme/epi/panda_cascade/%s/%s/full/draws/all_draws.h5' %
(envi, mvid))
store = pd.HDFStore(outfile)
print 'Creating index for mv %s for meid %s' % (mvid, meid)
store.create_table_index('draws',
columns=[
'measure_id', 'location_id',
'year_id', 'age_group_id', 'sex_id'
],
optlevel=9,
kind='full')
store.close()
print 'Closing file for mv %s for meid %s' % (mvid, meid)
mvid_q.put(mvid)
nmeids = nmeids - 1
except Exception as e:
print 'Uh oh, hit a writing error %s' % e
for i in range(nmeids):
mvid_q.put((500, str(e)))
from hierarchies import dbtrees
from transmogrifier import gopher, maths
import pandas as pd
import numpy as np
from itertools import cycle
from db import EpiDB
from multiprocessing import Queue, Process
from . import git_info
edb = EpiDB()
engine = edb.get_engine('epi')
sentinel = None
class SevSplitter(object):
def __init__(self, parent_meid, prop_drawfile=None):
self.parent_meid = parent_meid
self.lt = dbtrees.loctree(None, location_set_id=35)
self.ags = self.age_groups()
self.child_meids = self.get_children()
if prop_drawfile is None:
self.props = self.get_split_proportions()
self.props = self.gen_proportion_draws()
else:
self.props = pd.read_csv(prop_drawfile)
def get_children(self):
q = """
SELECT parent_meid, child_meid
FROM severity_splits.hierarchy h
JOIN severity_splits.hierarchy_version hv