def set_up_folders(output_directory, output_version_id):
"""Create a new CoDCorrect id, and then set up _temp, scalars, draws
returns parent directory string.
"""
new_folders = [
'_temp', 'input_data', 'draws', 'shocks', 'models', 'unaggregated',
'unaggregated/unscaled', 'logs', 'unaggregated/rescaled',
'unaggregated/shocks', 'aggregated', 'aggregated/unscaled',
'aggregated/rescaled', 'aggregated/shocks', 'debug', 'diagnostics',
'summaries', 'summaries/gbd', 'summaries/gbd/single',
'summaries/gbd/multi', 'summaries/gbd/single/1',
'summaries/gbd/single/4', 'summaries/gbd/multi/1',
'summaries/gbd/multi/4', 'summaries/cod'
]
for folder in new_folders:
directory = '{d}/{v}/{f}'.format(d=output_directory,
v=output_version_id,
f=folder)
if not os.path.exists(directory):
os.makedirs(directory)
change_permission(directory)
change_permission('{d}/{v}'.format(d=output_directory,
v=output_version_id))
return '{d}/{v}'.format(d=output_directory, v=output_version_id)
def set_up_summary_folders(parent_dir, years, change_years):
new_folders = []
change_start_years = change_years[:2]
for year in years:
new_folders.extend([
'summaries/gbd/single/1/{}'.format(year),
'summaries/gbd/single/4/{}'.format(year)
])
new_folders.extend([
'summaries/gbd/multi/1/{}'.format(year) for year in change_start_years
] + [
'summaries/gbd/multi/4/{}'.format(year) for year in change_start_years
])
for folder in new_folders:
if not os.path.exists(os.path.join(parent_dir, folder)):
os.makedirs(os.path.join(parent_dir, folder))
change_permission(os.path.join(parent_dir, folder))
if not os.path.exists(parent_dir):
change_permission(parent_dir)
if not os.path.exists(os.path.join(parent_dir, '_temp')):
change_permission(os.path.join(parent_dir, '_temp'))
if not os.path.exists(os.path.join(parent_dir, 'summaries')):
change_permission(os.path.join(parent_dir, 'summaries'),
recursively=True)
def set_up_summary_folders(parent_dir, years):
new_folders = []
change_start_years = [1990, 2006]
for year in years:
new_folders.extend(['FILEPATH/{}'.format(year)])
new_folders.extend(['FILEPATH/{}'.format(year)
for year in change_start_years])
for folder in new_folders:
if not os.path.exists(os.path.join(parent_dir, folder)):
os.makedirs(os.path.join(parent_dir, folder))
change_permission(os.path.join(parent_dir, folder))
if not os.path.exists(parent_dir):
change_permission(parent_dir)
def set_up_folders(output_directory, output_directory_j, output_version_id):
"""
Create a new CoDCorrect id, and then set up _temp, scalars, draws
returns parent directory string
"""
new_folders = ['LIST', 'OF', 'FILEPATHS']
for folder in new_folders:
directory = 'FILEPATH'
if not os.path.exists(directory):
os.makedirs(directory)
change_permission(directory)
change_permission('{d}/{v}'.format(d=output_directory,
v=output_version_id))
return '{d}/{v}'.format(d=output_directory, v=output_version_id)
def save_gbd_summaries(data, location_id, years, measure_id, change=False):
"""
Saves draws long in an h5 file.
Returns
None
"""
keep_cols = ['measure_id',
'location_id',
'sex_id',
'age_group_id',
'cause_id',
'metric_id',
'val',
'upper',
'lower']
if change is True:
keep_cols.extend(['year_start_id', 'year_end_id'])
data = data[keep_cols]
data.sort_values(by=['measure_id', 'year_start_id', 'year_end_id',
'location_id', 'sex_id', 'age_group_id',
'cause_id', 'metric_id'], inplace=True)
for year in data.year_start_id.unique():
filepath = (parent_dir + r'/FILEPATH/{m}/{y}/{loc}.csv'
.format(m=int(measure_id), y=int(year),
loc=int(location_id)))
data.ix[data.year_start_id == year].to_csv(filepath, index=False)
change_permission(filepath)
else:
keep_cols.extend(['year_id'])
data = data[keep_cols]
data.sort_values(by=['measure_id', 'year_id', 'location_id', 'sex_id',
'age_group_id', 'cause_id', 'metric_id'],
inplace=True)
for year in data.year_id.unique():
filepath = (parent_dir + r'/FILEPATH/{m}/{y}/{loc}.csv'
.format(m=int(measure_id), y=int(year),
loc=int(location_id)))
data.ix[data.year_id == year].to_csv(filepath, index=False)
change_permission(filepath)
def prep_upload(parent_dir):
change_permission(parent_dir, recursively=False)
change_permission(parent_dir + r'/_temp/', recursively=True)
output_upload_files = read_json(parent_dir + r'/_temp/output_upload.json')
return output_upload_files
print " {} running jobs".format(codcorrect_job_list.running)
print " {} not started jobs".format(codcorrect_job_list.not_started)
print " {} completed jobs".format(codcorrect_job_list.completed)
print " {} failed jobs".format(codcorrect_job_list.failed)
print " {} jobs whose retry attempts are exceeded".format(
codcorrect_job_list.retry_exceeded)
if codcorrect_job_list.retry_exceeded > 0:
codcorrect_job_list.display_jobs(status="Retry exceeded")
else:
print "Creating upload entry in database"
create_new_output_version_row(output_version_id,
"New version of CodCorrect",
envelope_version_id)
if upload_diagnostics_status:
print "Uploading diagnostics"
change_permission(parent_dir)
change_permission(parent_dir + r'/_temp/', recursively=True)
wipe_diagnostics()
new_diagnostic_version(output_version_id)
upload_diagnostics(parent_dir)
if upload_summary_status:
print "Preparing for upload"
change_permission(parent_dir)
change_permission(parent_dir + r'/_temp/', recursively=True)
output_files = prep_upload(parent_dir)
for f in output_files:
upload_summaries(f)
if best:
print "Marking upload as best"
unmark_best(output_version_id)
mark_best(output_version_id)
df = df.loc[df['year_id'].isin(diag_years)]
data.append(df)
# all aggregated locations need diagnostics created
else:
data.append(
calculate_diagnostics(parent_dir, location_id, diag_years))
logging.info("Concatenating in diagnostic files")
diag = pd.concat(data)
# Format for upload
diag['output_version_id'] = output_version_id
diagnostic_table_fields = [
'output_version_id', 'location_id', 'year_id', 'sex_id',
'age_group_id', 'cause_id', 'mean_before', 'mean_after'
]
diag = diag[diagnostic_table_fields]
# Save
logging.info("Saving single diagnostic file")
file_path = parent_dir + r'FILEPATH'
diag.to_csv(file_path, index=False)
change_permission(file_path)
logging.info('All done!')
except Exception:
logging.exception(
'uncaught exception in append_diagnostics.py: {}'.format(
sys.exc_info()[0]))
sys.exit(1)