def upload(self):
"""
Attempts to infile any .csv files found in the
/{output_dir}/{version_id} directory to
gbd.output_le_decomp_v{self.process_version_id}. Any
.csv files not to be uploaded should be stashed in subfolders.
"""
if not self._process_version_id:
raise RuntimeError("A process version must be created "
"before results can be uploaded.")
table_name = list(
GBDProcessVersion(self._process_version_id,
env=self.env).tables)[0]
session = get_session(conn_def=self.conn_def)
infiler = Infiles(table=table_name,
schema=self._schema,
session=session)
print(f"Starting upload at {datetime.now().time()}")
infiler.indir(path=self.write_dir,
partial_commit=True,
sort_files=True)
session.close()
print(f"Finished upload at {datetime.now().time()}")
def upload_cod_summaries(directories, conn_def):
sesh = get_session(conn_def)
exec_query("set unique_checks= 0", sesh)
inf = Infiles(table='output', schema='cod', session=sesh)
for directory in directories:
inf.indir(path=directory,
with_replace=True,
partial_commit=True,
commit=True)
def upload(self, sm, process_vers, in_dir, with_replace=True):
'''Args: sm: 'single' or 'multi', referring to which table to use
process_vers: comes from the output of above
in_dir: filepath of csvs to be uploaded
conn_def: 'gbd' or 'gbd_test'
'''
table = 'output_mmr_%s_year_v%s' % (sm, process_vers)
sesh = get_session(self.conn_def)
inf = Infiles(table=table, schema='gbd', session=sesh)
inf.indir(path=in_dir, with_replace=with_replace,
partial_commit=True, commit=True)
activate_process_version(process_version_id=process_vers, session=sesh)
print("Uploaded! You win at life.")
def load_component_location(self, component, location_id, year_type):
table_tmp = getattr(self, year_type)[component]
table = table_tmp.format(self.como_version.gbd_process_version_id)
sesh = get_session("gbd")
infiler = Infiles(table, "gbd", sesh)
if year_type == "single_year":
indir_glob = f"{self.como_version.como_dir}/summaries/{component}/*/{year_type}/{location_id}/*.csv"
else:
indir_glob = f"{self.como_version.como_dir}/summaries/{component}/*/{year_type}/{location_id}.csv"
infiler.indir(path=indir_glob,
commit=True,
partial_commit=True,
rename_cols={"mean": "val"},
no_raise=(IntegrityError))
def upload_gbd_summaries(process_version, conn_def, directories):
if any('multi' in d for d in directories):
table = 'output_cod_multi_year_v{pv}'.format(pv=process_version)
else:
table = 'output_cod_single_year_v{pv}'.format(pv=process_version)
sesh = get_session(conn_def)
inf = Infiles(table=table, schema='gbd', session=sesh)
for directory in directories:
print("Uploading from {}".format(directory))
inf.indir(path=directory,
with_replace=True,
partial_commit=True,
commit=True)
return "Uploaded"