def get_coll_data(db_filepath):
queryColls = ("SELECT coll.uuid as coll_uuid, "
"comm.short_name as Community, "
"coll.short_name as Collection "
"FROM Community comm "
"JOIN Collection coll ON comm.uuid = coll.comm_uuid "
"WHERE coll.reportable = 1;")
try:
with closing(sqlite3.connect(db_filepath)) as conn:
conn.execute("PRAGMA foreign_keys = 1")
with closing(conn.cursor()) as cursor:
coll_rows = cursor.execute(queryColls,).fetchall()
except sqlite3.Warning as w:
logging.warning(f"Ingest warning: {w}")
drp.cleanup_and_exit(1)
except sqlite3.Error as e:
logging.error(f"Ingest error: {e}")
drp.cleanup_and_exit(1)
#validate results
logging.debug(f'Count of all collection rows: {len(coll_rows)}')
assert len(coll_rows) > 0, f"Number of total collections: {len(coll_rows)}"
return coll_rows
def run(cfg):
logging.info(f"Using database file {cfg.db_filepath}")
# ingest and reporting
try:
with closing(sqlite3.connect(cfg.db_filepath)) as conn:
conn.execute("PRAGMA foreign_keys = 1")
with closing(conn.cursor()) as cursor:
if cfg.skip_ingest:
logging.info("Skipping data ingest")
else:
run_ingest(conn, cursor,
cfg.data_in_filename_filters, cfg.data_in_field_sep,
cfg.data_in_dirpath, cfg.data_done_dirpath)
if cfg.skip_output:
logging.info("Skipping report generation")
else:
itct_rows = query_rpt_data(conn, cursor,
cfg.rpt_year)
except AssertionError as msg:
logging.error(f"Error validating data ingest files: {msg} ")
drp.cleanup_and_exit(1)
except sqlite3.Warning as w:
logging.warning(f"Ingest warning: {w}")
drp.cleanup_and_exit(1)
except sqlite3.Error as e:
logging.error(f"Ingest error: {e}")
drp.cleanup_and_exit(1)
if not cfg.skip_output:
try:
generate_report_files(itct_rows,
cfg.rpts_out_dirpath, cfg.rpt_basename,
cfg.rpt_formats, cfg.rpt_year)
except AssertionError as msg:
logging.error(msg)
drp.cleanup_and_exit(1)
# if not cfg.skip_distr:
# try:
# email rpts
logging.info("End of DRP CICMTS run")
drp.cleanup_and_exit(0)
def main(argv):
cfg = config.get_config()
# init logging
shared.config.log_init(cfg.rpt_dirpath / "cicmts.log",
cfg.log_file_level,
cfg.log_console_level,
cfg.log_file_append)
logging.info(f"DRP module CICMTS, version {cfg.version}")
#commandline options, with potential to override config values
try:
config.get_commandline_options(argv, cfg)
config.validate_config(cfg)
except GetoptError as e:
logging.error(e.msg)
print(config.get_help_text())
drp.cleanup_and_exit(1)
except AssertionError as msg:
logging.error(msg)
#print(config.get_help_text())
drp.cleanup_and_exit(1)
except ValueError as msg:
logging.error(msg)
print(config.get_help_text())
drp.cleanup_and_exit(1)
run(cfg)
def run(cfg):
logging.info(f"Using database file {cfg.db_filepath}")
try:
coll_rows = get_coll_data(cfg.db_filepath)
itct_filepath = get_itct_filepath(cfg.data_in_filename_filters[0],
cfg.data_in_dirpath)
generate_reports(coll_rows, itct_filepath, cfg.data_in_field_sep,
cfg.rpt_formats, cfg.rpts_out_dirpath, cfg.rpt_basename)
#move processed file
itct_filepath.rename(cfg.data_done_dirpath / itct_filepath.name)
logging.info(f"moved {itct_filepath.name} to completed directory")
except AssertionError as msg:
logging.error(msg)
drp.cleanup_and_exit(1)
except ValueError as e:
logging.error(e)
drp.cleanup_and_exit(1)
# if not cfg.skip_distr:
# try:
# email rpts
logging.info("End of DRP CICAH run")
drp.cleanup_and_exit(0)