def log_status_result(self, conditions, work_time_secs,
database, failed=False, err_msg=None):
"""
log_status_result updates a record in the status table based on
what happened with a job.
Args:
conditions -- a dict with keys 'data_date' and 'table_versions'
work_time_secs -- the integer amount of time the job took in seconds
failed -- whether the job succeeded
err_msg -- an error message to go in the status table
Return:
---
"""
status = "error" if failed else "complete"
if err_msg:
err_msg = "{0}".format(err_msg)[:ERROR_MSG_COL_SIZE]
finish_time = datetime.utcnow()
update = StatusRecord(
et_status=status,
et_runtime=work_time_secs,
et_finished=finish_time,
error_message=err_msg, updated_at=finish_time
)
update_string = get_update_string(update.get_set_clause())
update.record['data_date'] = conditions['data_date']
update.record['table_versions'] = conditions['table_versions']
params_dict = update.minimized_record()
self.psql.run_sql(
update_string, database, 'update', params=params_dict
)
def update_status(self, db, dd, versions, status,
start_time_secs=None, error_msg=None):
"""
update_status updates the status of a job in the
sdw_pipeline status table
Args:
db -- the Redshift database containing the status_table
dd -- the date string of the data formatted YYYY/MM/DD
versions -- the table versions of config.yaml
status -- the new status
start_time_secs -- the start time of the job in seconds from the epoch
error_msg -- an optional error message
Returns:
---
"""
utc_now = datetime.utcnow()
params = {'load_status': status, 'updated_at': utc_now}
if start_time_secs is not None:
params['load_runtime'] = int(time.time() - start_time_secs)
params['load_finished'] = utc_now
if error_msg:
params['error_message'] = error_msg[:ERROR_MSG_COL_SIZE]
sr = StatusRecord(**params)
params['table_versions'] = versions
params['data_date'] = dd
self.psql.run_sql(get_update_string(sr.get_set_clause()), db,
'updating status', params=params)
def test_status_record_get_update_string():
staticconf.YamlConfiguration("config.yaml")
sr = StatusRecord(et_status="complete", et_runtime=1000)
expected_value = "UPDATE sdw_pipeline_status SET et_runtime = "\
"%(et_runtime)s, et_status = %(et_status)s WHERE "\
"data_date = %(data_date)s AND "\
"table_versions = %(table_versions)s"
output_under_test = get_update_string(sr.get_set_clause())
assert expected_value == output_under_test
def test_status_record_get_update_string():
staticconf.YamlConfiguration("config.yaml")
sr = StatusRecord(et_status="complete", et_runtime=1000)
expected_value = "UPDATE sdw_pipeline_status SET et_runtime = "\
"%(et_runtime)s, et_status = %(et_status)s WHERE "\
"data_date = %(data_date)s AND "\
"table_versions = %(table_versions)s"
output_under_test = get_update_string(sr.get_set_clause())
assert expected_value == output_under_test
def log_status_result(self,
conditions,
work_time_secs,
database,
failed=False,
err_msg=None):
"""
log_status_result updates a record in the status table based on
what happened with a job.
Args:
conditions -- a dict with keys 'data_date' and 'table_versions'
work_time_secs -- the integer amount of time the job took in seconds
failed -- whether the job succeeded
err_msg -- an error message to go in the status table
Return:
---
"""
status = "error" if failed else "complete"
if err_msg:
err_msg = "{0}".format(err_msg)[:ERROR_MSG_COL_SIZE]
finish_time = datetime.utcnow()
update = StatusRecord(et_status=status,
et_runtime=work_time_secs,
et_finished=finish_time,
error_message=err_msg,
updated_at=finish_time)
update_string = get_update_string(update.get_set_clause())
update.record['data_date'] = conditions['data_date']
update.record['table_versions'] = conditions['table_versions']
params_dict = update.minimized_record()
self.psql.run_sql(update_string,
database,
'update',
params=params_dict)
def update_status(self,
db,
dd,
versions,
status,
start_time_secs=None,
error_msg=None):
"""
update_status updates the status of a job in the
sdw_pipeline status table
Args:
db -- the Redshift database containing the status_table
dd -- the date string of the data formatted YYYY/MM/DD
versions -- the table versions of config.yaml
status -- the new status
start_time_secs -- the start time of the job in seconds from the epoch
error_msg -- an optional error message
Returns:
---
"""
utc_now = datetime.utcnow()
params = {'load_status': status, 'updated_at': utc_now}
if start_time_secs is not None:
params['load_runtime'] = int(time.time() - start_time_secs)
params['load_finished'] = utc_now
if error_msg:
params['error_message'] = error_msg[:ERROR_MSG_COL_SIZE]
sr = StatusRecord(**params)
params['table_versions'] = versions
params['data_date'] = dd
self.psql.run_sql(get_update_string(sr.get_set_clause()),
db,
'updating status',
params=params)
