def log_status_result(self, conditions, work_time_secs,
database, failed=False, err_msg=None):
"""
log_status_result updates a record in the status table based on
what happened with a job.
Args:
conditions -- a dict with keys 'data_date' and 'table_versions'
work_time_secs -- the integer amount of time the job took in seconds
failed -- whether the job succeeded
err_msg -- an error message to go in the status table
Return:
---
"""
status = "error" if failed else "complete"
if err_msg:
err_msg = "{0}".format(err_msg)[:ERROR_MSG_COL_SIZE]
finish_time = datetime.utcnow()
update = StatusRecord(
et_status=status,
et_runtime=work_time_secs,
et_finished=finish_time,
error_message=err_msg, updated_at=finish_time
)
update_string = get_update_string(update.get_set_clause())
update.record['data_date'] = conditions['data_date']
update.record['table_versions'] = conditions['table_versions']
params_dict = update.minimized_record()
self.psql.run_sql(
update_string, database, 'update', params=params_dict
)
def insert_et(self, input_dict, database):
"""
insert_et inserts a record into the status table to show an et job
has started
Args:
input_dict -- a dictionary with the following keys:
data_date -- the date of the data for the et job
table_versions -- a string of '<table_name>: <table_version>' pairs
Returns:
Boolean of whether the insert worked
"""
insert_record = StatusRecord(
data_date=input_dict['data_date'],
et_status="started",
table_versions=input_dict['table_versions']
)
insert_record.set_updated_at()
return self.psql.run_sql(
get_insert_string(insert_record.minimized_record()),
database,
'insert',
params=insert_record.minimized_record()
)
def update_status(self, db, dd, versions, status,
start_time_secs=None, error_msg=None):
"""
update_status updates the status of a job in the
sdw_pipeline status table
Args:
db -- the Redshift database containing the status_table
dd -- the date string of the data formatted YYYY/MM/DD
versions -- the table versions of config.yaml
status -- the new status
start_time_secs -- the start time of the job in seconds from the epoch
error_msg -- an optional error message
Returns:
---
"""
utc_now = datetime.utcnow()
params = {'load_status': status, 'updated_at': utc_now}
if start_time_secs is not None:
params['load_runtime'] = int(time.time() - start_time_secs)
params['load_finished'] = utc_now
if error_msg:
params['error_message'] = error_msg[:ERROR_MSG_COL_SIZE]
sr = StatusRecord(**params)
params['table_versions'] = versions
params['data_date'] = dd
self.psql.run_sql(get_update_string(sr.get_set_clause()), db,
'updating status', params=params)
def test_status_record_get_insert_string():
staticconf.YamlConfiguration("config.yaml")
sr = StatusRecord(data_date="2014/10/13", et_status="wip",
table_versions="search: 1")
output_under_test = get_insert_string(sr.minimized_record())
expected_value = "INSERT INTO sdw_pipeline_status (data_date, et_status, "\
"table_versions) VALUES (%(data_date)s, %(et_status)s, "\
"%(table_versions)s)"
assert expected_value == output_under_test
def test_status_record_get_insert_string():
staticconf.YamlConfiguration("config.yaml")
sr = StatusRecord(data_date="2014/10/13",
et_status="wip",
table_versions="search: 1")
output_under_test = get_insert_string(sr.minimized_record())
expected_value = "INSERT INTO sdw_pipeline_status (data_date, et_status, "\
"table_versions) VALUES (%(data_date)s, %(et_status)s, "\
"%(table_versions)s)"
assert expected_value == output_under_test
def test_status_record_get_update_string():
staticconf.YamlConfiguration("config.yaml")
sr = StatusRecord(et_status="complete", et_runtime=1000)
expected_value = "UPDATE sdw_pipeline_status SET et_runtime = "\
"%(et_runtime)s, et_status = %(et_status)s WHERE "\
"data_date = %(data_date)s AND "\
"table_versions = %(table_versions)s"
output_under_test = get_update_string(sr.get_set_clause())
assert expected_value == output_under_test
def test_status_record_get_update_string():
staticconf.YamlConfiguration("config.yaml")
sr = StatusRecord(et_status="complete", et_runtime=1000)
expected_value = "UPDATE sdw_pipeline_status SET et_runtime = "\
"%(et_runtime)s, et_status = %(et_status)s WHERE "\
"data_date = %(data_date)s AND "\
"table_versions = %(table_versions)s"
output_under_test = get_update_string(sr.get_set_clause())
assert expected_value == output_under_test
def update_status(self,
db,
dd,
versions,
status,
start_time_secs=None,
error_msg=None):
"""
update_status updates the status of a job in the
sdw_pipeline status table
Args:
db -- the Redshift database containing the status_table
dd -- the date string of the data formatted YYYY/MM/DD
versions -- the table versions of config.yaml
status -- the new status
start_time_secs -- the start time of the job in seconds from the epoch
error_msg -- an optional error message
Returns:
---
"""
utc_now = datetime.utcnow()
params = {'load_status': status, 'updated_at': utc_now}
if start_time_secs is not None:
params['load_runtime'] = int(time.time() - start_time_secs)
params['load_finished'] = utc_now
if error_msg:
params['error_message'] = error_msg[:ERROR_MSG_COL_SIZE]
sr = StatusRecord(**params)
params['table_versions'] = versions
params['data_date'] = dd
self.psql.run_sql(get_update_string(sr.get_set_clause()),
db,
'updating status',
params=params)
def insert_et(self, input_dict, database):
"""
insert_et inserts a record into the status table to show an et job
has started
Args:
input_dict -- a dictionary with the following keys:
data_date -- the date of the data for the et job
table_versions -- a string of '<table_name>: <table_version>' pairs
Returns:
Boolean of whether the insert worked
"""
insert_record = StatusRecord(
data_date=input_dict['data_date'],
et_status="started",
table_versions=input_dict['table_versions'])
insert_record.set_updated_at()
return self.psql.run_sql(get_insert_string(
insert_record.minimized_record()),
database,
'insert',
params=insert_record.minimized_record())
def log_status_result(self,
conditions,
work_time_secs,
database,
failed=False,
err_msg=None):
"""
log_status_result updates a record in the status table based on
what happened with a job.
Args:
conditions -- a dict with keys 'data_date' and 'table_versions'
work_time_secs -- the integer amount of time the job took in seconds
failed -- whether the job succeeded
err_msg -- an error message to go in the status table
Return:
---
"""
status = "error" if failed else "complete"
if err_msg:
err_msg = "{0}".format(err_msg)[:ERROR_MSG_COL_SIZE]
finish_time = datetime.utcnow()
update = StatusRecord(et_status=status,
et_runtime=work_time_secs,
et_finished=finish_time,
error_message=err_msg,
updated_at=finish_time)
update_string = get_update_string(update.get_set_clause())
update.record['data_date'] = conditions['data_date']
update.record['table_versions'] = conditions['table_versions']
params_dict = update.minimized_record()
self.psql.run_sql(update_string,
database,
'update',
params=params_dict)
def test_status_record_minimized_record():
expected_value = {'data_date': "2014/12/01"}
sr = StatusRecord(data_date="2014/12/01")
assert expected_value == sr.minimized_record()
def test_status_record_minimized_record():
expected_value = {'data_date': "2014/12/01"}
sr = StatusRecord(data_date="2014/12/01")
assert expected_value == sr.minimized_record()
