def update_status_in_db(a_file: Dict[str, Any], attempt: int, err_msg: str) -> None:
"""
Updates the status for the job in the database.
Args:
a_file: An input dict with keys for:
'request_id' (string): The request_id of the database entry to update.
'source_key' (string): The filename of the restored file.
'source_bucket' (string): The location of the restored file.
'target_bucket' (string): the archive bucket the file was copied to.
attempt: The attempt number for the copy (1 based).
err_msg: None, or the error message from the copy command.
Returns:
None: {a_file} will be modified with additional keys for:
'success' (boolean): True if the copy was successful,
otherwise False.
'err_msg' (string): When 'success' is False, this will contain
the error message from the copy error
Example: [{'request_id': '', 'source_key': 'file1.xml',
'source_bucket': 'my-dr-fake-glacier-bucket',
'target_bucket': 'unittest_xml_bucket', 'success': True,
'err_msg': ''},
{'request_id': '', 'source_key': 'file2.txt',
'source_bucket': 'my-dr-fake-glacier-bucket',
'target_bucket': 'unittest_txt_bucket', 'success': True,
'err_msg': ''}]
Raises:
requests_db.DatabaseError: Thrown if the update operation fails.
"""
new_status = ""
try:
if err_msg:
a_file[FILE_ERROR_MESSAGE_KEY] = err_msg
new_status = "error"
logging.error(f"Attempt {attempt}. Error copying file {a_file[FILE_SOURCE_KEY_KEY]}"
f" from {a_file[FILE_SOURCE_BUCKET_KEY]} to {a_file[FILE_TARGET_BUCKET_KEY]}."
f" msg: {err_msg}")
requests_db.update_request_status_for_job(a_file[FILE_REQUEST_ID_KEY], new_status, err_msg)
else:
a_file[FILE_SUCCESS_KEY] = True
a_file[FILE_ERROR_MESSAGE_KEY] = ''
new_status = "complete"
logging.info(f"Attempt {attempt}. Success copying file "
f"{a_file[FILE_SOURCE_KEY_KEY]} from {a_file[FILE_SOURCE_BUCKET_KEY]} "
f"to {a_file[FILE_TARGET_BUCKET_KEY]}.")
requests_db.update_request_status_for_job(a_file[FILE_REQUEST_ID_KEY], new_status)
except requests_db.DatabaseError as err:
logging.error(f"Failed to update request status in database. "
f"key: {a_file[FILE_SOURCE_KEY_KEY]} "
f"new status: {new_status}. Err: {str(err)}")
raise
return
def update_status_in_db(afile, attempt, err_msg):
"""
Updates the status for the job in the database.
Args:
afile: An input dict with keys for:
'request_id' (string): The request_id of the database entry to update.
'source_key' (string): The filename of the restored file
'source_bucket' (string): The location of the restored file
'target_bucket' (string): the archive bucket the file was copied to.
attempt (number): The attempt number for the copy
err_msg (string): None, or the error message from the copy command
Returns:
afile: The input dict with additional keys for:
'success' (boolean): True, if the copy was successful,
otherwise False.
'err_msg' (string): when success is False, this will contain
the error message from the copy error
Example: [{'request_id': '', 'source_key': 'file1.xml',
'source_bucket': 'my-dr-fake-glacier-bucket',
'target_bucket': 'unittest_xml_bucket', 'success': True,
'err_msg': ''},
{'request_id': '', 'source_key': 'file2.txt',
'source_bucket': 'my-dr-fake-glacier-bucket',
'target_bucket': 'unittest_txt_bucket', 'success': True,
'err_msg': ''}]
"""
old_status = ""
new_status = ""
try:
if err_msg:
afile['err_msg'] = err_msg
new_status = "error"
logging.error(f"Attempt {attempt}. Error copying file {afile['source_key']}"
f" from {afile['source_bucket']} to {afile['target_bucket']}."
f" msg: {err_msg}")
requests_db.update_request_status_for_job(afile['request_id'], new_status, err_msg)
else:
afile['success'] = True
afile['err_msg'] = ''
new_status = "complete"
logging.info(f"Attempt {attempt}. Success copying file "
f"{afile['source_key']} from {afile['source_bucket']} "
f"to {afile['target_bucket']}.")
requests_db.update_request_status_for_job(afile['request_id'], new_status)
except requests_db.DatabaseError as err:
logging.error(f"Failed to update request status in database. "
f"key: {afile['source_key']} old status: {old_status} "
f"new status: {new_status}. Err: {str(err)}")
raise
return afile
def test_update_request_status_missing_key(self):
"""
Tests updating a job where the object_key isn't given
"""
utc_now_exp = "2019-07-31 19:21:38.263364+00:00"
requests_db.get_utc_now_iso = Mock(return_value=utc_now_exp)
job_status = "invalid"
exp_msg = "No object_key provided"
database.single_query = Mock(side_effect=[requests_db.BadRequestError(exp_msg)])
mock_ssm_get_parameter(1)
try:
requests_db.update_request_status_for_job(REQUEST_ID1, job_status)
self.fail("expected requests_db.BadRequestError")
except requests_db.BadRequestError as err:
self.assertEqual(exp_msg, str(err))
def test_update_request_status_for_job_error(self):
"""
Tests updating an inprogress job to an 'error' status
"""
self.create_test_requests()
boto3.client = Mock()
mock_ssm_get_parameter(3)
request_id = REQUEST_ID8
row = requests_db.get_job_by_request_id(request_id)
self.assertEqual("inprogress", row[0]["job_status"])
print_rows("begin")
utc_now_exp = "2019-07-31 21:07:15.234362+00:00"
requests_db.get_utc_now_iso = Mock(return_value=utc_now_exp)
job_status = "error"
err_msg = "oh no an error"
try:
result = requests_db.update_request_status_for_job(
request_id, job_status, err_msg)
print_rows("end")
self.assertEqual([], result)
row = requests_db.get_job_by_request_id(request_id)
self.assertEqual(job_status, row[0]["job_status"])
self.assertEqual(err_msg, row[0]["err_msg"])
self.assertIn(utc_now_exp, row[0]["last_update_time"])
except requests_db.DatabaseError as err:
self.fail(f"update_request_status_for_job. {str(err)}")
def test_update_request_status_exception(self):
"""
Tests updating a job to an invalid status
"""
utc_now_exp = "2019-07-31 19:21:38.263364+00:00"
requests_db.get_utc_now_iso = Mock(return_value=utc_now_exp)
job_status = "invalid"
exp_msg = ('Database Error. new row for relation "request_status" violates '
'check constraint "request_status_job_status_check"')
database.single_query = Mock(side_effect=[requests_db.DatabaseError(exp_msg)])
mock_ssm_get_parameter(1)
try:
requests_db.update_request_status_for_job(REQUEST_ID6, job_status)
self.fail("expected DatabaseError")
except requests_db.DatabaseError as err:
self.assertIn(exp_msg, str(err))
database.single_query.assert_called_once()
def test_update_request_status_for_job_exceptions(self):
"""
Tests updating a job to an 'inprogress' status
"""
utc_now_exp = "2019-07-31 21:07:15.234362+00:00"
requests_db.get_utc_now_iso = Mock(return_value=utc_now_exp)
request_id = REQUEST_ID3
job_status = "inprogress"
exp_err = 'A new status must be provided'
try:
requests_db.update_request_status_for_job(request_id, None)
self.fail("expected BadRequestError")
except requests_db.BadRequestError as err:
self.assertEqual(exp_err, str(err))
exp_err = 'No request_id provided'
try:
requests_db.update_request_status_for_job(None, job_status)
self.fail("expected BadRequestError")
except requests_db.BadRequestError as err:
self.assertEqual(exp_err, str(err))
exp_err = 'Database Error. Internal database error, please contact LP DAAC User Services'
database.single_query = Mock(side_effect=[DbError(exp_err)])
mock_ssm_get_parameter(1)
try:
requests_db.update_request_status_for_job(request_id, job_status)
self.fail("expected DatabaseError")
except requests_db.DatabaseError as err:
self.assertEqual(exp_err, str(err))
database.single_query.assert_called_once()
def test_update_request_status_missing_status(self):
"""
Tests updating a job where the status isn't given
"""
utc_now_exp = "2019-07-31 19:21:38.263364+00:00"
requests_db.get_utc_now_iso = Mock(return_value=utc_now_exp)
request_id = None
job_status = None
exp_msg = "No request_id provided"
try:
result = requests_db.update_request_status_for_job(request_id, job_status)
self.assertEqual([], result)
except requests_db.BadRequestError as err:
self.assertEqual(exp_msg, str(err))
def test_update_request_status_complete(self):
"""
Tests updating a job to a 'complete' status
"""
self.create_test_requests()
boto3.client = Mock()
mock_ssm_get_parameter(1)
utc_now_exp = "2019-07-31 21:07:15.234362+00:00"
requests_db.get_utc_now_iso = Mock(return_value=utc_now_exp)
job_status = "complete"
try:
result = requests_db.update_request_status_for_job(
REQUEST_ID1, job_status)
self.assertEqual([], result)
except requests_db.DatabaseError as err:
self.fail(f"update_request_status_for_job. {str(err)}")
def test_update_request_status_notfound(self):
"""
Tests updating a job where the object_key doesn't exist
"""
utc_now_exp = "2019-07-31 19:21:38.263364+00:00"
requests_db.get_utc_now_iso = Mock(return_value=utc_now_exp)
request_id = "notexists"
job_status = "invalid"
exp_result = []
database.single_query = Mock(side_effect=[exp_result])
mock_ssm_get_parameter(1)
try:
result = requests_db.update_request_status_for_job(request_id, job_status)
self.assertEqual([], result)
database.single_query.assert_called_once()
except requests_db.DatabaseError as err:
self.fail(f"update_request_status. {str(err)}")
def test_update_request_status_complete(self):
"""
Tests updating a job to a 'complete' status
"""
utc_now_exp = "2019-07-31 21:07:15.234362+00:00"
requests_db.get_utc_now_iso = Mock(return_value=utc_now_exp)
request_id = "thisisarequestid"
job_status = "complete"
exp_result = []
database.single_query = Mock(side_effect=[exp_result])
mock_ssm_get_parameter(1)
try:
result = requests_db.update_request_status_for_job(request_id, job_status)
self.assertEqual([], result)
database.single_query.assert_called_once()
except requests_db.DatabaseError as err:
self.fail(f"update_request_status. {str(err)}")
def test_update_request_status_for_job(self):
"""
Tests updating a job to an 'inprogress' status
"""
utc_now_exp = "2019-07-31 21:07:15.234362+00:00"
requests_db.get_utc_now_iso = Mock(return_value=utc_now_exp)
request_id = REQUEST_ID3
job_status = "inprogress"
exp_result = []
database.single_query = Mock(side_effect=[exp_result])
mock_secretsmanager_get_parameter(1)
try:
result = requests_db.update_request_status_for_job(
request_id, job_status)
self.assertEqual([], result)
database.single_query.assert_called_once()
except requests_db.DatabaseError as err:
self.fail(f"update_request_status_for_job. {str(err)}")
def test_update_request_status_error(self):
"""
Tests updating a job to a 'error' status
"""
self.create_test_requests()
boto3.client = Mock()
mock_ssm_get_parameter(2)
utc_now_exp = "2019-07-31 19:21:38.263364+00:00"
requests_db.get_utc_now_iso = Mock(return_value=utc_now_exp)
granule_id = "granule_5"
job_status = "error"
err_msg = "copy error msg goes here"
try:
result = requests_db.update_request_status_for_job(
REQUEST_ID5, job_status, err_msg)
self.assertEqual([], result)
except requests_db.DatabaseError as err:
self.fail(f"update_request_status_for_job. {str(err)}")
result = requests_db.get_jobs_by_granule_id(granule_id)
self.assertEqual(err_msg, result[0]["err_msg"])
def test_update_request_status_error(self):
"""
Tests updating a job to an 'error' status
"""
_, exp_result = create_select_requests([REQUEST_ID4])
utc_now_exp = "2019-07-31 19:21:38.263364+00:00"
requests_db.get_utc_now_iso = Mock(return_value=utc_now_exp)
granule_id = "granule_4"
job_status = "error"
err_msg = "Error message goes here"
empty_result = []
database.single_query = Mock(side_effect=[empty_result, exp_result])
mock_ssm_get_parameter(2)
try:
result = requests_db.update_request_status_for_job(REQUEST_ID4, job_status, err_msg)
self.assertEqual([], result)
database.single_query.assert_called_once()
except requests_db.DatabaseError as err:
self.fail(f"update_request_status. {str(err)}")
result = requests_db.get_jobs_by_granule_id(granule_id)
self.assertEqual(err_msg, result[0]["err_msg"])
def test_update_request_status_for_job_inprogress(self):
"""
Tests updating an 'error' job to an 'inprogress' status
"""
self.create_test_requests()
boto3.client = Mock()
mock_ssm_get_parameter(2)
print_rows("begin")
utc_now_exp = requests_db.get_utc_now_iso()
requests_db.get_utc_now_iso = Mock(return_value=utc_now_exp)
request_id = REQUEST_ID4
job_status = "inprogress"
try:
result = requests_db.update_request_status_for_job(
request_id, job_status)
print_rows("end")
self.assertEqual([], result)
row = requests_db.get_job_by_request_id(request_id)
self.assertEqual(job_status, row[0]["job_status"])
self.assertEqual(None, row[0]["err_msg"])
except requests_db.DatabaseError as err:
self.fail(f"update_request_status_for_job. {str(err)}")