def test_get_all_requests(self):
"""
Tests reading all requests
"""
exp_request_ids = [REQUEST_ID1, REQUEST_ID2, REQUEST_ID3, REQUEST_ID4, REQUEST_ID5,
REQUEST_ID6, REQUEST_ID7, REQUEST_ID8, REQUEST_ID9, REQUEST_ID10,
REQUEST_ID11]
qresult, exp_result = create_select_requests(exp_request_ids)
mock_ssm_get_parameter(1)
database.single_query = Mock(side_effect=[qresult])
expected = result_to_json(exp_result)
result = requests_db.get_all_requests()
database.single_query.assert_called_once()
self.assertEqual(expected, result)
mock_ssm_get_parameter(1)
err_msg = 'Database Error. could not connect to server'
database.single_query = Mock(side_effect=[DbError(
err_msg)])
try:
requests_db.get_all_requests()
self.fail("expected DatabaseError")
except requests_db.DatabaseError as err:
database.single_query.assert_called_once()
self.assertEqual(err_msg, str(err))
def query_requests(event):
"""
Queries the database for requests
"""
try:
granule_id = event['granule_id']
except KeyError:
granule_id = None
try:
request_group_id = event['request_group_id']
except KeyError:
request_group_id = None
try:
request_id = event['request_id']
except KeyError:
request_id = None
try:
object_key = event['object_key']
except KeyError:
object_key = None
if request_id:
result = requests_db.get_job_by_request_id(request_id)
else:
if request_group_id:
result = requests_db.get_jobs_by_request_group_id(request_group_id)
else:
if granule_id:
result = requests_db.get_jobs_by_granule_id(granule_id)
else:
if object_key:
result = requests_db.get_jobs_by_object_key(object_key)
else:
result = requests_db.get_all_requests()
return result
def test_get_all_requests(self):
"""
Tests reading all requests
"""
qresult = self.create_test_requests()
boto3.client = Mock()
mock_ssm_get_parameter(1)
expected = result_to_json(qresult)
result = requests_db.get_all_requests()
self.assertEqual(expected, result)
def print_rows(label):
"""
prints the rows of a list
"""
try:
develop_tests = os.environ['DEVELOP_TESTS']
except KeyError:
develop_tests = False
if develop_tests:
print("****REQUEST_IDS: 1) ", REQUEST_ID1, " 2) ", REQUEST_ID2, " 3) ",
REQUEST_ID3, " 4) ", REQUEST_ID4, " 5) ", REQUEST_ID5, " 6) ",
REQUEST_ID6, " 7) ", REQUEST_ID7, " 8) ", REQUEST_ID8, " 9) ",
REQUEST_ID9, " 10) ", REQUEST_ID10, " 11) ", REQUEST_ID11,
" 12) ", REQUEST_ID12)
print("****REQUEST_GROUP_EXP_IDs: 1) ", REQUEST_GROUP_ID_EXP_1, " 2) ",
REQUEST_GROUP_ID_EXP_2, " 3) ", REQUEST_GROUP_ID_EXP_3, " 4) ",
REQUEST_GROUP_ID_EXP_4, " 5) ", REQUEST_GROUP_ID_EXP_5, " 6) ",
REQUEST_GROUP_ID_EXP_6)
rows = requests_db.get_all_requests()
print("**** ", label)
for row in rows:
print(row)
print("****")
def test_task_one_granule_4_files_success(self):
"""
Test four files for one granule - successful
"""
granule_id = "MOD09GQ.A0219114.N5aUCG.006.0656338553321"
files = [KEY1, KEY2, KEY3, KEY4]
input_event = {
"input": {
"granules": [{
"granuleId": granule_id,
"keys": files
}]
},
"config": {
"glacier-bucket": "my-dr-fake-glacier-bucket"
}
}
requests_db.request_id_generator = Mock(side_effect=[
REQUEST_GROUP_ID_EXP_1, REQUEST_ID1, REQUEST_ID2, REQUEST_ID3,
REQUEST_ID4
])
boto3.client = Mock()
s3_cli = boto3.client('s3')
s3_cli.restore_object = Mock(side_effect=[None, None, None, None])
s3_cli.head_object = Mock()
CumulusLogger.info = Mock()
mock_ssm_get_parameter(5)
try:
result = request_files.task(input_event, self.context)
except requests_db.DatabaseError as err:
self.fail(str(err))
exp_gran = {}
exp_gran['granuleId'] = granule_id
exp_files = []
exp_file = {}
exp_file['key'] = FILE1
exp_file['dest_bucket'] = PROTECTED_BUCKET
exp_file['success'] = True
exp_file['err_msg'] = ''
exp_files.append(exp_file)
exp_file = {}
exp_file['key'] = FILE2
exp_file['dest_bucket'] = PROTECTED_BUCKET
exp_file['success'] = True
exp_file['err_msg'] = ''
exp_files.append(exp_file)
exp_file = {}
exp_file['key'] = FILE3
exp_file['dest_bucket'] = None
exp_file['success'] = True
exp_file['err_msg'] = ''
exp_files.append(exp_file)
exp_file = {}
exp_file['key'] = FILE4
exp_file['dest_bucket'] = PUBLIC_BUCKET
exp_file['success'] = True
exp_file['err_msg'] = ''
exp_files.append(exp_file)
exp_gran['files'] = exp_files
self.assertEqual(exp_gran, result)
end_rows = requests_db.get_all_requests()
self.assertEqual(4, len(end_rows))
for row in end_rows:
self.assertEqual("inprogress", row['job_status'])
def create_test_requests(self):
"""
creates jobs in the db
"""
boto3.client = Mock()
mock_ssm_get_parameter(7)
requests_db.get_utc_now_iso = Mock(side_effect=[
UTC_NOW_EXP_1,
UTC_NOW_EXP_4,
UTC_NOW_EXP_4,
UTC_NOW_EXP_4,
UTC_NOW_EXP_5,
UTC_NOW_EXP_5,
UTC_NOW_EXP_6,
UTC_NOW_EXP_6,
UTC_NOW_EXP_7,
UTC_NOW_EXP_7,
UTC_NOW_EXP_8,
UTC_NOW_EXP_8,
])
requests_db.request_id_generator = Mock(side_effect=[
REQUEST_ID1, REQUEST_ID2, REQUEST_ID3, REQUEST_ID4, REQUEST_ID5,
REQUEST_ID6
])
obj = {}
try:
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_1
obj["granule_id"] = "granule_1"
obj["key"] = "objectkey_1"
obj["glacier_bucket"] = "my_s3_bucket"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "complete", UTC_NOW_EXP_1,
UTC_NOW_EXP_4)
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_2
obj["granule_id"] = "granule_4"
obj["key"] = "objectkey_4"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "error", UTC_NOW_EXP_4, None,
"oh oh, an error happened")
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_3
obj["granule_id"] = "granule_5"
obj["key"] = "dr-glacier/MOD09GQ.A0219114.N5aUCG.006.0656338553321.txt"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "inprogress", UTC_NOW_EXP_5,
UTC_NOW_EXP_5)
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_3
obj["granule_id"] = "granule_6"
obj["key"] = "dr-glacier/MOD09GQ.A0219114.N5aUCG.006.0656338553321.hdf"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "inprogress", UTC_NOW_EXP_6,
UTC_NOW_EXP_6)
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_4
obj["granule_id"] = "granule_4"
obj["key"] = "objectkey_4"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "inprogress", UTC_NOW_EXP_7,
UTC_NOW_EXP_7)
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_5
obj["granule_id"] = "granule_1"
obj["key"] = "objectkey_1"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "inprogress", UTC_NOW_EXP_8,
UTC_NOW_EXP_8)
requests_db.submit_request(data)
results = requests_db.get_all_requests()
return results
except requests_db.DatabaseError as err:
self.fail(f"submit_request. {str(err)}")
def test_task_one_granule_4_files_success(self):
"""
Test four files for one granule - successful
"""
granule_id = "MOD09GQ.A0219114.N5aUCG.006.0656338553321"
files = [KEY1, KEY2, KEY3, KEY4]
input_event = {
"input": {
"granules": [{
"granuleId": granule_id,
"keys": files
}]
},
"config": {
"glacier-bucket": "my-dr-fake-glacier-bucket"
}
}
requests_db.request_id_generator = Mock(side_effect=[
REQUEST_GROUP_ID_EXP_1, REQUEST_ID1, REQUEST_ID2, REQUEST_ID3,
REQUEST_ID4
])
boto3.client = Mock()
s3_cli = boto3.client('s3')
s3_cli.restore_object = Mock(side_effect=[None, None, None, None])
s3_cli.head_object = Mock()
CumulusLogger.info = Mock()
mock_secretsmanager_get_parameter(5)
try:
result = request_files.task(input_event, self.context)
except requests_db.DatabaseError as err:
self.fail(str(err))
exp_gran = {
'granuleId':
granule_id,
'recover_files': [{
'key': FILE1,
'dest_bucket': PROTECTED_BUCKET,
'success': True,
'err_msg': ''
}, {
'key': FILE2,
'dest_bucket': PROTECTED_BUCKET,
'success': True,
'err_msg': ''
}, {
'key': FILE3,
'dest_bucket': None,
'success': True,
'err_msg': ''
}, {
'key': FILE4,
'dest_bucket': PUBLIC_BUCKET,
'success': True,
'err_msg': ''
}],
'keys': [{
'key': FILE1,
'dest_bucket': PROTECTED_BUCKET
}, {
'key': FILE2,
'dest_bucket': PROTECTED_BUCKET
}, {
'key': FILE3,
'dest_bucket': None
}, {
'key': FILE4,
'dest_bucket': PUBLIC_BUCKET
}]
}
exp_granules = {'granules': [exp_gran]}
self.assertEqual(exp_granules, result)
end_rows = requests_db.get_all_requests()
self.assertEqual(4, len(end_rows))
for row in end_rows:
self.assertEqual("inprogress", row['job_status'])
def create_test_requests(self): #pylint: disable-msg=too-many-statements
"""
creates jobs in the db
"""
boto3.client = Mock()
mock_ssm_get_parameter(12)
requests_db.get_utc_now_iso = Mock(side_effect=[
UTC_NOW_EXP_1, UTC_NOW_EXP_4, UTC_NOW_EXP_2, UTC_NOW_EXP_5,
UTC_NOW_EXP_3, UTC_NOW_EXP_6, UTC_NOW_EXP_4, UTC_NOW_EXP_4,
UTC_NOW_EXP_5, UTC_NOW_EXP_5, UTC_NOW_EXP_6, UTC_NOW_EXP_6,
UTC_NOW_EXP_7, UTC_NOW_EXP_7, UTC_NOW_EXP_8, UTC_NOW_EXP_8,
UTC_NOW_EXP_9, UTC_NOW_EXP_9, UTC_NOW_EXP_10, UTC_NOW_EXP_10,
UTC_NOW_EXP_11, UTC_NOW_EXP_11
])
requests_db.request_id_generator = Mock(side_effect=[
REQUEST_ID1, REQUEST_ID2, REQUEST_ID3, REQUEST_ID4, REQUEST_ID5,
REQUEST_ID6, REQUEST_ID7, REQUEST_ID8, REQUEST_ID9, REQUEST_ID10,
REQUEST_ID11
])
obj = {}
try:
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_1
obj["granule_id"] = "granule_1"
obj["key"] = "objectkey_1"
obj["glacier_bucket"] = "my_s3_bucket"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "complete", UTC_NOW_EXP_1,
UTC_NOW_EXP_4)
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_1
obj["granule_id"] = "granule_2"
obj["key"] = "objectkey_2"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "complete", UTC_NOW_EXP_2,
UTC_NOW_EXP_5)
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_1
obj["granule_id"] = "granule_3"
obj["key"] = "objectkey_3"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "complete", UTC_NOW_EXP_3,
UTC_NOW_EXP_6)
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_2
obj["granule_id"] = "granule_4"
obj["key"] = "objectkey_4"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "error", UTC_NOW_EXP_4, None,
"oh oh, an error happened")
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_3
obj["granule_id"] = "granule_5"
obj["key"] = "objectkey_5"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "inprogress", UTC_NOW_EXP_5,
UTC_NOW_EXP_5)
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_3
obj["granule_id"] = "granule_6"
obj["key"] = "objectkey_6"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "inprogress", UTC_NOW_EXP_6,
UTC_NOW_EXP_6)
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_4
obj["granule_id"] = "granule_4"
obj["key"] = "objectkey_4"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "inprogress", UTC_NOW_EXP_7,
UTC_NOW_EXP_7)
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_5
obj["granule_id"] = "granule_1"
obj["key"] = "objectkey_1"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "inprogress", UTC_NOW_EXP_8,
UTC_NOW_EXP_8)
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_5
obj["granule_id"] = "granule_2"
obj["key"] = "objectkey_2"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "inprogress", UTC_NOW_EXP_9,
UTC_NOW_EXP_9)
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_5
obj["granule_id"] = "granule_3"
obj["key"] = "objectkey_3"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "inprogress", UTC_NOW_EXP_10,
UTC_NOW_EXP_10)
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_6
obj["granule_id"] = "granule_7"
obj["key"] = "objectkey_7"
obj["glacier_bucket"] = None
obj["dest_bucket"] = None
data = create_data(obj, "regenerate", "inprogress", UTC_NOW_EXP_11,
UTC_NOW_EXP_11)
requests_db.submit_request(data)
results = requests_db.get_all_requests()
return results
except requests_db.DatabaseError as err:
self.fail(f"submit_request. {str(err)}")