def test_create_data(self):
"""
Tests the create_data function
"""
utc_now_exp = "2019-07-31 18:05:19.161362+00:00"
obj = {}
obj["request_group_id"] = "my_request_group_id"
obj["granule_id"] = "granule_1"
obj["glacier_bucket"] = "my_bucket"
obj["dest_bucket"] = "your_bucket"
obj["key"] = "my_file"
requests_db.request_id_generator = Mock(side_effect=[REQUEST_ID1])
exp_data = {}
exp_data["request_id"] = REQUEST_ID1
exp_data["request_group_id"] = obj["request_group_id"]
exp_data["granule_id"] = obj["granule_id"]
exp_data["object_key"] = obj["key"]
exp_data["job_type"] = "restore"
exp_data["restore_bucket_dest"] = obj["glacier_bucket"]
exp_data["archive_bucket_dest"] = obj["dest_bucket"]
exp_data["job_status"] = "inprogress"
exp_data["request_time"] = utc_now_exp
exp_data["last_update_time"] = utc_now_exp
data = requests_db.create_data(obj, exp_data["job_type"], exp_data["job_status"],
utc_now_exp, utc_now_exp)
self.assertEqual(exp_data, data)
def restore_object(s3_cli, obj, attempt, retries, retrieval_type='Standard'):
"""Restore an archived S3 Glacier object in an Amazon S3 bucket.
Args:
s3_cli (object): An instance of boto3 s3 client
obj (dict): A dictionary containing:
request_group_id (string): A uuid identifying all objects in
a granule restore request
granule_id (string): The granule_id to which the object_name being restored belongs
glacier_bucket (string): The S3 bucket name
key (string): The key of the Glacier object being restored
dest_bucket (string): The bucket the restored file will be moved
to after the restore completes
days (number): How many days the restored file will be accessible in the S3 bucket
before it expires
attempt (number): The attempt number for retry purposes
retries (number): The number of retries that will be attempted
retrieval_type (string, optional, default=Standard): Glacier Tier.
Valid values are 'Standard'|'Bulk'|'Expedited'.
Returns:
uuid: request_Id.
"""
data = requests_db.create_data(obj, "restore", "inprogress", None, None)
request_id = data["request_id"]
request = {'Days': obj["days"],
'GlacierJobParameters': {'Tier': retrieval_type}}
# Submit the request
try:
s3_cli.restore_object(Bucket=obj["glacier_bucket"],
Key=obj["key"],
RestoreRequest=request)
try:
requests_db.submit_request(data)
LOGGER.info(f"Job {request_id} created.")
except requests_db.DatabaseError as err:
LOGGER.error("Failed to log request in database. Error {}. Request: {}",
str(err), data)
except ClientError as c_err:
# NoSuchBucket, NoSuchKey, or InvalidObjectState error == the object's
# storage class was not GLACIER
LOGGER.error("{}. bucket: {} file: {}", c_err, obj["glacier_bucket"], obj["key"])
if attempt == retries:
try:
data["err_msg"] = str(c_err)
data["job_status"] = "error"
requests_db.submit_request(data)
LOGGER.info(f"Job {request_id} created.")
except requests_db.DatabaseError as err:
LOGGER.error("Failed to log request in database. Error {}. Request: {}",
str(err), data)
raise c_err
return request_id
def create_test_requests(self):
"""
creates jobs in the db
"""
boto3.client = Mock()
mock_ssm_get_parameter(7)
requests_db.get_utc_now_iso = Mock(side_effect=[
UTC_NOW_EXP_1,
UTC_NOW_EXP_4,
UTC_NOW_EXP_4,
UTC_NOW_EXP_4,
UTC_NOW_EXP_5,
UTC_NOW_EXP_5,
UTC_NOW_EXP_6,
UTC_NOW_EXP_6,
UTC_NOW_EXP_7,
UTC_NOW_EXP_7,
UTC_NOW_EXP_8,
UTC_NOW_EXP_8,
])
requests_db.request_id_generator = Mock(side_effect=[
REQUEST_ID1, REQUEST_ID2, REQUEST_ID3, REQUEST_ID4, REQUEST_ID5,
REQUEST_ID6
])
obj = {}
try:
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_1
obj["granule_id"] = "granule_1"
obj["key"] = "objectkey_1"
obj["glacier_bucket"] = "my_s3_bucket"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "complete", UTC_NOW_EXP_1,
UTC_NOW_EXP_4)
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_2
obj["granule_id"] = "granule_4"
obj["key"] = "objectkey_4"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "error", UTC_NOW_EXP_4, None,
"oh oh, an error happened")
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_3
obj["granule_id"] = "granule_5"
obj["key"] = "dr-glacier/MOD09GQ.A0219114.N5aUCG.006.0656338553321.txt"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "inprogress", UTC_NOW_EXP_5,
UTC_NOW_EXP_5)
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_3
obj["granule_id"] = "granule_6"
obj["key"] = "dr-glacier/MOD09GQ.A0219114.N5aUCG.006.0656338553321.hdf"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "inprogress", UTC_NOW_EXP_6,
UTC_NOW_EXP_6)
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_4
obj["granule_id"] = "granule_4"
obj["key"] = "objectkey_4"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "inprogress", UTC_NOW_EXP_7,
UTC_NOW_EXP_7)
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_5
obj["granule_id"] = "granule_1"
obj["key"] = "objectkey_1"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "inprogress", UTC_NOW_EXP_8,
UTC_NOW_EXP_8)
requests_db.submit_request(data)
results = requests_db.get_all_requests()
return results
except requests_db.DatabaseError as err:
self.fail(f"submit_request. {str(err)}")
def create_test_requests(self): #pylint: disable-msg=too-many-statements
"""
creates jobs in the db
"""
boto3.client = Mock()
mock_ssm_get_parameter(12)
requests_db.get_utc_now_iso = Mock(side_effect=[
UTC_NOW_EXP_1, UTC_NOW_EXP_4, UTC_NOW_EXP_2, UTC_NOW_EXP_5,
UTC_NOW_EXP_3, UTC_NOW_EXP_6, UTC_NOW_EXP_4, UTC_NOW_EXP_4,
UTC_NOW_EXP_5, UTC_NOW_EXP_5, UTC_NOW_EXP_6, UTC_NOW_EXP_6,
UTC_NOW_EXP_7, UTC_NOW_EXP_7, UTC_NOW_EXP_8, UTC_NOW_EXP_8,
UTC_NOW_EXP_9, UTC_NOW_EXP_9, UTC_NOW_EXP_10, UTC_NOW_EXP_10,
UTC_NOW_EXP_11, UTC_NOW_EXP_11
])
requests_db.request_id_generator = Mock(side_effect=[
REQUEST_ID1, REQUEST_ID2, REQUEST_ID3, REQUEST_ID4, REQUEST_ID5,
REQUEST_ID6, REQUEST_ID7, REQUEST_ID8, REQUEST_ID9, REQUEST_ID10,
REQUEST_ID11
])
obj = {}
try:
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_1
obj["granule_id"] = "granule_1"
obj["key"] = "objectkey_1"
obj["glacier_bucket"] = "my_s3_bucket"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "complete", UTC_NOW_EXP_1,
UTC_NOW_EXP_4)
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_1
obj["granule_id"] = "granule_2"
obj["key"] = "objectkey_2"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "complete", UTC_NOW_EXP_2,
UTC_NOW_EXP_5)
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_1
obj["granule_id"] = "granule_3"
obj["key"] = "objectkey_3"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "complete", UTC_NOW_EXP_3,
UTC_NOW_EXP_6)
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_2
obj["granule_id"] = "granule_4"
obj["key"] = "objectkey_4"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "error", UTC_NOW_EXP_4, None,
"oh oh, an error happened")
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_3
obj["granule_id"] = "granule_5"
obj["key"] = "objectkey_5"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "inprogress", UTC_NOW_EXP_5,
UTC_NOW_EXP_5)
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_3
obj["granule_id"] = "granule_6"
obj["key"] = "objectkey_6"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "inprogress", UTC_NOW_EXP_6,
UTC_NOW_EXP_6)
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_4
obj["granule_id"] = "granule_4"
obj["key"] = "objectkey_4"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "inprogress", UTC_NOW_EXP_7,
UTC_NOW_EXP_7)
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_5
obj["granule_id"] = "granule_1"
obj["key"] = "objectkey_1"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "inprogress", UTC_NOW_EXP_8,
UTC_NOW_EXP_8)
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_5
obj["granule_id"] = "granule_2"
obj["key"] = "objectkey_2"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "inprogress", UTC_NOW_EXP_9,
UTC_NOW_EXP_9)
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_5
obj["granule_id"] = "granule_3"
obj["key"] = "objectkey_3"
obj["dest_bucket"] = PROTECTED_BUCKET
data = create_data(obj, "restore", "inprogress", UTC_NOW_EXP_10,
UTC_NOW_EXP_10)
requests_db.submit_request(data)
obj["request_group_id"] = REQUEST_GROUP_ID_EXP_6
obj["granule_id"] = "granule_7"
obj["key"] = "objectkey_7"
obj["glacier_bucket"] = None
obj["dest_bucket"] = None
data = create_data(obj, "regenerate", "inprogress", UTC_NOW_EXP_11,
UTC_NOW_EXP_11)
requests_db.submit_request(data)
results = requests_db.get_all_requests()
return results
except requests_db.DatabaseError as err:
self.fail(f"submit_request. {str(err)}")