def test_set_feedback(self):
filename = "c37a3efa-593c-4bab-b49c-bee0613c4fb2"
expected = "009 feedback response for period 2019 sample unit 49900108249D"
meta_data = MetaWrapper(filename)
meta_data.set_feedback(self.test_survey, self.test_bytes)
actual = meta_data.get_description()
self.assertEqual(expected, actual)
self.assertEqual(f'{filename}:ftp', meta_data.filename)
def test_set_comments(self):
filename = "c37a3efa-593c-4bab-b49c-bee0613c4fb2"
expected = "Comments.zip"
meta_data = MetaWrapper(filename)
meta_data.set_comments(self.test_bytes)
actual = meta_data.get_description()
self.assertEqual(expected, actual)
self.assertEqual(f'{filename}:ftp', meta_data.filename)
def test_deliver_seft(self, mock_send_message, mock_write_to_bucket):
path = "dap/"
filename = "9010576d-f3df-4011-aa41-adecd9bee011"
meta_data = MetaWrapper(filename)
meta_data.output_type = OutputType.SEFT
mock_write_to_bucket.return_value = path
deliver(meta_data, b"bytes")
mock_send_message.assert_called_with(meta_data, path)
def test_set_seft(self):
filename = "c37a3efa-593c-4bab-b49c-bee0613c4fb2"
expected = "057 seft response for period 202009 sample unit 20210121143526"
meta_data = MetaWrapper(filename)
meta_data.set_seft(self.test_seft)
actual = meta_data.get_description()
self.assertEqual(expected, actual)
self.assertEqual(f'{filename}:ftp', meta_data.filename)
def test_set_feedback(self, mock_time):
mock_time.return_value = 1629452867.587326
filename = "c37a3efa-593c-4bab-b49c-bee0613c4fb2"
expected = "009 feedback response for period 2019 sample unit 49900108249D"
meta_data = MetaWrapper(filename)
meta_data.set_feedback(self.test_survey)
actual = meta_data.get_description()
self.assertEqual(expected, actual)
self.assertEqual(f'{filename}-fb-1629452867:ftp', meta_data.filename)
def test_set_seft(self):
filename = "11110000004H_202103_074_20210622072550.xlsx.gpg"
expected = "057 seft response for period 202009 sample unit 20210121143526"
meta_data = MetaWrapper(filename)
meta_data.set_seft(self.test_seft)
actual = meta_data.get_description()
self.assertEqual(expected, actual)
self.assertEqual('11110000004H_202103_074_20210622072550.xlsx.gpg:ftp',
meta_data.filename)
def deliver_comments():
"""
Endpoint for delivering daily comment report. POST request requires the zipped up comments to be uploaded as
"zip", and the filename passed in the query parameters.
"""
logger.info('Processing Comments submission')
filename = request.args.get("filename")
meta = MetaWrapper(filename)
files = request.files
data_bytes = files[ZIP_FILE].read()
meta.set_comments()
return process(meta, data_bytes)
def deliver_feedback():
"""
Endpoint for feedback submissions only. POST request requires the feedback JSON to be uploaded as
"submission", and the filename passed in the query parameters.
"""
logger.info('Processing Feedback submission')
filename = request.args.get("filename")
meta = MetaWrapper(filename)
files = request.files
submission_bytes = files[SUBMISSION_FILE].read()
survey_dict = json.loads(submission_bytes.decode())
data_bytes = submission_bytes
meta.set_feedback(survey_dict)
return process(meta, data_bytes)
def deliver_hybrid():
"""
Endpoint for submissions intended for dap and legacy systems. POST request requires the submission JSON to be
uploaded as "submission", the zipped transformed artifact as "transformed", and the filename passed in the
query parameters.
"""
logger.info('Processing Hybrid submission')
filename = request.args.get("filename")
meta = MetaWrapper(filename)
files = request.files
submission_bytes = files[SUBMISSION_FILE].read()
survey_dict = json.loads(submission_bytes.decode())
data_bytes = files[TRANSFORMED_FILE].read()
meta.set_hybrid(survey_dict)
return process(meta, data_bytes)
def deliver_seft():
"""
Endpoint for delivering SEFT submissions. POST request requires the encrypted SEFT to be uploaded as
"seft", metadata JSON as "metadata", and the filename passed in the query parameters.
Metadata file is required to provide information about the submissions to construct the PubSub message.
"""
logger.info('Processing SEFT submission')
filename = request.args.get("filename")
meta = MetaWrapper(filename)
files = request.files
meta_bytes = files[METADATA_FILE].read()
meta_dict = json.loads(meta_bytes.decode())
data_bytes = files[SEFT_FILE].read()
meta.set_seft(meta_dict)
return process(meta, data_bytes)
def setUp(self):
self.meta_data = MetaWrapper('test_file_name')
self.meta_data.output_type = None
self.meta_data.survey_id = "023"
self.meta_data.period = "0216"
self.meta_data.ru_ref = "12345"
self.meta_data.sizeBytes = len(b"bytes")
self.meta_data.md5sum = hashlib.md5(b"bytes").hexdigest()
self.expected = {
'version':
'1',
'files': [{
'name': self.meta_data.filename,
'sizeBytes': self.meta_data.sizeBytes,
'md5sum': self.meta_data.md5sum
}],
'sensitivity':
'High',
'sourceName':
CONFIG.PROJECT_ID,
'manifestCreated':
'',
'description':
'',
'dataset':
self.meta_data.survey_id,
'schemaversion':
'1',
'iterationL1':
self.meta_data.period
}
def test_deliver(self, mock_send_message, mock_write_to_bucket,
mock_encrypt_output):
encrypted_message = "This has been encrypted"
path = "dap/"
filename = "9010576d-f3df-4011-aa41-adecd9bee011"
meta_data = MetaWrapper(filename)
mock_encrypt_output.return_value = encrypted_message
mock_write_to_bucket.return_value = path
deliver(meta_data, b"bytes")
mock_send_message.assert_called_with(meta_data, path)
def deliver(meta_data: MetaWrapper, data_bytes: bytes):
"""
Encrypts any unencrypted data, writes to the appropriate location within the outputs GCP bucket and notifies DAP
via PubSub
"""
if meta_data.output_type == OutputType.SEFT:
encrypted_output = data_bytes
else:
logger.info("Encrypting output")
encrypted_output = encrypt_output(data_bytes)
encrypted_bytes = encrypted_output.encode()
meta_data.md5sum = hashlib.md5(encrypted_bytes).hexdigest()
meta_data.sizeBytes = len(encrypted_bytes)
logger.info("Storing to bucket")
path = write_to_bucket(encrypted_output,
filename=meta_data.filename,
output_type=meta_data.output_type)
logger.info("Sending DAP notification")
send_message(meta_data, path)
def create_message_data(meta_data: MetaWrapper) -> str:
"""
Generates PubSub message using MetaWrapper
"""
if meta_data.output_type == OutputType.COMMENTS:
dataset = "sdx_comments"
iteration1 = None
else:
dataset = meta_data.survey_id
iteration1 = meta_data.period
message_data = {
'version':
'1',
'files': [{
'name': meta_data.filename,
'sizeBytes': meta_data.sizeBytes,
'md5sum': meta_data.md5sum
}],
'sensitivity':
CONFIG.DATA_SENSITIVITY,
'sourceName':
CONFIG.PROJECT_ID,
'manifestCreated':
get_formatted_current_utc(),
'description':
meta_data.get_description(),
'dataset':
dataset,
'schemaversion':
'1'
}
if iteration1 is not None:
message_data['iterationL1'] = iteration1
logger.info("Created pubsub message")
str_dap_message = json.dumps(message_data)
return str_dap_message
