def test_update_event_with_checksum_event(self, mock_format_and_send_notification):
checksum_id = str(uuid.uuid4())
area_uuid = self._create_area()
s3obj = self.mock_upload_file_to_s3(area_uuid, 'foo.json')
upload_area = UploadArea(area_uuid)
uploaded_file = UploadedFile(upload_area, s3object=s3obj)
checksum_event = ChecksumEvent(file_id=uploaded_file.db_id,
checksum_id=checksum_id,
job_id='12345',
status="SCHEDULED")
checksum_event.create_record()
checksum_event.status = "CHECKSUMMING"
response = update_event(checksum_event, uploaded_file.info(), self.client)
self.assertEqual(204, response.status_code)
record = UploadDB().get_pg_record("checksum", checksum_id)
self.assertEqual("CHECKSUMMING", record["status"])
self.assertEqual("<class 'datetime.datetime'>", str(type(record.get("checksum_started_at"))))
self.assertEqual(None, record["checksum_ended_at"])
checksum_event.status = "CHECKSUMMED"
response = update_event(checksum_event, uploaded_file.info(), self.client)
self.assertEqual(204, response.status_code)
record = UploadDB().get_pg_record("checksum", checksum_id)
self.assertEqual("CHECKSUMMED", record["status"])
self.assertEqual("<class 'datetime.datetime'>", str(type(record.get("checksum_started_at"))))
self.assertEqual("<class 'datetime.datetime'>", str(type(record.get("checksum_ended_at"))))
def test_update_event_with_validation_event(self, mock_format_and_send_notification):
validation_id = str(uuid.uuid4())
area_id = self._create_area()
s3obj = self.mock_upload_file_to_s3(area_id, 'foo.json')
upload_area = UploadArea(area_id)
uploaded_file = UploadedFile(upload_area, s3object=s3obj)
validation_event = ValidationEvent(file_ids=[uploaded_file.db_id],
validation_id=validation_id,
job_id='12345',
status="SCHEDULED")
validation_event.create_record()
validation_event.status = "VALIDATING"
response = update_event(validation_event, uploaded_file.info(), self.client)
self.assertEqual(204, response.status_code)
record = UploadDB().get_pg_record("validation", validation_id)
self.assertEqual("VALIDATING", record["status"])
self.assertEqual("<class 'datetime.datetime'>", str(type(record.get("validation_started_at"))))
self.assertEqual(None, record["validation_ended_at"])
self.assertEqual(None, record.get("results"))
validation_event.status = "VALIDATED"
response = update_event(validation_event, uploaded_file.info(), self.client)
self.assertEqual(204, response.status_code)
record = UploadDB().get_pg_record("validation", validation_id)
self.assertEqual("VALIDATED", record["status"])
self.assertEqual("<class 'datetime.datetime'>", str(type(record.get("validation_started_at"))))
self.assertEqual("<class 'datetime.datetime'>", str(type(record.get("validation_ended_at"))))
self.assertEqual(uploaded_file.info(), record.get("results"))
def test_validating_status_file_validation(self, mock_format_and_send_notification):
validation_id = str(uuid.uuid4())
orig_val_id = str(uuid.uuid4())
area_id = self._create_area()
s3obj = self.mock_upload_file_to_s3(area_id, 'foo.json')
upload_area = UploadArea(area_id)
uploaded_file = UploadedFile(upload_area, s3object=s3obj)
validation_event = ValidationEvent(file_ids=[uploaded_file.db_id],
validation_id=validation_id,
job_id='12345',
status="SCHEDULED",
docker_image="test_docker_image",
original_validation_id=orig_val_id)
validation_event.create_record()
data = {
"status": "VALIDATING",
"job_id": validation_event.job_id,
"payload": uploaded_file.info()
}
response = self.client.post(f"/v1/area/{area_id}/update_validation/{validation_id}",
headers=self.authentication_header,
data=json.dumps(data))
self.assertEqual(204, response.status_code)
record = UploadDB().get_pg_record("validation", validation_id)
self.assertEqual("test_docker_image", record["docker_image"])
self.assertEqual(validation_id, record["id"])
self.assertEqual(orig_val_id, record["original_validation_id"])
self.assertEqual("VALIDATING", record["status"])
self.assertEqual("<class 'datetime.datetime'>", str(type(record.get("validation_started_at"))))
self.assertEqual(None, record["validation_ended_at"])
self.assertEqual(None, record.get("results"))
response = self.client.get(f"/v1/area/{area_id}/foo.json/validate")
validation_status = response.get_json()['validation_status']
self.assertEqual(validation_status, "VALIDATING")
mock_format_and_send_notification.assert_not_called()
def test_validated_status_file_validation(self, mock_format_and_send_notification):
validation_id = str(uuid.uuid4())
area_id = self._create_area()
s3obj = self.mock_upload_file_to_s3(area_id, 'foo.json')
upload_area = UploadArea(area_id)
uploaded_file = UploadedFile(upload_area, s3object=s3obj)
validation_event = ValidationEvent(file_ids=[uploaded_file.db_id],
validation_id=validation_id,
job_id='12345',
status="SCHEDULED",
docker_image="test_docker_image")
validation_event.create_record()
data = {
"status": "VALIDATING",
"job_id": validation_event.job_id,
"payload": uploaded_file.info()
}
response = self.client.post(f"/v1/area/{area_id}/update_validation/{validation_id}",
headers=self.authentication_header,
data=json.dumps(data))
data = {
"status": "VALIDATED",
"job_id": validation_event.job_id,
"payload": uploaded_file.info()
}
response = self.client.post(f"/v1/area/{area_id}/update_validation/{validation_id}",
headers=self.authentication_header,
data=json.dumps(data))
self.assertEqual(204, response.status_code)
mock_format_and_send_notification.assert_called_once_with({
'upload_area_id': area_id,
'name': 'foo.json',
'size': 3,
'last_modified': s3obj.last_modified.isoformat(),
'content_type': "application/json",
'url': f"s3://{self.upload_config.bucket_name}/{area_id}/foo.json",
'checksums': {'s3_etag': '1', 'sha1': '2', 'sha256': '3', 'crc32c': '4'}
})
record = UploadDB().get_pg_record("validation", validation_id)
self.assertEqual("VALIDATED", record["status"])
self.assertEqual("test_docker_image", record["docker_image"])
self.assertEqual("<class 'datetime.datetime'>", str(type(record.get("validation_started_at"))))
self.assertEqual("<class 'datetime.datetime'>", str(type(record.get("validation_ended_at"))))
self.assertEqual(uploaded_file.info(), record.get("results"))
response = self.client.get(f"/v1/area/{area_id}/foo.json/validate")
validation_status = response.get_json()['validation_status']
self.assertEqual(validation_status, "VALIDATED")