def setUp(self):
_start_time = time.time()
self.api_url = f"https://{os.environ['API_HOST']}/v1"
self.upload_config = UploadConfig()
self.auth_headers = {'Api-Key': self.upload_config.api_key}
self.deployment_stage = os.environ['DEPLOYMENT_STAGE']
self.upload_area_uuid = "deadbeef-dead-dead-dead-%012d" % random.randint(
0, 999999999999)
self.verbose = True
_end_time = time.time()
print(f"Total startup time: {_end_time - _start_time} seconds.")
def setUp(self):
self.test_start_time = time.time()
self.upload_config = UploadConfig()
self.upload_client = UploadService(
deployment_stage=os.environ['DEPLOYMENT_STAGE'],
api_token=self.upload_config.api_key)
self.upload_area_uuid = "deadbeef-dead-dead-dead-%012d" % random.randint(
0, 999999999999)
print("")
self._execute_create_upload_area()
print("\tstartup time: %0.02f seconds." %
(time.time() - self.test_start_time))
def test_call_without_auth_setup(self):
upload_config = UploadConfig()
upload_config.__class__._config = {}
client = client_for_test_api_server()
response = client.post(f"/v1/area/{str(uuid.uuid4())}",
headers={'Api-Key': 'foo'})
self.assertEqual(500, response.status_code)
self.assertIn("api_key is not in secrets",
response.data.decode('utf8'))
def setUp(self):
super().setUp()
# Environment
self.environment = {
'CSUM_DOCKER_IMAGE': 'bogoimage',
'API_HOST': 'bogohost'
}
self.environmentor = EnvironmentSetup(self.environment)
self.environmentor.enter()
# Setup authentication
self.authentication_header = {'Api-Key': UploadConfig().api_key}
# Setup app
self.client = client_for_test_api_server()
def setUp(self):
# Common Environment
if os.environ['DEPLOYMENT_STAGE'] in ['local', 'test']:
self.deployment_stage = os.environ['DEPLOYMENT_STAGE']
else:
self.deployment_stage = 'test'
self.environment = {'DEPLOYMENT_STAGE': self.deployment_stage}
self.common_environmentor = EnvironmentSetup(self.environment)
self.common_environmentor.enter()
# UploadConfig
self.upload_config = UploadConfig()
self.upload_config.set(self.__class__.BOGO_CONFIG)
# AuthConfig
self.upload_auth_config = UploadOutgoingIngestAuthConfig()
self.upload_auth_config.set(
self.__class__.UPLOAD_OUTGOING_INGEST_AUTH_CONFIG)
# UploadVersion
self.upload_version = UploadVersion()
self.upload_version.set({"upload_service_version": "0"})
def __init__(self):
self.env = os.environ['DEPLOYMENT_STAGE']
self.db = UploadDB()
logger.debug(
f"Running a health check for {self.env}. Results will be posted in #upload-service"
)
self.webhook = UploadConfig().slack_webhook
self.stale_checksum_job_count_query = "SELECT COUNT(*) FROM checksum " \
"WHERE status='CHECKSUMMING' " \
"AND created_at > CURRENT_DATE - interval '4 weeks' " \
"AND updated_at > CURRENT_TIMESTAMP - interval '2 hours'"
self.stale_validation_job_count_query = "SELECT COUNT(*) FROM validation " \
"WHERE status='VALIDATING' " \
"AND created_at > CURRENT_DATE - interval '4 weeks' " \
"AND updated_at > CURRENT_TIMESTAMP - interval '2 hours'"
self.scheduled_checksum_job_count_query = "SELECT COUNT(*) FROM checksum " \
"WHERE status='SCHEDULED' " \
"AND created_at > CURRENT_DATE - interval '4 weeks' " \
"AND updated_at > CURRENT_TIMESTAMP - interval '2 hours'"
self.scheduled_validation_job_count_query = "SELECT COUNT(*) FROM validation " \
"WHERE status='SCHEDULED' " \
"AND created_at > CURRENT_DATE - interval '4 weeks' " \
"AND updated_at > CURRENT_TIMESTAMP - interval '2 hours'"
self.undeleted_areas_count_query = "SELECT COUNT(*) FROM upload_area " \
"WHERE created_at > CURRENT_DATE - interval '4 weeks' " \
"AND status != 'DELETED'"
self.failed_checksum_count_query = "SELECT COUNT(*) FROM checksum " \
"WHERE status='FAILED' " \
"AND updated_at >= NOW() - '1 day'::INTERVAL"
self.failed_validation_count_query = "SELECT COUNT(*) FROM validation " \
"WHERE status='FAILED' " \
"AND updated_at >= NOW() - '1 day'::INTERVAL"
self.deadletter_metric_queries = [{
'Id': 'visible_messages',
'MetricStat': {
'Metric': {
'Namespace':
'AWS/SQS',
'MetricName':
'ApproximateNumberOfMessagesVisible',
'Dimensions': [{
'Name':
'QueueName',
'Value':
f'dcp-upload-pre-csum-deadletter-queue-{self.env}'
}]
},
'Period': 90000,
'Stat': 'Average'
}
}, {
'Id': 'received_messages',
'MetricStat': {
'Metric': {
'Namespace':
'AWS/SQS',
'MetricName':
'NumberOfMessagesReceived',
'Dimensions': [{
'Name':
'QueueName',
'Value':
f'dcp-upload-pre-csum-deadletter-queue-{self.env}'
}]
},
'Period': 90000,
'Stat': 'Average'
}
}]
self.lambda_error_queries = [{
'Id': 'upload_api_lambda_errors',
'MetricStat': {
'Metric': {
'Namespace':
'AWS/Lambda',
'MetricName':
'Errors',
'Dimensions': [{
'Name': 'FunctionName',
'Value': f'upload-api-{self.env}'
}]
},
'Period': 90000,
'Stat': 'Sum'
}
}, {
'Id': 'checksum_daemon_lambda_errors',
'MetricStat': {
'Metric': {
'Namespace':
'AWS/Lambda',
'MetricName':
'Errors',
'Dimensions': [{
'Name': 'FunctionName',
'Value': f'dcp-upload-csum-{self.env}'
}]
},
'Period': 90000,
'Stat': 'Sum'
}
}]
class UploadTestCase(unittest.TestCase):
BOGO_CONFIG = {
'bucket_name': 'bogobucket',
'csum_job_q_arn': 'bogo_arn',
'csum_upload_q_url': 'bogo_url',
'area_deletion_q_url': 'delete_sqs_url',
'csum_job_role_arn': 'bogo_role_arn',
'upload_submitter_role_arn': 'bogo_submitter_role_arn',
'slack_webhook': 'bogo_slack_url',
'area_deletion_lambda_name': 'delete_lambda_name',
'staging_bucket_arn': 'staging_bucket_arn',
'ingest_api_host': 'test_ingest_api_host',
'validation_q_url': 'test_validation_q_url'
}
TEST_SERVICE_CREDS = {
'client_email': 'test_client_email',
'private_key_id': 'test_private_key_id',
'private_key': 'test_private_key'
}
UPLOAD_OUTGOING_INGEST_AUTH_CONFIG = {
'gcp_service_acct_creds':
base64.b64encode(json.dumps(TEST_SERVICE_CREDS).encode()),
'dcp_auth0_audience':
'test_dcp_auth0_audience'
}
def setUp(self):
# Common Environment
if os.environ['DEPLOYMENT_STAGE'] in ['local', 'test']:
self.deployment_stage = os.environ['DEPLOYMENT_STAGE']
else:
self.deployment_stage = 'test'
self.environment = {'DEPLOYMENT_STAGE': self.deployment_stage}
self.common_environmentor = EnvironmentSetup(self.environment)
self.common_environmentor.enter()
# UploadConfig
self.upload_config = UploadConfig()
self.upload_config.set(self.__class__.BOGO_CONFIG)
# AuthConfig
self.upload_auth_config = UploadOutgoingIngestAuthConfig()
self.upload_auth_config.set(
self.__class__.UPLOAD_OUTGOING_INGEST_AUTH_CONFIG)
# UploadVersion
self.upload_version = UploadVersion()
self.upload_version.set({"upload_service_version": "0"})
def tearDown(self):
self.common_environmentor.exit()
def create_upload_area(self,
area_uuid=None,
status='UNLOCKED',
db_session=None):
area_uuid = area_uuid or str(uuid.uuid4())
db_session = db_session or DBSessionMaker().session()
db_area = DbUploadArea(uuid=area_uuid,
bucket_name=self.upload_config.bucket_name,
status=status)
db_session.add(db_area)
db_session.commit()
return db_area