def test__get_tag_given_no_data_nor_filename__returns_warning(self):
with self.assertLogs(
logger=get_logger(logger_name)) as context_manager:
_checksum_handler = ClientSideChecksumHandler()
_checksums = _checksum_handler.get_checksum_metadata_tag()
self.assertIn("no data was provided", str(context_manager.output))
self.assertIn("No checksums have been computed",
str(context_manager.output))
self.assertEquals(_checksums, {})
def test__get_tag_given_both_data_and_filename__returns_warning(self):
_test_file = FixtureFile.factory("small_file")
with self.assertLogs(
logger=get_logger(logger_name)) as context_manager:
_checksum_handler = ClientSideChecksumHandler(
data=_test_file.contents,
filename=FixtureFile.fixture_file_path(_test_file.name))
_checksums = _checksum_handler.get_checksum_metadata_tag()
self.assertIn("both a file and raw data was provided",
str(context_manager.output))
self.assertIn("No checksums have been computed",
str(context_manager.output))
self.assertEquals(_checksums, {})
def test__get_tag_given_s3_file__returns_warning(self):
_test_file = FixtureFile.factory("10241MB_file")
with self.assertLogs(
logger=get_logger(logger_name)) as context_manager:
_checksum_handler = ClientSideChecksumHandler(
filename=_test_file.url)
_checksums = _checksum_handler.get_checksum_metadata_tag()
self.assertIn(
"Did not perform client-side checksumming for file in S3",
str(context_manager.output))
self.assertIn("No checksums have been computed",
str(context_manager.output))
self.assertEquals(_checksums, {})
from datetime import datetime, timedelta
import json
import os
import boto3
import requests
from upload.common.database import UploadDB
from upload.common.logging import get_logger
from upload.common.upload_config import UploadConfig
logger = get_logger(__name__)
client = boto3.client('cloudwatch')
class HealthCheck:
def __init__(self):
self.env = os.environ['DEPLOYMENT_STAGE']
self.db = UploadDB()
logger.debug(
f"Running a health check for {self.env}. Results will be posted in #upload-service"
)
self.webhook = UploadConfig().slack_webhook
self.stale_checksum_job_count_query = "SELECT COUNT(*) FROM checksum " \
"WHERE status='CHECKSUMMING' " \
"AND created_at > CURRENT_DATE - interval '4 weeks' " \
"AND updated_at > CURRENT_TIMESTAMP - interval '2 hours'"
self.stale_validation_job_count_query = "SELECT COUNT(*) FROM validation " \
"WHERE status='VALIDATING' " \
#!/usr/bin/env python
import argparse
import os
import sys
from urllib3.util import parse_url
import boto3
from upload.common.logging import get_logger
from upload.common.dss_checksums import DssChecksums
from upload.common.checksum_event import ChecksumEvent
from upload.common.upload_api_client import update_event
from upload.common.upload_config import UploadConfig
logger = get_logger(f"CHECKSUMMER [{os.environ.get('AWS_BATCH_JOB_ID')}]")
class Checksummer:
def __init__(self, argv):
self.bucket_name = None
self.s3_object_key = None
self.upload_area_id = None
self.file_name = None
UploadConfig.use_env = True # AWS Secrets are not available to batch jobs, use environment
self._parse_args(argv)
s3 = boto3.resource('s3')
s3obj = s3.Bucket(self.bucket_name).Object(self.s3_object_key)
self.checksums = DssChecksums(s3obj)
self.checksum_event = ChecksumEvent(
