def test__get_tag_given_no_data_nor_filename__returns_warning(self): with self.assertLogs( logger=get_logger(logger_name)) as context_manager: _checksum_handler = ClientSideChecksumHandler() _checksums = _checksum_handler.get_checksum_metadata_tag() self.assertIn("no data was provided", str(context_manager.output)) self.assertIn("No checksums have been computed", str(context_manager.output)) self.assertEquals(_checksums, {})
def test__get_tag_given_both_data_and_filename__returns_warning(self): _test_file = FixtureFile.factory("small_file") with self.assertLogs( logger=get_logger(logger_name)) as context_manager: _checksum_handler = ClientSideChecksumHandler( data=_test_file.contents, filename=FixtureFile.fixture_file_path(_test_file.name)) _checksums = _checksum_handler.get_checksum_metadata_tag() self.assertIn("both a file and raw data was provided", str(context_manager.output)) self.assertIn("No checksums have been computed", str(context_manager.output)) self.assertEquals(_checksums, {})
def test__get_tag_given_s3_file__returns_warning(self): _test_file = FixtureFile.factory("10241MB_file") with self.assertLogs( logger=get_logger(logger_name)) as context_manager: _checksum_handler = ClientSideChecksumHandler( filename=_test_file.url) _checksums = _checksum_handler.get_checksum_metadata_tag() self.assertIn( "Did not perform client-side checksumming for file in S3", str(context_manager.output)) self.assertIn("No checksums have been computed", str(context_manager.output)) self.assertEquals(_checksums, {})
from datetime import datetime, timedelta import json import os import boto3 import requests from upload.common.database import UploadDB from upload.common.logging import get_logger from upload.common.upload_config import UploadConfig logger = get_logger(__name__) client = boto3.client('cloudwatch') class HealthCheck: def __init__(self): self.env = os.environ['DEPLOYMENT_STAGE'] self.db = UploadDB() logger.debug( f"Running a health check for {self.env}. Results will be posted in #upload-service" ) self.webhook = UploadConfig().slack_webhook self.stale_checksum_job_count_query = "SELECT COUNT(*) FROM checksum " \ "WHERE status='CHECKSUMMING' " \ "AND created_at > CURRENT_DATE - interval '4 weeks' " \ "AND updated_at > CURRENT_TIMESTAMP - interval '2 hours'" self.stale_validation_job_count_query = "SELECT COUNT(*) FROM validation " \ "WHERE status='VALIDATING' " \
#!/usr/bin/env python import argparse import os import sys from urllib3.util import parse_url import boto3 from upload.common.logging import get_logger from upload.common.dss_checksums import DssChecksums from upload.common.checksum_event import ChecksumEvent from upload.common.upload_api_client import update_event from upload.common.upload_config import UploadConfig logger = get_logger(f"CHECKSUMMER [{os.environ.get('AWS_BATCH_JOB_ID')}]") class Checksummer: def __init__(self, argv): self.bucket_name = None self.s3_object_key = None self.upload_area_id = None self.file_name = None UploadConfig.use_env = True # AWS Secrets are not available to batch jobs, use environment self._parse_args(argv) s3 = boto3.resource('s3') s3obj = s3.Bucket(self.bucket_name).Object(self.s3_object_key) self.checksums = DssChecksums(s3obj) self.checksum_event = ChecksumEvent(