def __init__(self): """Init method for the Report helper class.""" self.s3 = S3Helper() self.curr_dir = os.path.join( '/tmp', "dynamic_manifests") if not os.path.exists(self.curr_dir): os.makedirs(self.curr_dir)
def __init__(self): """Init method for the Report helper class.""" self.s3 = S3Helper() self.pg = Postgres() self.conn = self.pg.conn self.cursor = self.pg.cursor self.unknown_deps_helper = UnknownDepsReportHelper() self.sentry_helper = SentryReportHelper() self.github_token = os.environ.get('GITHUB_TOKEN') self.npm_model_bucket = os.getenv('NPM_MODEL_BUCKET') self.maven_model_bucket = os.getenv('MAVEN_MODEL_BUCKET') self.pypi_model_bucket = os.getenv('PYPI_MODEL_BUCKET') self.golang_model_bucket = os.getenv('GOLANG_MODEL_BUCKET') self.maven_training_repo = os.getenv( 'MAVEN_TRAINING_REPO', 'https://github.com/fabric8-analytics/f8a-hpf-insights') self.npm_training_repo = os.getenv( 'NPM_TRAINING_REPO', 'https://github.com/fabric8-analytics/fabric8-analytics-npm-insights' ) self.golang_training_repo = os.getenv( 'GOLANG_TRAINING_REPO', 'https://github.com/fabric8-analytics/f8a-golang-insights') self.pypi_training_repo = os.getenv( 'PYPI_TRAINING_REPO', 'https://github.com/fabric8-analytics/f8a-pypi-insights') self.emr_api = os.getenv('EMR_API', 'http://f8a-emr-deployment:6006')
def test_store_file_object(): """Test to validate list_object method.""" s3 = boto3.client('s3') s3.create_bucket(Bucket=BUCKET) S3 = S3Helper(aws_access_key_id=AWS_KEY, aws_secret_access_key=AWS_SECRET) S3.store_file_object('tests/data/dev/weekly/data.json', BUCKET, 'dev/weekly/data.json') obj = S3.list_objects(BUCKET, 'weekly') assert len(obj['objects']) > 0
def test_read_json_object(): """Test to validate read_json method.""" s3 = boto3.resource('s3') s3.create_bucket(Bucket=BUCKET) S3 = S3Helper(aws_access_key_id=AWS_KEY, aws_secret_access_key=AWS_SECRET) s3.meta.client.upload_file('tests/data/data.json', BUCKET, 'data.json') data = S3.read_json_object(BUCKET, 'data.json') assert data.get("key1") == "value1" data = S3.read_json_object('dummy', 'data.json') assert data is None
def test_list_objects(): """Test to validate list_object method.""" s3 = boto3.resource('s3') s3.create_bucket(Bucket=BUCKET) S3 = S3Helper(aws_access_key_id=AWS_KEY, aws_secret_access_key=AWS_SECRET) s3.meta.client.upload_file('tests/data/dev/weekly/data.json', BUCKET, 'dev/weekly/data.json') obj = S3.list_objects(BUCKET, 'weekly') assert len(obj['objects']) > 0 data = S3.list_objects('dummy', 'weekly') assert len(data['objects']) == 0
def save_worker_result_to_s3(frequency, report_name, content) -> bool: """Save worker result in S3 bucket. :param frequency: Frequency of Reporting ( daily/ monthly) :param report_name: Name of File/ Report. :param content: File Content to be saved in S3 :return: True: Save Success, False: Saved Fail """ logger.info("Trying to save report file") try: s3 = S3Helper() obj_key = f'v2/{frequency}/{report_name}.json' s3.store_json_content(content=content, obj_key=obj_key, bucket_name=s3.report_bucket_name) logger.info(f"Successfully saved report in {obj_key}.") return True except Exception as e: logger.exception(f'Unable to store the report on S3. Reason: {e}') return False
def test_store_json_content(): """Test to validate store_json method.""" s3 = boto3.resource('s3') s3.create_bucket(Bucket=BUCKET) S3 = S3Helper(aws_access_key_id=AWS_KEY, aws_secret_access_key=AWS_SECRET) S3.store_json_content({"keyA": "valueB"}, BUCKET, 'dummy.json')
def test_s3_client(): """Test to validate the s3 client method.""" S3 = S3Helper(aws_access_key_id=AWS_KEY, aws_secret_access_key=AWS_SECRET) s3 = S3.s3_client(BUCKET) assert s3
def test_s3_helper(): """Test to validate the s3_helper constructor function.""" assert S3Helper()
def __init__(self): """Init method for UnknownDepReportHelper.""" self.s3 = S3Helper()