def __init__(self):
"""Init method for the Report helper class."""
self.s3 = S3Helper()
self.curr_dir = os.path.join(
'/tmp', "dynamic_manifests")
if not os.path.exists(self.curr_dir):
os.makedirs(self.curr_dir)
def __init__(self):
"""Init method for the Report helper class."""
self.s3 = S3Helper()
self.pg = Postgres()
self.conn = self.pg.conn
self.cursor = self.pg.cursor
self.unknown_deps_helper = UnknownDepsReportHelper()
self.sentry_helper = SentryReportHelper()
self.github_token = os.environ.get('GITHUB_TOKEN')
self.npm_model_bucket = os.getenv('NPM_MODEL_BUCKET')
self.maven_model_bucket = os.getenv('MAVEN_MODEL_BUCKET')
self.pypi_model_bucket = os.getenv('PYPI_MODEL_BUCKET')
self.golang_model_bucket = os.getenv('GOLANG_MODEL_BUCKET')
self.maven_training_repo = os.getenv(
'MAVEN_TRAINING_REPO',
'https://github.com/fabric8-analytics/f8a-hpf-insights')
self.npm_training_repo = os.getenv(
'NPM_TRAINING_REPO',
'https://github.com/fabric8-analytics/fabric8-analytics-npm-insights'
)
self.golang_training_repo = os.getenv(
'GOLANG_TRAINING_REPO',
'https://github.com/fabric8-analytics/f8a-golang-insights')
self.pypi_training_repo = os.getenv(
'PYPI_TRAINING_REPO',
'https://github.com/fabric8-analytics/f8a-pypi-insights')
self.emr_api = os.getenv('EMR_API', 'http://f8a-emr-deployment:6006')
def test_store_file_object():
"""Test to validate list_object method."""
s3 = boto3.client('s3')
s3.create_bucket(Bucket=BUCKET)
S3 = S3Helper(aws_access_key_id=AWS_KEY, aws_secret_access_key=AWS_SECRET)
S3.store_file_object('tests/data/dev/weekly/data.json', BUCKET,
'dev/weekly/data.json')
obj = S3.list_objects(BUCKET, 'weekly')
assert len(obj['objects']) > 0
def test_read_json_object():
"""Test to validate read_json method."""
s3 = boto3.resource('s3')
s3.create_bucket(Bucket=BUCKET)
S3 = S3Helper(aws_access_key_id=AWS_KEY, aws_secret_access_key=AWS_SECRET)
s3.meta.client.upload_file('tests/data/data.json', BUCKET, 'data.json')
data = S3.read_json_object(BUCKET, 'data.json')
assert data.get("key1") == "value1"
data = S3.read_json_object('dummy', 'data.json')
assert data is None
def test_list_objects():
"""Test to validate list_object method."""
s3 = boto3.resource('s3')
s3.create_bucket(Bucket=BUCKET)
S3 = S3Helper(aws_access_key_id=AWS_KEY, aws_secret_access_key=AWS_SECRET)
s3.meta.client.upload_file('tests/data/dev/weekly/data.json', BUCKET,
'dev/weekly/data.json')
obj = S3.list_objects(BUCKET, 'weekly')
assert len(obj['objects']) > 0
data = S3.list_objects('dummy', 'weekly')
assert len(data['objects']) == 0
def save_worker_result_to_s3(frequency, report_name, content) -> bool:
"""Save worker result in S3 bucket.
:param frequency: Frequency of Reporting ( daily/ monthly)
:param report_name: Name of File/ Report.
:param content: File Content to be saved in S3
:return: True: Save Success, False: Saved Fail
"""
logger.info("Trying to save report file")
try:
s3 = S3Helper()
obj_key = f'v2/{frequency}/{report_name}.json'
s3.store_json_content(content=content,
obj_key=obj_key,
bucket_name=s3.report_bucket_name)
logger.info(f"Successfully saved report in {obj_key}.")
return True
except Exception as e:
logger.exception(f'Unable to store the report on S3. Reason: {e}')
return False
def test_store_json_content():
"""Test to validate store_json method."""
s3 = boto3.resource('s3')
s3.create_bucket(Bucket=BUCKET)
S3 = S3Helper(aws_access_key_id=AWS_KEY, aws_secret_access_key=AWS_SECRET)
S3.store_json_content({"keyA": "valueB"}, BUCKET, 'dummy.json')
def test_s3_client():
"""Test to validate the s3 client method."""
S3 = S3Helper(aws_access_key_id=AWS_KEY, aws_secret_access_key=AWS_SECRET)
s3 = S3.s3_client(BUCKET)
assert s3
def test_s3_helper():
"""Test to validate the s3_helper constructor function."""
assert S3Helper()
def __init__(self):
"""Init method for UnknownDepReportHelper."""
self.s3 = S3Helper()