def test_kms():
import os
work_queue, done_queue, ologger = utils.comm_binders(test_kms)
ologger.info('Key Master test')
with bert_aws.kms(os.environ['KMS_KEY_ALIAS']) as keymaster:
github_token: str = keymaster.decrypt(os.environ['GITHUB_TOKEN'])
github_username: str = keymaster.decrypt(os.environ['GITHUB_USERNAME'])
ologger.info(f'Github Token[{github_token}]')
ologger.info(f'Github Username[{github_username}]')
def run_secrets(options: argparse.Namespace) -> None:
bert_configuration = bert_shortcuts.load_configuration() or {}
secrets = bert_shortcuts.obtain_secrets_config(bert_configuration)
with bert_aws.kms(secrets.key_alias, secrets.usernames, True) as keymaster:
if options.encrypt:
value: typing.Any = bert_shortcuts.load_if_exists(
options.keypath, bert_configuration)
logger.info(f'Encrypting Keypath[{options.keypath}]')
if isinstance(value, dict):
for key, sub_value in value.items():
value[key] = keymaster.encrypt(sub_value)
elif isinstance(value, list):
for idx, sub_value in enumerate(value):
value[idx] = keymaster.encrypt(sub_value)
elif isinstance(value, str):
value = keymaster.encrypt(value)
else:
raise NotImplementedError
bert_shortcuts.write_keypath_value(options.keypath,
bert_configuration, value)
elif options.decrypt:
value: typing.Any = bert_shortcuts.load_if_exists(
options.keypath, bert_configuration)
logger.info(f'Decrypting Keypath[{options.keypath}]')
if isinstance(value, dict):
for key, sub_value in value.items():
value[key] = keymaster.decrypt(sub_value)
elif isinstance(value, list):
for idx, sub_value in enumerate(value):
value[idx] = keymaster.decrypt(sub_value)
elif isinstance(value, str):
value = keymaster.decrypt(value)
else:
raise NotImplementedError
bert_shortcuts.write_keypath_value(options.keypath,
bert_configuration, value)
elif options.update_key_policy:
keymaster.update_usernames()
else:
raise NotImplementedError
if options.dry_run_off is True:
bert_shortcuts.save_configuration(bert_configuration)
def map_s3_bucket():
import copy
import csv
import os
import boto3
import json
import lzma
import requests
import time
import typing
from bert import aws
from collectMetrics import shortcuts
from datetime import datetime, timedelta
from requests.auth import HTTPBasicAuth
from urllib.parse import urlencode
work_queue, done_queue, ologger = utils.comm_binders(map_s3_bucket)
outputs_dir: str = os.path.join('/tmp', 'outputs')
if not os.path.exists(outputs_dir):
os.makedirs(outputs_dir)
ascii_date: str = shortcuts.obtain_latest_ascii_date(outputs_dir)
config = {}
with aws.kms('bert-etl') as master_key:
config['username'] = master_key.decrypt(os.environ['GITHUB_USERNAME'])
config['password'] = master_key.decrypt(os.environ['GITHUB_PASSWORD'])
config['delay'] = 1
config['logger'] = ologger
config['headers'] = {}
astroconda_contrib_repos = shortcuts.org_name_contents(
'astroconda-contrib', {}, config)
astroconda_dev_repos = shortcuts.org_name_contents('astroconda-dev', {},
config)
shortcuts.sync_s3_datum('astroconda-contrib-repos',
astroconda_contrib_repos)
shortcuts.sync_s3_datum('astroconda-dev-repos', astroconda_dev_repos)
for details in work_queue:
done_queue.put(details)
def auth(self: PWN) -> HTTPBasicAuth:
with aws.kms('bert-etl') as key_handler:
return HTTPBasicAuth(key_handler.decrypt(self.username),
key_handler.decrypt(self.password))
def mine_github_repo():
work_queue, done_queue, ologger = utils.comm_binders(mine_github_repo)
import boto3
import lzma
import os
import json
import requests
import time
from bert import aws
from datetime import datetime
from collectMetrics import shortcuts
from requests.auth import HTTPBasicAuth
from urllib.parse import urlencode
output_dir: str = os.path.join('/tmp', 'outputs')
if not os.path.exists(output_dir):
os.makedirs(output_dir)
config = {}
with aws.kms('bert-etl') as keymaster:
config['username'] = keymaster.decrypt(os.environ['GITHUB_USERNAME'])
config['password'] = keymaster.decrypt(os.environ['GITHUB_PASSWORD'])
config['delay'] = 1
config['base-url'] = 'https://api.github.com'
config['headers'] = {
'User-Agent': 'repostats-tool',
'Accept': 'application/vnd.github.v3+json'
}
config['logger'] = ologger
for details in work_queue:
url: str = f'{config["base-url"]}/repos/{details["org_name"]}/{details["repo_name"]}'
response = requests.get(url,
auth=HTTPBasicAuth(config['username'],
config['password']),
headers=config['headers'])
time.sleep(config['delay'])
views = shortcuts.mine_repo_attribute(details['org_name'],
details['repo_name'],
'traffic/views', {}, config)
time.sleep(config['delay'])
clones = shortcuts.mine_repo_attribute(details['org_name'],
details['repo_name'],
'traffic/clones', {}, config)
time.sleep(config['delay'])
issues = shortcuts.mine_repo_attribute(details['org_name'],
details['repo_name'], 'issues',
{}, config)
time.sleep(config['delay'])
releases = shortcuts.mine_repo_attribute(details['org_name'],
details['repo_name'],
'releases', {}, config)
time.sleep(config['delay'])
pull_requests = shortcuts.mine_repo_attribute(details['org_name'],
details['repo_name'],
'pulls', {}, config)
time.sleep(config['delay'])
contributors = shortcuts.mine_repo_attribute(details['org_name'],
details['repo_name'],
'contributors',
{'anon': 'true'}, config)
time.sleep(config['delay'])
commits = shortcuts.mine_repo_attribute(details['org_name'],
details['repo_name'],
'commits', {}, config)
time.sleep(config['delay'])
tags = shortcuts.mine_repo_attribute(details['org_name'],
details['repo_name'], 'tags', {},
config)
time.sleep(config['delay'])
contents = shortcuts.mine_repo_attribute(details['org_name'],
details['repo_name'],
'contents', {}, config)
time.sleep(config['delay'])
date: str = datetime.utcnow().strftime('%Y-%m-%d')
filename: str = f'{details["org_name"]}-{details["repo_name"]}.json.xz'
filepath: str = os.path.join(output_dir, date, filename)
dir_path: str = os.path.dirname(filepath)
if not os.path.exists(dir_path):
os.makedirs(dir_path)
with lzma.open(filepath, mode='w', format=lzma.FORMAT_XZ) as stream:
data = json.dumps({
'base': response.json(),
'views': views,
'clones': clones,
'issues': issues,
'releases': releases,
'pull_requests': pull_requests,
'contributors': contributors,
'commits': commits,
'tags': tags,
'contents': contents,
}).encode('utf-8')
stream.write(data)
s3_key: str = f'daily/{date}/{filename}'
ologger.info(
f'Saving Timeseries Data for Repo[{details["repo_name"]}] to S3 Key[{s3_key}]'
)
s3_client = boto3.client('s3')
response = s3_client.upload_file(filepath,
os.environ['DATASET_BUCKET'], s3_key)
done_queue.put({
'key': s3_key,
'filename': os.path.basename(filepath),
})
os.remove(filepath)
else:
outputs_dir: str = os.path.join('/tmp', 'outputs')
latest_s3_key: str = f'cache/latest-date.txt'
latest_filepath: str = os.path.join(outputs_dir, latest_s3_key)
outputs_dir_path: str = os.path.dirname(latest_filepath)
if not os.path.exists(outputs_dir_path):
os.makedirs(outputs_dir_path)
with open(latest_filepath, 'w') as stream:
stream.write(date)
s3_client.upload_file(latest_filepath, os.environ['DATASET_BUCKET'],
latest_s3_key)