def test_kms(): import os work_queue, done_queue, ologger = utils.comm_binders(test_kms) ologger.info('Key Master test') with bert_aws.kms(os.environ['KMS_KEY_ALIAS']) as keymaster: github_token: str = keymaster.decrypt(os.environ['GITHUB_TOKEN']) github_username: str = keymaster.decrypt(os.environ['GITHUB_USERNAME']) ologger.info(f'Github Token[{github_token}]') ologger.info(f'Github Username[{github_username}]')
def run_secrets(options: argparse.Namespace) -> None: bert_configuration = bert_shortcuts.load_configuration() or {} secrets = bert_shortcuts.obtain_secrets_config(bert_configuration) with bert_aws.kms(secrets.key_alias, secrets.usernames, True) as keymaster: if options.encrypt: value: typing.Any = bert_shortcuts.load_if_exists( options.keypath, bert_configuration) logger.info(f'Encrypting Keypath[{options.keypath}]') if isinstance(value, dict): for key, sub_value in value.items(): value[key] = keymaster.encrypt(sub_value) elif isinstance(value, list): for idx, sub_value in enumerate(value): value[idx] = keymaster.encrypt(sub_value) elif isinstance(value, str): value = keymaster.encrypt(value) else: raise NotImplementedError bert_shortcuts.write_keypath_value(options.keypath, bert_configuration, value) elif options.decrypt: value: typing.Any = bert_shortcuts.load_if_exists( options.keypath, bert_configuration) logger.info(f'Decrypting Keypath[{options.keypath}]') if isinstance(value, dict): for key, sub_value in value.items(): value[key] = keymaster.decrypt(sub_value) elif isinstance(value, list): for idx, sub_value in enumerate(value): value[idx] = keymaster.decrypt(sub_value) elif isinstance(value, str): value = keymaster.decrypt(value) else: raise NotImplementedError bert_shortcuts.write_keypath_value(options.keypath, bert_configuration, value) elif options.update_key_policy: keymaster.update_usernames() else: raise NotImplementedError if options.dry_run_off is True: bert_shortcuts.save_configuration(bert_configuration)
def map_s3_bucket(): import copy import csv import os import boto3 import json import lzma import requests import time import typing from bert import aws from collectMetrics import shortcuts from datetime import datetime, timedelta from requests.auth import HTTPBasicAuth from urllib.parse import urlencode work_queue, done_queue, ologger = utils.comm_binders(map_s3_bucket) outputs_dir: str = os.path.join('/tmp', 'outputs') if not os.path.exists(outputs_dir): os.makedirs(outputs_dir) ascii_date: str = shortcuts.obtain_latest_ascii_date(outputs_dir) config = {} with aws.kms('bert-etl') as master_key: config['username'] = master_key.decrypt(os.environ['GITHUB_USERNAME']) config['password'] = master_key.decrypt(os.environ['GITHUB_PASSWORD']) config['delay'] = 1 config['logger'] = ologger config['headers'] = {} astroconda_contrib_repos = shortcuts.org_name_contents( 'astroconda-contrib', {}, config) astroconda_dev_repos = shortcuts.org_name_contents('astroconda-dev', {}, config) shortcuts.sync_s3_datum('astroconda-contrib-repos', astroconda_contrib_repos) shortcuts.sync_s3_datum('astroconda-dev-repos', astroconda_dev_repos) for details in work_queue: done_queue.put(details)
def auth(self: PWN) -> HTTPBasicAuth: with aws.kms('bert-etl') as key_handler: return HTTPBasicAuth(key_handler.decrypt(self.username), key_handler.decrypt(self.password))
def mine_github_repo(): work_queue, done_queue, ologger = utils.comm_binders(mine_github_repo) import boto3 import lzma import os import json import requests import time from bert import aws from datetime import datetime from collectMetrics import shortcuts from requests.auth import HTTPBasicAuth from urllib.parse import urlencode output_dir: str = os.path.join('/tmp', 'outputs') if not os.path.exists(output_dir): os.makedirs(output_dir) config = {} with aws.kms('bert-etl') as keymaster: config['username'] = keymaster.decrypt(os.environ['GITHUB_USERNAME']) config['password'] = keymaster.decrypt(os.environ['GITHUB_PASSWORD']) config['delay'] = 1 config['base-url'] = 'https://api.github.com' config['headers'] = { 'User-Agent': 'repostats-tool', 'Accept': 'application/vnd.github.v3+json' } config['logger'] = ologger for details in work_queue: url: str = f'{config["base-url"]}/repos/{details["org_name"]}/{details["repo_name"]}' response = requests.get(url, auth=HTTPBasicAuth(config['username'], config['password']), headers=config['headers']) time.sleep(config['delay']) views = shortcuts.mine_repo_attribute(details['org_name'], details['repo_name'], 'traffic/views', {}, config) time.sleep(config['delay']) clones = shortcuts.mine_repo_attribute(details['org_name'], details['repo_name'], 'traffic/clones', {}, config) time.sleep(config['delay']) issues = shortcuts.mine_repo_attribute(details['org_name'], details['repo_name'], 'issues', {}, config) time.sleep(config['delay']) releases = shortcuts.mine_repo_attribute(details['org_name'], details['repo_name'], 'releases', {}, config) time.sleep(config['delay']) pull_requests = shortcuts.mine_repo_attribute(details['org_name'], details['repo_name'], 'pulls', {}, config) time.sleep(config['delay']) contributors = shortcuts.mine_repo_attribute(details['org_name'], details['repo_name'], 'contributors', {'anon': 'true'}, config) time.sleep(config['delay']) commits = shortcuts.mine_repo_attribute(details['org_name'], details['repo_name'], 'commits', {}, config) time.sleep(config['delay']) tags = shortcuts.mine_repo_attribute(details['org_name'], details['repo_name'], 'tags', {}, config) time.sleep(config['delay']) contents = shortcuts.mine_repo_attribute(details['org_name'], details['repo_name'], 'contents', {}, config) time.sleep(config['delay']) date: str = datetime.utcnow().strftime('%Y-%m-%d') filename: str = f'{details["org_name"]}-{details["repo_name"]}.json.xz' filepath: str = os.path.join(output_dir, date, filename) dir_path: str = os.path.dirname(filepath) if not os.path.exists(dir_path): os.makedirs(dir_path) with lzma.open(filepath, mode='w', format=lzma.FORMAT_XZ) as stream: data = json.dumps({ 'base': response.json(), 'views': views, 'clones': clones, 'issues': issues, 'releases': releases, 'pull_requests': pull_requests, 'contributors': contributors, 'commits': commits, 'tags': tags, 'contents': contents, }).encode('utf-8') stream.write(data) s3_key: str = f'daily/{date}/{filename}' ologger.info( f'Saving Timeseries Data for Repo[{details["repo_name"]}] to S3 Key[{s3_key}]' ) s3_client = boto3.client('s3') response = s3_client.upload_file(filepath, os.environ['DATASET_BUCKET'], s3_key) done_queue.put({ 'key': s3_key, 'filename': os.path.basename(filepath), }) os.remove(filepath) else: outputs_dir: str = os.path.join('/tmp', 'outputs') latest_s3_key: str = f'cache/latest-date.txt' latest_filepath: str = os.path.join(outputs_dir, latest_s3_key) outputs_dir_path: str = os.path.dirname(latest_filepath) if not os.path.exists(outputs_dir_path): os.makedirs(outputs_dir_path) with open(latest_filepath, 'w') as stream: stream.write(date) s3_client.upload_file(latest_filepath, os.environ['DATASET_BUCKET'], latest_s3_key)