def __init__(self, event, context, **kwargs): super(CatalogHandler, self).__init__(event, context) env_vars = self.retrieve(event, 'stage-variables', 'payload') self.cdn_url = self.retrieve(env_vars, 'cdn_url').rstrip('/') self.cdn_bucket = self.retrieve(env_vars, 'cdn_bucket') self.api_bucket = self.retrieve(env_vars, 'api_bucket') self.api_url = self.retrieve(env_vars, 'api_url').rstrip('/') self.to_email = self.retrieve(env_vars, 'to_email') self.from_email = self.retrieve(env_vars, 'from_email') self.api_version = self.retrieve(env_vars, 'version') in_progress_db = self.retrieve_with_default( env_vars, 'in_progress_db', '{}d43-catalog-in-progress'.format(self.stage_prefix())) status_db = self.retrieve_with_default( env_vars, 'status_db', '{}d43-catalog-status'.format(self.stage_prefix())) errors_db = self.retrieve_with_default( env_vars, 'errors_db', '{}d43-catalog-errors'.format(self.stage_prefix())) if 'dynamodb_handler' in kwargs: db_handler = kwargs['dynamodb_handler'] self.progress_table = db_handler(in_progress_db) self.status_table = db_handler(status_db) self.errors_table = db_handler(errors_db) else: self.progress_table = DynamoDBHandler( in_progress_db) # pragma: no cover self.status_table = DynamoDBHandler(status_db) # pragma: no cover self.errors_table = DynamoDBHandler(errors_db) # pragma: no cover self.catalog = {"languages": []} if 's3_handler' in kwargs: self.api_handler = kwargs['s3_handler'](self.api_bucket) else: self.api_handler = S3Handler(self.api_bucket) # pragma: no cover if 'ses_handler' in kwargs: self.ses_handler = kwargs['ses_handler']() else: self.ses_handler = SESHandler() # pragma: no cover if 'consistency_checker' in kwargs: self.checker = kwargs['consistency_checker']() else: self.checker = ConsistencyChecker( self.cdn_bucket, self.api_bucket) # pragma: no cover if 'get_url_handler' in kwargs: self.get_url = kwargs['get_url_handler'] else: self.get_url = get_url # pragma: no cover if 'url_exists_handler' in kwargs: self.url_exists = kwargs['url_exists_handler'] else: self.url_exists = url_exists # pragma: no cover
def is_lambda_running(context, dbname, lambda_suffix=None, dynamodb_handler=None): """ Retrieves the last recorded process information for this lambda. This is used to determine if the lambda is already running. :param context: :param string dbname: the database holding a list of lambda run times :param string lambda_suffix: name of the lambda handler :return: """ if not context: return False if dynamodb_handler: db = dynamodb_handler(dbname) else: db = DynamoDBHandler(dbname) lambda_name = context.function_name if lambda_suffix: lambda_name = '{}.{}'.format(lambda_name, lambda_suffix) request = db.get_item({"lambda": lambda_name}) if request: start_time = arrow.get(request['started_at']).to('local') expires_time = start_time.shift(minutes=request['expires'] / 60000) return arrow.now() > expires_time else: return False
def set_lambda_running(context, dbname, lambda_suffix=None, dynamodb_handler=None): """ Sets the process information for this lambda. This is used to indicate the lambda is currently running. :param context: :param string dbname: the database holding a list of lambda run times. :param string lambda_suffix: name of the lambda handler :return: """ if not context: return if dynamodb_handler: db = dynamodb_handler(dbname) else: db = DynamoDBHandler(dbname) lambda_name = context.function_name if lambda_suffix: lambda_name = '{}.{}'.format(lambda_name, lambda_suffix) db.insert_item({ "lambda": lambda_name, "request_id": context.aws_request_id, "started_at": arrow.utcnow().isoformat(), "expires": context.get_remaining_time_in_millis() })
def lambda_sec_remaining(context, dbname, lambda_suffix=None, dynamodb_handler=None): """ Returns the time remaining in seconds before the lambda times out :param context: :return: """ if not context: return timedelta(seconds=0) if dynamodb_handler: db = dynamodb_handler(dbname) else: db = DynamoDBHandler(dbname) lambda_name = context.function_name if lambda_suffix: lambda_name = '{}.{}'.format(lambda_name, lambda_suffix) request = db.get_item({"lambda": lambda_name}) if request: start_time = arrow.get(request['started_at']).to('local') # TRICKY: expires is in milliseconds expires_time = start_time.shift(microseconds=(int(request['expires']) * 1000)) return expires_time - arrow.now() else: return timedelta(seconds=0)
def __init__(self, event, context, **kwargs): super(ForkHandler, self).__init__(event, context) self.stage_vars = self.retrieve(self.event, 'stage-variables', 'payload') gogs_token = self.retrieve(self.stage_vars, 'gogs_token', 'Environment Vars') self.gogs_url = self.retrieve(self.stage_vars, 'gogs_url', 'Environment Vars') self.gogs_org = self.retrieve(self.stage_vars, 'gogs_org', 'Environment Vars') self.from_email = self.retrieve(self.stage_vars, 'from_email', 'Environment Vars') self.to_email = self.retrieve(self.stage_vars, 'to_email', 'Environment Vars') if 'dynamodb_handler' in kwargs: self.progress_table = kwargs['dynamodb_handler'] else: self.progress_table = DynamoDBHandler( '{}d43-catalog-in-progress'.format( self.stage_prefix())) # pragma: no cover if 'gitea_client' in kwargs: self.gitea_client = kwargs['gitea_client'] else: self.gitea_client = GiteaClient # pragma: no cover if 'boto_handler' in kwargs: self.boto = kwargs['boto_handler'] else: self.boto = boto3 # pragma: no cover if 'logger' in kwargs: self.logger = kwargs['logger'] self.gogs_api = self.gitea_client.GiteaApi(self.gogs_url) self.gogs_auth = self.gitea_client.Token(gogs_token)
def is_lambda_running(context, dbname, lambda_suffix=None, dynamodb_handler=None): """ Retrieves the last recorded process information for this lambda. This is used to determine if the lambda is already running. :param context: :param string dbname: the database holding a list of lambda run times :param string lambda_suffix: name of the lambda handler :return: """ if not context: return False if dynamodb_handler: db = dynamodb_handler(dbname) else: db = DynamoDBHandler(dbname) lambda_name = context.function_name if lambda_suffix: lambda_name = '{}.{}'.format(lambda_name, lambda_suffix) request = db.get_item({ "lambda": lambda_name }) if request: last_time = arrow.get(request['started_at']).to('local') # TRICKY: we use this lambda's expires time instead of the recorded value avoid delays in applying changes to expiration times. timeout = arrow.now().shift(minutes=-lambda_min_remaining(context)) return last_time > timeout else: return False
def _clear_report(self): """ Removes the error report from the db :return: """ db = DynamoDBHandler(self.__table_name) db.delete_item({'lambda': self.__reporter}) self._report = None
def _record_report(self): """ Stores the error report in the database :return: """ if self._report: db = DynamoDBHandler(self.__table_name) db.update_item({'lambda': self.__reporter}, self._report)
def __init__(self, event, context, logger, **kwargs): super(WebhookHandler, self).__init__(event, context) env_vars = self.retrieve(event, 'stage-variables', 'payload') self.gogs_url = self.retrieve(env_vars, 'gogs_url', 'Environment Vars') self.gogs_token = self.retrieve(env_vars, 'gogs_token', 'Environment Vars') self.gogs_org = self.retrieve(env_vars, 'gogs_org', 'Environment Vars') self.cdn_bucket = self.retrieve(env_vars, 'cdn_bucket', 'Environment Vars') self.cdn_url = self.retrieve(env_vars, 'cdn_url', 'Environment Vars') self.from_email = self.retrieve(env_vars, 'from_email', 'Environment Vars') self.to_email = self.retrieve(env_vars, 'to_email', 'Environment Vars') self.api_url = self.retrieve(env_vars, 'api_url', 'Environment Vars') self.repo_commit = self.retrieve(event, 'body-json', 'payload') self.api_version = self.retrieve(env_vars, 'version') # NOTE: it would be better to use the header X-GitHub-Event to determine the type of event. if 'pull_request' in self.repo_commit: # TODO: this is deprecated self.__parse_pull_request(self.repo_commit) elif 'forkee' in self.repo_commit or ('action' in self.repo_commit and self.repo_commit['action'] == 'created'): # handles fork and create events self.__parse_fork(self.repo_commit) elif 'pusher' in self.repo_commit: self.__parse_push(self.repo_commit) else: raise Exception('Unsupported webhook request received ' + self.repo_commit['repository']['name'] + ' ' + json.dumps(self.repo_commit)) self.resource_id = None # set in self._build self.logger = logger # type: logging._loggerClass if 'dynamodb_handler' in kwargs: self.db_handler = kwargs['dynamodb_handler'] else: self.db_handler = DynamoDBHandler( '{}d43-catalog-in-progress'.format( self.stage_prefix())) # pragma: no cover if 's3_handler' in kwargs: self.s3_handler = kwargs['s3_handler'] else: self.s3_handler = S3Handler(self.cdn_bucket) # pragma: no cover if 'download_handler' in kwargs: self.download_file = kwargs['download_handler'] else: self.download_file = download_file # pragma: no cover
def add_error(self, message): """ Adds an error to the report :param string|list message: :return: """ if isinstance(message, list): self.logger.info('Reporting Error: {}'.format(json.dumps(message)), exc_info=1) elif isinstance(message, str) or isinstance(message, unicode): self.logger.info('Reporting Error: {}'.format(message), exc_info=1) else: self.logger.warning( 'Unable to report error. Invalid type "{}"'.format( type(message)), exc_info=1) return db = DynamoDBHandler(self.__table_name) # load report if not self._report: item = db.get_item({'lambda': self.__reporter}) if not item: item = {} self._report = { 'errors': [], 'lambda': self.__reporter, 'reporters': [] } self._report.update(item) # start new report if self.__request_id not in self._report['reporters']: self._report['errors'] = [] self._report['reporters'].append(self.__request_id) # append errors to report if isinstance(message, list): timestamp = arrow.utcnow().isoformat() for m in message: self._report['errors'].append({ 'message': m.decode('utf-8'), 'timestamp': timestamp }) else: self._report['errors'].append({ 'message': message.decode('utf-8'), 'timestamp': arrow.utcnow().isoformat() })
def __init__(self, event, context, logger, **kwargs): super(WebhookHandler, self).__init__(event, context) env_vars = self.retrieve(event, 'stage-variables', 'payload') self.gogs_url = self.retrieve(env_vars, 'gogs_url', 'Environment Vars') self.gogs_org = self.retrieve(env_vars, 'gogs_org', 'Environment Vars') self.cdn_bucket = self.retrieve(env_vars, 'cdn_bucket', 'Environment Vars') self.cdn_url = self.retrieve(env_vars, 'cdn_url', 'Environment Vars') self.from_email = self.retrieve(env_vars, 'from_email', 'Environment Vars') self.to_email = self.retrieve(env_vars, 'to_email', 'Environment Vars') self.api_url = self.retrieve(env_vars, 'api_url', 'Environment Vars') self.repo_commit = self.retrieve(event, 'body-json', 'payload') self.api_version = self.retrieve(env_vars, 'version') self.in_progress_db = self.retrieve_with_default( env_vars, 'in_progress_db', '{}d43-catalog-in-progress'.format(self.stage_prefix())) if 'pull_request' in self.repo_commit: self.__parse_pull_request(self.repo_commit) else: self.__parse_push(self.repo_commit) self.resource_id = None # set in self._build self.logger = logger # type: logging._loggerClass if 'dynamodb_handler' in kwargs: self.db_handler = kwargs['dynamodb_handler'] else: self.logger.debug( "Creating Dynamodb handler pointing to {}".format( self.in_progress_db)) self.db_handler = DynamoDBHandler( self.in_progress_db) # pragma: no cover if 's3_handler' in kwargs: self.s3_handler = kwargs['s3_handler'] else: self.s3_handler = S3Handler(self.cdn_bucket) # pragma: no cover if 'download_handler' in kwargs: self.download_file = kwargs['download_handler'] else: self.download_file = download_file # pragma: no cover
def __init__(self, event, context, logger, signer, **kwargs): super(SigningHandler, self).__init__(event, context) env_vars = self.retrieve(event, 'stage-variables', 'payload') self.cdn_bucket = self.retrieve(env_vars, 'cdn_bucket', 'Environment Vars') self.cdn_url = self.retrieve(env_vars, 'cdn_url', 'Environment Vars') self.from_email = self.retrieve(env_vars, 'from_email', 'Environment Vars') self.to_email = self.retrieve(env_vars, 'to_email', 'Environment Vars') self.api_version = self.retrieve(env_vars, 'version', 'Environment Vars') self.api_bucket = self.retrieve(env_vars, 'api_bucket', 'Environment Vars') self.logger = logger # type: logging._loggerClass self.signer = signer self.in_progress_db = self.retrieve_with_default( env_vars, 'in_progress_db', '{}d43-catalog-in-progress'.format(self.stage_prefix())) if 's3_handler' in kwargs: self.cdn_handler = kwargs['s3_handler'] else: self.cdn_handler = S3Handler(self.cdn_bucket) # pragma: no cover self.temp_dir = tempfile.mkdtemp(prefix='signing_') if 'dynamodb_handler' in kwargs: self.db_handler = kwargs['dynamodb_handler'] else: self.db_handler = DynamoDBHandler( self.in_progress_db) # pragma: no cover if 'download_handler' in kwargs: self.download_file = kwargs['download_handler'] else: self.download_file = download_file # pragma: no cover if 'url_exists_handler' in kwargs: self.url_exists = kwargs['url_exists_handler'] else: self.url_exists = url_exists # pragma: no cover if 'url_headers_handler' in kwargs: self.url_headers = kwargs['url_headers_handler'] else: self.url_headers = url_headers # pragma: no cover
def __init__(self, event, context, logger, **kwargs): super(TsV2CatalogHandler, self).__init__(event, context) env_vars = self.retrieve(event, 'stage-variables', 'payload') self.cdn_bucket = self.retrieve(env_vars, 'cdn_bucket', 'Environment Vars') self.cdn_url = self.retrieve(env_vars, 'cdn_url', 'Environment Vars').rstrip('/') self.from_email = self.retrieve(env_vars, 'from_email', 'Environment Vars') self.to_email = self.retrieve(env_vars, 'to_email', 'Environment Vars') self.max_usfm_size = int( self.retrieve_with_default(env_vars, 'max_usfm_size', '2000000')) self.status_db = self.retrieve_with_default( env_vars, 'status_db', '{}d43-catalog-status'.format(self.stage_prefix())) self.logger = logger # type: logging._loggerClass if 's3_handler' in kwargs: self.cdn_handler = kwargs['s3_handler'] else: self.cdn_handler = S3Handler(self.cdn_bucket) # pragma: no cover if 'dynamodb_handler' in kwargs: self.db_handler = kwargs['dynamodb_handler'] else: self.db_handler = DynamoDBHandler( self.status_db) # pragma: no cover if self.db_handler.logger: self.db_handler.logger.setLevel(logger.level) if 'url_handler' in kwargs: self.get_url = kwargs['url_handler'] else: self.get_url = get_url # pragma: no cover if 'download_handler' in kwargs: self.download_file = kwargs['download_handler'] else: self.download_file = download_file # pragma: no cover if 'url_exists_handler' in kwargs: self.url_exists = kwargs['url_exists_handler'] else: self.url_exists = url_exists # pragma: no cover self.temp_dir = tempfile.mkdtemp('', 'tsv2', None)
def __init__(self, event, context, logger, **kwargs): super(UwV2CatalogHandler, self).__init__(event, context) env_vars = self.retrieve(event, 'stage-variables', 'payload') self.cdn_bucket = self.retrieve(env_vars, 'cdn_bucket', 'Environment Vars') self.cdn_url = self.retrieve(env_vars, 'cdn_url', 'Environment Vars').rstrip('/') self.from_email = self.retrieve(env_vars, 'from_email', 'Environment Vars') self.to_email = self.retrieve(env_vars, 'to_email', 'Environment Vars') self.logger = logger # type: logging._loggerClass self.temp_dir = tempfile.mkdtemp('', 'uw_v2', None) self.status_db = self.retrieve_with_default( env_vars, 'status_db', '{}d43-catalog-status'.format(self.stage_prefix())) if 's3_handler' in kwargs: self.cdn_handler = kwargs['s3_handler'] else: self.cdn_handler = S3Handler(self.cdn_bucket) # pragma: no cover if 'dynamodb_handler' in kwargs: self.db_handler = kwargs['dynamodb_handler'] else: self.db_handler = DynamoDBHandler( self.status_db) # pragma: no cover if 'url_handler' in kwargs: self.get_url = kwargs['url_handler'] else: self.get_url = get_url # pragma: no cover if 'download_handler' in kwargs: self.download_file = kwargs['download_handler'] else: self.download_file = download_file # pragma: no cover if 'signing_handler' in kwargs: self.signer = kwargs['signing_handler'] else: self.signer = Signer(ENC_PRIV_PEM_PATH) # pragma: no cover
def clear_lambda_running(context, dbname, lambda_suffix=None, dynamodb_handler=None): """ This is a convenience method to clear a lambda from the list of running lambdas :param context: :param string dbname: :param string lambda_suffix: the name of the lambda handler :param dynamodb_handler: :return: """ if dynamodb_handler: db = dynamodb_handler(dbname) else: db = DynamoDBHandler(dbname) lambda_name = context.function_name if lambda_suffix: lambda_name = '{}.{}'.format(lambda_name, lambda_suffix) db.delete_item({"lambda": lambda_name})