def __init__(self, event, context, **kwargs):
super(CatalogHandler, self).__init__(event, context)
env_vars = self.retrieve(event, 'stage-variables', 'payload')
self.cdn_url = self.retrieve(env_vars, 'cdn_url').rstrip('/')
self.cdn_bucket = self.retrieve(env_vars, 'cdn_bucket')
self.api_bucket = self.retrieve(env_vars, 'api_bucket')
self.api_url = self.retrieve(env_vars, 'api_url').rstrip('/')
self.to_email = self.retrieve(env_vars, 'to_email')
self.from_email = self.retrieve(env_vars, 'from_email')
self.api_version = self.retrieve(env_vars, 'version')
in_progress_db = self.retrieve_with_default(
env_vars, 'in_progress_db',
'{}d43-catalog-in-progress'.format(self.stage_prefix()))
status_db = self.retrieve_with_default(
env_vars, 'status_db',
'{}d43-catalog-status'.format(self.stage_prefix()))
errors_db = self.retrieve_with_default(
env_vars, 'errors_db',
'{}d43-catalog-errors'.format(self.stage_prefix()))
if 'dynamodb_handler' in kwargs:
db_handler = kwargs['dynamodb_handler']
self.progress_table = db_handler(in_progress_db)
self.status_table = db_handler(status_db)
self.errors_table = db_handler(errors_db)
else:
self.progress_table = DynamoDBHandler(
in_progress_db) # pragma: no cover
self.status_table = DynamoDBHandler(status_db) # pragma: no cover
self.errors_table = DynamoDBHandler(errors_db) # pragma: no cover
self.catalog = {"languages": []}
if 's3_handler' in kwargs:
self.api_handler = kwargs['s3_handler'](self.api_bucket)
else:
self.api_handler = S3Handler(self.api_bucket) # pragma: no cover
if 'ses_handler' in kwargs:
self.ses_handler = kwargs['ses_handler']()
else:
self.ses_handler = SESHandler() # pragma: no cover
if 'consistency_checker' in kwargs:
self.checker = kwargs['consistency_checker']()
else:
self.checker = ConsistencyChecker(
self.cdn_bucket, self.api_bucket) # pragma: no cover
if 'get_url_handler' in kwargs:
self.get_url = kwargs['get_url_handler']
else:
self.get_url = get_url # pragma: no cover
if 'url_exists_handler' in kwargs:
self.url_exists = kwargs['url_exists_handler']
else:
self.url_exists = url_exists # pragma: no cover
def is_lambda_running(context,
dbname,
lambda_suffix=None,
dynamodb_handler=None):
"""
Retrieves the last recorded process information for this lambda.
This is used to determine if the lambda is already running.
:param context:
:param string dbname: the database holding a list of lambda run times
:param string lambda_suffix: name of the lambda handler
:return:
"""
if not context:
return False
if dynamodb_handler:
db = dynamodb_handler(dbname)
else:
db = DynamoDBHandler(dbname)
lambda_name = context.function_name
if lambda_suffix:
lambda_name = '{}.{}'.format(lambda_name, lambda_suffix)
request = db.get_item({"lambda": lambda_name})
if request:
start_time = arrow.get(request['started_at']).to('local')
expires_time = start_time.shift(minutes=request['expires'] / 60000)
return arrow.now() > expires_time
else:
return False
def set_lambda_running(context,
dbname,
lambda_suffix=None,
dynamodb_handler=None):
"""
Sets the process information for this lambda.
This is used to indicate the lambda is currently running.
:param context:
:param string dbname: the database holding a list of lambda run times.
:param string lambda_suffix: name of the lambda handler
:return:
"""
if not context:
return
if dynamodb_handler:
db = dynamodb_handler(dbname)
else:
db = DynamoDBHandler(dbname)
lambda_name = context.function_name
if lambda_suffix:
lambda_name = '{}.{}'.format(lambda_name, lambda_suffix)
db.insert_item({
"lambda": lambda_name,
"request_id": context.aws_request_id,
"started_at": arrow.utcnow().isoformat(),
"expires": context.get_remaining_time_in_millis()
})
def lambda_sec_remaining(context,
dbname,
lambda_suffix=None,
dynamodb_handler=None):
"""
Returns the time remaining in seconds before the lambda times out
:param context:
:return:
"""
if not context:
return timedelta(seconds=0)
if dynamodb_handler:
db = dynamodb_handler(dbname)
else:
db = DynamoDBHandler(dbname)
lambda_name = context.function_name
if lambda_suffix:
lambda_name = '{}.{}'.format(lambda_name, lambda_suffix)
request = db.get_item({"lambda": lambda_name})
if request:
start_time = arrow.get(request['started_at']).to('local')
# TRICKY: expires is in milliseconds
expires_time = start_time.shift(microseconds=(int(request['expires']) *
1000))
return expires_time - arrow.now()
else:
return timedelta(seconds=0)
def __init__(self, event, context, **kwargs):
super(ForkHandler, self).__init__(event, context)
self.stage_vars = self.retrieve(self.event, 'stage-variables',
'payload')
gogs_token = self.retrieve(self.stage_vars, 'gogs_token',
'Environment Vars')
self.gogs_url = self.retrieve(self.stage_vars, 'gogs_url',
'Environment Vars')
self.gogs_org = self.retrieve(self.stage_vars, 'gogs_org',
'Environment Vars')
self.from_email = self.retrieve(self.stage_vars, 'from_email',
'Environment Vars')
self.to_email = self.retrieve(self.stage_vars, 'to_email',
'Environment Vars')
if 'dynamodb_handler' in kwargs:
self.progress_table = kwargs['dynamodb_handler']
else:
self.progress_table = DynamoDBHandler(
'{}d43-catalog-in-progress'.format(
self.stage_prefix())) # pragma: no cover
if 'gitea_client' in kwargs:
self.gitea_client = kwargs['gitea_client']
else:
self.gitea_client = GiteaClient # pragma: no cover
if 'boto_handler' in kwargs:
self.boto = kwargs['boto_handler']
else:
self.boto = boto3 # pragma: no cover
if 'logger' in kwargs:
self.logger = kwargs['logger']
self.gogs_api = self.gitea_client.GiteaApi(self.gogs_url)
self.gogs_auth = self.gitea_client.Token(gogs_token)
def is_lambda_running(context, dbname, lambda_suffix=None, dynamodb_handler=None):
"""
Retrieves the last recorded process information for this lambda.
This is used to determine if the lambda is already running.
:param context:
:param string dbname: the database holding a list of lambda run times
:param string lambda_suffix: name of the lambda handler
:return:
"""
if not context:
return False
if dynamodb_handler:
db = dynamodb_handler(dbname)
else:
db = DynamoDBHandler(dbname)
lambda_name = context.function_name
if lambda_suffix:
lambda_name = '{}.{}'.format(lambda_name, lambda_suffix)
request = db.get_item({
"lambda": lambda_name
})
if request:
last_time = arrow.get(request['started_at']).to('local')
# TRICKY: we use this lambda's expires time instead of the recorded value avoid delays in applying changes to expiration times.
timeout = arrow.now().shift(minutes=-lambda_min_remaining(context))
return last_time > timeout
else:
return False
def _clear_report(self):
"""
Removes the error report from the db
:return:
"""
db = DynamoDBHandler(self.__table_name)
db.delete_item({'lambda': self.__reporter})
self._report = None
def _record_report(self):
"""
Stores the error report in the database
:return:
"""
if self._report:
db = DynamoDBHandler(self.__table_name)
db.update_item({'lambda': self.__reporter}, self._report)
def __init__(self, event, context, logger, **kwargs):
super(WebhookHandler, self).__init__(event, context)
env_vars = self.retrieve(event, 'stage-variables', 'payload')
self.gogs_url = self.retrieve(env_vars, 'gogs_url', 'Environment Vars')
self.gogs_token = self.retrieve(env_vars, 'gogs_token',
'Environment Vars')
self.gogs_org = self.retrieve(env_vars, 'gogs_org', 'Environment Vars')
self.cdn_bucket = self.retrieve(env_vars, 'cdn_bucket',
'Environment Vars')
self.cdn_url = self.retrieve(env_vars, 'cdn_url', 'Environment Vars')
self.from_email = self.retrieve(env_vars, 'from_email',
'Environment Vars')
self.to_email = self.retrieve(env_vars, 'to_email', 'Environment Vars')
self.api_url = self.retrieve(env_vars, 'api_url', 'Environment Vars')
self.repo_commit = self.retrieve(event, 'body-json', 'payload')
self.api_version = self.retrieve(env_vars, 'version')
# NOTE: it would be better to use the header X-GitHub-Event to determine the type of event.
if 'pull_request' in self.repo_commit:
# TODO: this is deprecated
self.__parse_pull_request(self.repo_commit)
elif 'forkee' in self.repo_commit or ('action' in self.repo_commit
and self.repo_commit['action']
== 'created'):
# handles fork and create events
self.__parse_fork(self.repo_commit)
elif 'pusher' in self.repo_commit:
self.__parse_push(self.repo_commit)
else:
raise Exception('Unsupported webhook request received ' +
self.repo_commit['repository']['name'] + ' ' +
json.dumps(self.repo_commit))
self.resource_id = None # set in self._build
self.logger = logger # type: logging._loggerClass
if 'dynamodb_handler' in kwargs:
self.db_handler = kwargs['dynamodb_handler']
else:
self.db_handler = DynamoDBHandler(
'{}d43-catalog-in-progress'.format(
self.stage_prefix())) # pragma: no cover
if 's3_handler' in kwargs:
self.s3_handler = kwargs['s3_handler']
else:
self.s3_handler = S3Handler(self.cdn_bucket) # pragma: no cover
if 'download_handler' in kwargs:
self.download_file = kwargs['download_handler']
else:
self.download_file = download_file # pragma: no cover
def add_error(self, message):
"""
Adds an error to the report
:param string|list message:
:return:
"""
if isinstance(message, list):
self.logger.info('Reporting Error: {}'.format(json.dumps(message)),
exc_info=1)
elif isinstance(message, str) or isinstance(message, unicode):
self.logger.info('Reporting Error: {}'.format(message), exc_info=1)
else:
self.logger.warning(
'Unable to report error. Invalid type "{}"'.format(
type(message)),
exc_info=1)
return
db = DynamoDBHandler(self.__table_name)
# load report
if not self._report:
item = db.get_item({'lambda': self.__reporter})
if not item:
item = {}
self._report = {
'errors': [],
'lambda': self.__reporter,
'reporters': []
}
self._report.update(item)
# start new report
if self.__request_id not in self._report['reporters']:
self._report['errors'] = []
self._report['reporters'].append(self.__request_id)
# append errors to report
if isinstance(message, list):
timestamp = arrow.utcnow().isoformat()
for m in message:
self._report['errors'].append({
'message': m.decode('utf-8'),
'timestamp': timestamp
})
else:
self._report['errors'].append({
'message':
message.decode('utf-8'),
'timestamp':
arrow.utcnow().isoformat()
})
def __init__(self, event, context, logger, **kwargs):
super(WebhookHandler, self).__init__(event, context)
env_vars = self.retrieve(event, 'stage-variables', 'payload')
self.gogs_url = self.retrieve(env_vars, 'gogs_url', 'Environment Vars')
self.gogs_org = self.retrieve(env_vars, 'gogs_org', 'Environment Vars')
self.cdn_bucket = self.retrieve(env_vars, 'cdn_bucket',
'Environment Vars')
self.cdn_url = self.retrieve(env_vars, 'cdn_url', 'Environment Vars')
self.from_email = self.retrieve(env_vars, 'from_email',
'Environment Vars')
self.to_email = self.retrieve(env_vars, 'to_email', 'Environment Vars')
self.api_url = self.retrieve(env_vars, 'api_url', 'Environment Vars')
self.repo_commit = self.retrieve(event, 'body-json', 'payload')
self.api_version = self.retrieve(env_vars, 'version')
self.in_progress_db = self.retrieve_with_default(
env_vars, 'in_progress_db',
'{}d43-catalog-in-progress'.format(self.stage_prefix()))
if 'pull_request' in self.repo_commit:
self.__parse_pull_request(self.repo_commit)
else:
self.__parse_push(self.repo_commit)
self.resource_id = None # set in self._build
self.logger = logger # type: logging._loggerClass
if 'dynamodb_handler' in kwargs:
self.db_handler = kwargs['dynamodb_handler']
else:
self.logger.debug(
"Creating Dynamodb handler pointing to {}".format(
self.in_progress_db))
self.db_handler = DynamoDBHandler(
self.in_progress_db) # pragma: no cover
if 's3_handler' in kwargs:
self.s3_handler = kwargs['s3_handler']
else:
self.s3_handler = S3Handler(self.cdn_bucket) # pragma: no cover
if 'download_handler' in kwargs:
self.download_file = kwargs['download_handler']
else:
self.download_file = download_file # pragma: no cover
def __init__(self, event, context, logger, signer, **kwargs):
super(SigningHandler, self).__init__(event, context)
env_vars = self.retrieve(event, 'stage-variables', 'payload')
self.cdn_bucket = self.retrieve(env_vars, 'cdn_bucket',
'Environment Vars')
self.cdn_url = self.retrieve(env_vars, 'cdn_url', 'Environment Vars')
self.from_email = self.retrieve(env_vars, 'from_email',
'Environment Vars')
self.to_email = self.retrieve(env_vars, 'to_email', 'Environment Vars')
self.api_version = self.retrieve(env_vars, 'version',
'Environment Vars')
self.api_bucket = self.retrieve(env_vars, 'api_bucket',
'Environment Vars')
self.logger = logger # type: logging._loggerClass
self.signer = signer
self.in_progress_db = self.retrieve_with_default(
env_vars, 'in_progress_db',
'{}d43-catalog-in-progress'.format(self.stage_prefix()))
if 's3_handler' in kwargs:
self.cdn_handler = kwargs['s3_handler']
else:
self.cdn_handler = S3Handler(self.cdn_bucket) # pragma: no cover
self.temp_dir = tempfile.mkdtemp(prefix='signing_')
if 'dynamodb_handler' in kwargs:
self.db_handler = kwargs['dynamodb_handler']
else:
self.db_handler = DynamoDBHandler(
self.in_progress_db) # pragma: no cover
if 'download_handler' in kwargs:
self.download_file = kwargs['download_handler']
else:
self.download_file = download_file # pragma: no cover
if 'url_exists_handler' in kwargs:
self.url_exists = kwargs['url_exists_handler']
else:
self.url_exists = url_exists # pragma: no cover
if 'url_headers_handler' in kwargs:
self.url_headers = kwargs['url_headers_handler']
else:
self.url_headers = url_headers # pragma: no cover
def __init__(self, event, context, logger, **kwargs):
super(TsV2CatalogHandler, self).__init__(event, context)
env_vars = self.retrieve(event, 'stage-variables', 'payload')
self.cdn_bucket = self.retrieve(env_vars, 'cdn_bucket',
'Environment Vars')
self.cdn_url = self.retrieve(env_vars, 'cdn_url',
'Environment Vars').rstrip('/')
self.from_email = self.retrieve(env_vars, 'from_email',
'Environment Vars')
self.to_email = self.retrieve(env_vars, 'to_email', 'Environment Vars')
self.max_usfm_size = int(
self.retrieve_with_default(env_vars, 'max_usfm_size', '2000000'))
self.status_db = self.retrieve_with_default(
env_vars, 'status_db',
'{}d43-catalog-status'.format(self.stage_prefix()))
self.logger = logger # type: logging._loggerClass
if 's3_handler' in kwargs:
self.cdn_handler = kwargs['s3_handler']
else:
self.cdn_handler = S3Handler(self.cdn_bucket) # pragma: no cover
if 'dynamodb_handler' in kwargs:
self.db_handler = kwargs['dynamodb_handler']
else:
self.db_handler = DynamoDBHandler(
self.status_db) # pragma: no cover
if self.db_handler.logger:
self.db_handler.logger.setLevel(logger.level)
if 'url_handler' in kwargs:
self.get_url = kwargs['url_handler']
else:
self.get_url = get_url # pragma: no cover
if 'download_handler' in kwargs:
self.download_file = kwargs['download_handler']
else:
self.download_file = download_file # pragma: no cover
if 'url_exists_handler' in kwargs:
self.url_exists = kwargs['url_exists_handler']
else:
self.url_exists = url_exists # pragma: no cover
self.temp_dir = tempfile.mkdtemp('', 'tsv2', None)
def __init__(self, event, context, logger, **kwargs):
super(UwV2CatalogHandler, self).__init__(event, context)
env_vars = self.retrieve(event, 'stage-variables', 'payload')
self.cdn_bucket = self.retrieve(env_vars, 'cdn_bucket',
'Environment Vars')
self.cdn_url = self.retrieve(env_vars, 'cdn_url',
'Environment Vars').rstrip('/')
self.from_email = self.retrieve(env_vars, 'from_email',
'Environment Vars')
self.to_email = self.retrieve(env_vars, 'to_email', 'Environment Vars')
self.logger = logger # type: logging._loggerClass
self.temp_dir = tempfile.mkdtemp('', 'uw_v2', None)
self.status_db = self.retrieve_with_default(
env_vars, 'status_db',
'{}d43-catalog-status'.format(self.stage_prefix()))
if 's3_handler' in kwargs:
self.cdn_handler = kwargs['s3_handler']
else:
self.cdn_handler = S3Handler(self.cdn_bucket) # pragma: no cover
if 'dynamodb_handler' in kwargs:
self.db_handler = kwargs['dynamodb_handler']
else:
self.db_handler = DynamoDBHandler(
self.status_db) # pragma: no cover
if 'url_handler' in kwargs:
self.get_url = kwargs['url_handler']
else:
self.get_url = get_url # pragma: no cover
if 'download_handler' in kwargs:
self.download_file = kwargs['download_handler']
else:
self.download_file = download_file # pragma: no cover
if 'signing_handler' in kwargs:
self.signer = kwargs['signing_handler']
else:
self.signer = Signer(ENC_PRIV_PEM_PATH) # pragma: no cover
def clear_lambda_running(context,
dbname,
lambda_suffix=None,
dynamodb_handler=None):
"""
This is a convenience method to clear a lambda from the list of running lambdas
:param context:
:param string dbname:
:param string lambda_suffix: the name of the lambda handler
:param dynamodb_handler:
:return:
"""
if dynamodb_handler:
db = dynamodb_handler(dbname)
else:
db = DynamoDBHandler(dbname)
lambda_name = context.function_name
if lambda_suffix:
lambda_name = '{}.{}'.format(lambda_name, lambda_suffix)
db.delete_item({"lambda": lambda_name})