def init(gitlab_project_id=None, sqs_or_gitlab=None):
"""
Creates the Merge Request client to of a given type.
:param gitlab_project_id: used when the client type is 'gitlab'
:param sqs_or_gitlab: 'gitlab' or 'sqs'
:return: an instance of the selected MR client.
"""
if sqs_or_gitlab is None:
settings = queries.get_app_interface_settings()
client_type = settings.get('mergeRequestGateway', 'gitlab')
else:
client_type = sqs_or_gitlab
if client_type == 'gitlab':
if gitlab_project_id is None:
raise MRClientGatewayError('Missing "gitlab_project_id".')
instance = queries.get_gitlab_instance()
settings = queries.get_app_interface_settings()
saas_files = queries.get_saas_files_minimal(v1=True, v2=True)
return GitLabApi(instance, project_id=gitlab_project_id,
settings=settings, saas_files=saas_files)
elif client_type == 'sqs':
accounts = queries.get_aws_accounts()
settings = queries.get_app_interface_settings()
return SQSGateway(accounts, settings=settings)
else:
raise MRClientGatewayError(f'Invalid client type: {client_type}')
def run(dry_run, gitlab_project_id):
settings = queries.get_app_interface_settings()
accounts = queries.get_queue_aws_accounts()
sqs_cli = SQSGateway(accounts, settings=settings)
instance = queries.get_gitlab_instance()
saas_files = queries.get_saas_files_minimal(v1=True, v2=True)
gitlab_cli = GitLabApi(instance,
project_id=gitlab_project_id,
settings=settings,
saas_files=saas_files)
errors_occured = False
while True:
messages = sqs_cli.receive_messages()
logging.info("received %s messages", len(messages))
if not messages:
# sqs_cli.receive_messages delivers messages in chunks
# until the queue is empty... when that happens,
# we end this integration run
break
# not all integrations are going to resend their MR messages
# therefore we need to be careful not to delete any messages
# before they have been properly handled
for m in messages:
receipt_handle, body = m[0], m[1]
logging.info("received message %s with body %s",
receipt_handle[:6], json.dumps(body))
if not dry_run:
try:
merge_request = mr.init_from_sqs_message(body)
merge_request.submit_to_gitlab(gitlab_cli=gitlab_cli)
sqs_cli.delete_message(str(receipt_handle))
except mr.UnknownMergeRequestType as ex:
# Received an unknown MR type.
# This could be a producer being on a newer version
# of qontract-reconcile than the consumer.
# Therefore we don't delete it from the queue for
# potential future processing.
# TODO - monitor age of messages in queue
logging.warning(ex)
errors_occured = True
except mr.MergeRequestProcessingError as processing_error:
logging.error(processing_error)
errors_occured = True
if errors_occured:
sys.exit(1)
def run(dry_run, gitlab_project_id):
settings = queries.get_app_interface_settings()
accounts = queries.get_aws_accounts()
sqs_cli = SQSGateway(accounts, settings=settings)
instance = queries.get_gitlab_instance()
saas_files = queries.get_saas_files_minimal()
gitlab_cli = GitLabApi(instance,
project_id=gitlab_project_id,
settings=settings,
saas_files=saas_files)
while True:
messages = sqs_cli.receive_messages()
logging.info('received %s messages', len(messages))
if not messages:
break
for message in messages:
# Let's first delete all the message we received,
# otherwise they will come back in 30s.
receipt_handle = message[0]
sqs_cli.delete_message(str(receipt_handle))
for message in messages:
# Time to process the messages. Any failure here is not
# critical, even though we already deleted the messaged,
# since the producers will keep re-sending the message
# until the MR gets merged to app-interface
receipt_handle, body = message[0], message[1]
logging.info('received message %s with body %s',
receipt_handle[:6], json.dumps(body))
if not dry_run:
merge_request = mr.init_from_sqs_message(body)
merge_request.submit_to_gitlab(gitlab_cli=gitlab_cli)