def mock_client_linter_callback(self, error=None):
clcb = ClientLinterCallback(
identifier=self.lint_callback_data['identifier'],
success=self.lint_callback_data['success'],
info=self.lint_callback_data['info'],
warnings=self.lint_callback_data['warnings'],
errors=self.lint_callback_data['errors'],
s3_results_key=self.lint_callback_data['s3_results_key'])
return clcb
def test_callbackSimpleJob_LintNotFinished(self):
# given
self.unzip_resource_files("id_mat_ulb.zip", convert_finished=True)
identifier = self.lint_callback_data['identifier']
# when
results = ClientLinterCallback.deploy_if_conversion_finished(self.results_key, identifier)
# then
self.assertIsNone(results)
def test_callbackSimpleJob_LintNotFinished(self):
# given
self.unzip_resource_files("id_mat_ulb.zip", convert_finished=True)
identifier = self.lint_callback_data['identifier']
# when
results = ClientLinterCallback.deploy_if_conversion_finished(
self.results_key, identifier)
# then
self.assertIsNone(results)
def test_callbackMultpleJob_last_job_LintNotFinished(self):
# given
self.results_key = 'u/tx-manager-test-data/en-ulb/22f3d09f7a'
self.lint_callback_data['s3_results_key'] = self.results_key + '/3'
self.lint_callback_data['identifier'] = '1234567890/4/3/05-DEU.usfm'
self.unzip_resource_files("en_ulb.zip", convert_finished=True)
lint_log_path = self.get_source_path(file_name='lint_log.json')
file_utils.remove(lint_log_path)
identifier = self.lint_callback_data['identifier']
# when
results = ClientLinterCallback.deploy_if_conversion_finished(self.results_key, identifier)
# then
self.assertIsNone(results)
def _handle(self, event, context):
"""
:param dict event:
:param context:
:return dict:
"""
# Gather arguments
identifier = self.retrieve(self.data, 'identifier', 'Payload')
success = self.retrieve(self.data, 'success', 'Payload', required=False, default=False)
info = self.retrieve(self.data, 'info', 'Payload', required=False, default=[])
warnings = self.retrieve(self.data, 'warnings', 'Payload', required=False, default=[])
errors = self.retrieve(self.data, 'errors', 'Payload', required=False, default=[])
s3_results_key = self.retrieve(self.data, 's3_results_key', 'Payload')
# Execute
return ClientLinterCallback(identifier, success, info, warnings, errors, s3_results_key).process_callback()
def test_callbackMultpleJob_first_job_BuildLogMissing(self):
# given
self.results_key = 'u/tx-manager-test-data/en-ulb/22f3d09f7a'
self.lint_callback_data['s3_results_key'] = self.results_key + '/0'
self.lint_callback_data['identifier'] = '1234567890/4/0/01-GEN.usfm'
self.unzip_resource_files("en_ulb.zip", convert_finished=False)
build_log_path = self.get_source_path('build_log.json')
file_utils.remove(build_log_path)
self.finish_convert(self.source_folder)
identifier = self.lint_callback_data['identifier']
# when
results = ClientLinterCallback.deploy_if_conversion_finished(self.results_key, identifier)
# then
self.assertIsNone(results)
def test_callbackMultpleJob_last_job_LintNotFinished(self):
# given
self.results_key = 'u/tx-manager-test-data/en-ulb/22f3d09f7a'
self.lint_callback_data['s3_results_key'] = self.results_key + '/3'
self.lint_callback_data['identifier'] = '1234567890/4/3/05-DEU.usfm'
self.unzip_resource_files("en_ulb.zip", convert_finished=True)
lint_log_path = self.get_source_path(file_name='lint_log.json')
file_utils.remove(lint_log_path)
identifier = self.lint_callback_data['identifier']
# when
results = ClientLinterCallback.deploy_if_conversion_finished(
self.results_key, identifier)
# then
self.assertIsNone(results)
def test_callbackMultpleJob_first_job_BuildLogMissing(self):
# given
self.results_key = 'u/tx-manager-test-data/en-ulb/22f3d09f7a'
self.lint_callback_data['s3_results_key'] = self.results_key + '/0'
self.lint_callback_data['identifier'] = '1234567890/4/0/01-GEN.usfm'
self.unzip_resource_files("en_ulb.zip", convert_finished=False)
build_log_path = self.get_source_path('build_log.json')
file_utils.remove(build_log_path)
self.finish_convert(self.source_folder)
identifier = self.lint_callback_data['identifier']
# when
results = ClientLinterCallback.deploy_if_conversion_finished(
self.results_key, identifier)
# then
self.assertIsNone(results)
def process_callback(self):
job_id_parts = self.identifier.split('/')
job_id = job_id_parts[0]
self.job = TxJob.get(job_id)
if not self.job:
error = 'No job found for job_id = {0}, identifier = {1}'.format(
job_id, self.identifier)
App.logger.error(error)
raise Exception(error)
if len(job_id_parts) == 4:
part_count, part_id, book = job_id_parts[1:]
App.logger.debug(
'Multiple project, part {0} of {1}, converting book {2}'.
format(part_id, part_count, book))
multiple_project = True
else:
App.logger.debug('Single project')
part_id = None
multiple_project = False
self.job.ended_at = datetime.utcnow()
self.job.success = self.success
for message in self.log:
self.job.log_message(message)
for message in self.warnings:
self.job.warnings_message(message)
for message in self.errors:
self.job.error_message(message)
if len(self.errors):
self.job.log_message('{0} function returned with errors.'.format(
self.job.convert_module))
elif len(self.warnings):
self.job.log_message('{0} function returned with warnings.'.format(
self.job.convert_module))
else:
self.job.log_message('{0} function returned successfully.'.format(
self.job.convert_module))
if not self.success or len(self.job.errors):
self.job.success = False
self.job.status = "failed"
message = "Conversion failed"
App.logger.debug(
"Conversion failed, success: {0}, errors: {1}".format(
self.success, self.job.errors))
elif len(self.job.warnings) > 0:
self.job.success = True
self.job.status = "warnings"
message = "Conversion successful with warnings"
else:
self.job.success = True
self.job.status = "success"
message = "Conversion successful"
self.job.message = message
self.job.log_message(message)
self.job.log_message('Finished job {0} at {1}'.format(
self.job.job_id, self.job.ended_at.strftime("%Y-%m-%dT%H:%M:%SZ")))
s3_commit_key = 'u/{0}/{1}/{2}'.format(self.job.user_name,
self.job.repo_name,
self.job.commit_id)
upload_key = s3_commit_key
if multiple_project:
upload_key += "/" + part_id
App.logger.debug('Callback for commit {0}...'.format(s3_commit_key))
# Download the ZIP file of the converted files
converted_zip_url = self.job.output
converted_zip_file = os.path.join(self.temp_dir,
converted_zip_url.rpartition('/')[2])
remove(converted_zip_file) # make sure old file not present
download_success = True
App.logger.debug('Downloading converted zip file from {0}...'.format(
converted_zip_url))
try:
download_file(converted_zip_url, converted_zip_file)
except:
download_success = False # if multiple project we note fail and move on
if not multiple_project:
remove_tree(self.temp_dir) # cleanup
if self.job.errors is None:
self.job.errors = []
self.job.errors.append("Missing converted file: " +
converted_zip_url)
finally:
App.logger.debug('download finished, success={0}'.format(
str(download_success)))
self.job.update()
if download_success:
# Unzip the archive
unzip_dir = self.unzip_converted_files(converted_zip_file)
# Upload all files to the cdn_bucket with the key of <user>/<repo_name>/<commit> of the repo
self.upload_converted_files(upload_key, unzip_dir)
if multiple_project:
# Now download the existing build_log.json file, update it and upload it back to S3 as convert_log
build_log_json = self.update_convert_log(s3_commit_key,
part_id + "/")
# mark current part as finished
self.cdn_upload_contents({}, s3_commit_key + '/' + part_id +
'/finished')
else: # single part conversion
# Now download the existing build_log.json file, update it and upload it back to S3 as convert_log
build_log_json = self.update_convert_log(s3_commit_key)
self.cdn_upload_contents({}, s3_commit_key +
'/finished') # flag finished
results = ClientLinterCallback.deploy_if_conversion_finished(
s3_commit_key, self.identifier)
if results:
self.all_parts_completed = True
build_log_json = results
remove_tree(self.temp_dir) # cleanup
return build_log_json
def process_callback(self):
job_id_parts = self.identifier.split('/')
job_id = job_id_parts[0]
self.job = TxJob.get(job_id)
if not self.job:
error = 'No job found for job_id = {0}, identifier = {0}'.format(job_id, self.identifier)
App.logger.error(error)
raise Exception(error)
if len(job_id_parts) == 4:
part_count, part_id, book = job_id_parts[1:]
App.logger.debug('Multiple project, part {0} of {1}, converting book {2}'.
format(part_id, part_count, book))
multiple_project = True
else:
App.logger.debug('Single project')
part_id = None
multiple_project = False
self.job.ended_at = datetime.utcnow()
self.job.success = self.success
for message in self.log:
self.job.log_message(message)
for message in self.warnings:
self.job.warnings_message(message)
for message in self.errors:
self.job.error_message(message)
if len(self.errors):
self.job.log_message('{0} function returned with errors.'.format(self.job.convert_module))
elif len(self.warnings):
self.job.log_message('{0} function returned with warnings.'.format(self.job.convert_module))
else:
self.job.log_message('{0} function returned successfully.'.format(self.job.convert_module))
if not self.success or len(self.job.errors):
self.job.success = False
self.job.status = "failed"
message = "Conversion failed"
App.logger.debug("Conversion failed, success: {0}, errors: {1}".format(self.success, self.job.errors))
elif len(self.job.warnings) > 0:
self.job.success = True
self.job.status = "warnings"
message = "Conversion successful with warnings"
else:
self.job.success = True
self.job.status = "success"
message = "Conversion successful"
self.job.message = message
self.job.log_message(message)
self.job.log_message('Finished job {0} at {1}'.format(self.job.job_id, self.job.ended_at.strftime("%Y-%m-%dT%H:%M:%SZ")))
s3_commit_key = 'u/{0}/{1}/{2}'.format(self.job.user_name, self.job.repo_name, self.job.commit_id)
upload_key = s3_commit_key
if multiple_project:
upload_key += "/" + part_id
App.logger.debug('Callback for commit {0}...'.format(s3_commit_key))
# Download the ZIP file of the converted files
converted_zip_url = self.job.output
converted_zip_file = os.path.join(self.temp_dir, converted_zip_url.rpartition('/')[2])
remove(converted_zip_file) # make sure old file not present
download_success = True
App.logger.debug('Downloading converted zip file from {0}...'.format(converted_zip_url))
try:
download_file(converted_zip_url, converted_zip_file)
except:
download_success = False # if multiple project we note fail and move on
if not multiple_project:
remove_tree(self.temp_dir) # cleanup
if self.job.errors is None:
self.job.errors = []
self.job.errors.append("Missing converted file: " + converted_zip_url)
finally:
App.logger.debug('download finished, success={0}'.format(str(download_success)))
self.job.update()
if download_success:
# Unzip the archive
unzip_dir = self.unzip_converted_files(converted_zip_file)
# Upload all files to the cdn_bucket with the key of <user>/<repo_name>/<commit> of the repo
self.upload_converted_files(upload_key, unzip_dir)
if multiple_project:
# Now download the existing build_log.json file, update it and upload it back to S3 as convert_log
build_log_json = self.update_convert_log(s3_commit_key, part_id + "/")
# mark current part as finished
self.cdn_upload_contents({}, s3_commit_key + '/' + part_id + '/finished')
else: # single part conversion
# Now download the existing build_log.json file, update it and upload it back to S3 as convert_log
build_log_json = self.update_convert_log(s3_commit_key)
self.cdn_upload_contents({}, s3_commit_key + '/finished') # flag finished
results = ClientLinterCallback.deploy_if_conversion_finished(s3_commit_key, self.identifier)
if results:
self.all_parts_completed = True
build_log_json = results
remove_tree(self.temp_dir) # cleanup
return build_log_json