def poll_until_all_jobs_finished(self, build_logs):
job = None
finished = []
job_count = len(build_logs)
polling_timeout = 5 * 60 # poll for up to 5 minutes for job to complete or error
sleep_interval = 5 # how often to check for completion
done = False
start = time.time()
end = start + polling_timeout
while (time.time() < end) and not done:
time.sleep(sleep_interval) # delay before polling again
for build_log in build_logs: # check for completion of each part
job_id = build_log['job_id']
if job_id in finished:
continue # skip if job already finished
job = TxJob.get(job_id)
self.assertIsNotNone(job)
App.logger.debug("job " + job_id + " status at " + str(elapsed_time(start)) + ":\n" + str(job.log))
if job.ended_at is not None:
finished.append(job_id)
end = time.time() + polling_timeout # reset timeout
if len(finished) >= job_count:
done = True # finished
break
if len(finished) < job_count:
for build_log in build_logs: # check for completion of each part
job_id = build_log['job_id']
if job_id not in finished:
self.warn("Timeout waiting for start on job: " + job_id)
return done, job
def update_jobs_table(s3_results_key, build_log, output_dir):
job_id = build_log['job_id']
App.logger.debug('merging build_logs for job : ' + job_id)
build_log['ended_at'] = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
job = TxJob.get(job_id)
if job:
job.status = build_log['status']
job.log = build_log['log']
job.warnings = build_log['warnings']
job.errors = build_log['errors']
job.message = build_log['message']
job.success = build_log['success']
job.ended_at = build_log['ended_at']
# set overall status
if len(job.errors):
job.status = 'errors'
job.success = False
elif len(job.warnings):
job.status = 'warnings'
job.update()
else:
job_data = {'manifests_id': 0} # set a default if not present
for key in build_log:
if hasattr(TxJob, key):
job_data[key] = build_log[key]
job = TxJob(**job_data)
job.insert()
# flag this part as done
ClientLinterCallback.upload_build_log(build_log, 'merged.json', output_dir, s3_results_key)
# update build_log to start deploy of this part
ClientLinterCallback.upload_build_log(build_log, 'build_log.json', output_dir, s3_results_key, cache_time=600)
return
def get_unique_job_id(self):
"""
:return string:
"""
job_id = hashlib.sha256(datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S.%f")).hexdigest()
while TxJob.get(job_id):
job_id = hashlib.sha256(datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S.%f")).hexdigest()
return job_id
def get_unique_job_id(self):
"""
:return string:
"""
job_id = hashlib.sha256(datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S.%f")).hexdigest()
while TxJob.get(job_id):
job_id = hashlib.sha256(datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S.%f")).hexdigest()
return job_id
def test_client_converter_callback_multiple_job_complete_error(self, mock_download_file):
# given
self.source_zip = os.path.join(self.resources_dir, "raw_sources/en-ulb.zip")
identifier = 'job1/2/0/01-GEN.usfm'
tx_job = TxJob.get('job1')
tx_job.errors = ['conversion failed']
tx_job.update()
self.s3_results_key = 'u/tx-manager-test-data/en-ulb/22f3d09f7a/0'
mock_cccb = self.mock_client_converter_callback(identifier, mock_download_file)
self.generate_parts_completed(0, 2)
expect_error = True
# when
results = mock_cccb.process_callback()
# then
self.validate_results(expect_error, results)
def test_client_converter_callback_multiple_job_complete_error(
self, mock_download_file):
# given
self.source_zip = os.path.join(self.resources_dir,
"raw_sources/en-ulb.zip")
identifier = 'job1/2/0/01-GEN.usfm'
tx_job = TxJob.get('job1')
tx_job.errors = ['conversion failed']
tx_job.update()
self.s3_results_key = 'u/tx-manager-test-data/en-ulb/22f3d09f7a/0'
mock_cccb = self.mock_client_converter_callback(
identifier, mock_download_file)
self.generate_parts_completed(0, 2)
expect_error = True
# when
results = mock_cccb.process_callback()
# then
self.validate_results(expect_error, results)
def test_process_webhook(self, mock_download_file):
# given
client_web_hook = self.setup_client_webhook_mock('kpb_mat_text_udb_repo', mock_download_file)
expected_job_count = 1
expected_error_count = 0
# when
results = client_web_hook.process_webhook()
# then
self.validateResults(results, expected_job_count, expected_error_count)
# Check repo was added to manifest table
repo_name = client_web_hook.commit_data['repository']['name']
user_name = client_web_hook.commit_data['repository']['owner']['username']
tx_manifest = TxManifest.get(repo_name=repo_name, user_name=user_name)
tx_job = TxJob.get(results['job_id'])
self.assertEqual(tx_manifest.repo_name, client_web_hook.commit_data['repository']['name'])
self.assertEqual(tx_manifest.resource_id, 'udb')
self.assertEqual(tx_manifest.lang_code, 'kpb')
self.assertEqual(tx_manifest.id, tx_job.manifests_id)
def poll_until_job_finished(self, job_id):
success = False
job = None
polling_timeout = 5 * 60 # poll for up to 5 minutes for job to complete or error
sleep_interval = 5 # how often to check for completion
start = time.time()
end = start + polling_timeout
while time.time() < end:
time.sleep(sleep_interval)
job = TxJob.get(job_id)
self.assertIsNotNone(job)
elapsed_seconds = elapsed_time(start)
App.logger.debug("job " + job_id + " status at " + str(elapsed_seconds) + ":\n" + str(job.log))
if job.ended_at is not None:
success = True
break
if not success:
self.warn("Timeout Waiting for start on job: " + job_id)
return success, job
def poll_until_job_finished(self, job_id):
success = False
job = None
polling_timeout = 5 * 60 # poll for up to 5 minutes for job to complete or error
sleep_interval = 5 # how often to check for completion
start = time.time()
end = start + polling_timeout
while time.time() < end:
time.sleep(sleep_interval)
job = TxJob.get(job_id)
self.assertIsNotNone(job)
elapsed_seconds = elapsed_time(start)
App.logger.debug("job " + job_id + " status at " +
str(elapsed_seconds) + ":\n" + str(job.log))
if job.ended_at is not None:
success = True
break
if not success:
self.warn("Timeout Waiting for start on job: " + job_id)
return success, job
def test_process_webhook(self, mock_download_file):
# given
client_web_hook = self.setup_client_webhook_mock(
'kpb_mat_text_udb_repo', mock_download_file)
expected_job_count = 1
expected_error_count = 0
# when
results = client_web_hook.process_webhook()
# then
self.validateResults(results, expected_job_count, expected_error_count)
# Check repo was added to manifest table
repo_name = client_web_hook.commit_data['repository']['name']
user_name = client_web_hook.commit_data['repository']['owner'][
'username']
tx_manifest = TxManifest.get(repo_name=repo_name, user_name=user_name)
tx_job = TxJob.get(results['job_id'])
self.assertEqual(tx_manifest.repo_name,
client_web_hook.commit_data['repository']['name'])
self.assertEqual(tx_manifest.resource_id, 'udb')
self.assertEqual(tx_manifest.lang_code, 'kpb')
self.assertEqual(tx_manifest.id, tx_job.manifests_id)
def poll_until_all_jobs_finished(self, build_logs):
job = None
finished = []
job_count = len(build_logs)
polling_timeout = 5 * 60 # poll for up to 5 minutes for job to complete or error
sleep_interval = 5 # how often to check for completion
done = False
start = time.time()
end = start + polling_timeout
while (time.time() < end) and not done:
time.sleep(sleep_interval) # delay before polling again
for build_log in build_logs: # check for completion of each part
job_id = build_log['job_id']
if job_id in finished:
continue # skip if job already finished
job = TxJob.get(job_id)
self.assertIsNotNone(job)
App.logger.debug("job " + job_id + " status at " +
str(elapsed_time(start)) + ":\n" +
str(job.log))
if job.ended_at is not None:
finished.append(job_id)
end = time.time() + polling_timeout # reset timeout
if len(finished) >= job_count:
done = True # finished
break
if len(finished) < job_count:
for build_log in build_logs: # check for completion of each part
job_id = build_log['job_id']
if job_id not in finished:
self.warn("Timeout waiting for start on job: " + job_id)
return done, job
def process_callback(self):
job_id_parts = self.identifier.split('/')
job_id = job_id_parts[0]
self.job = TxJob.get(job_id)
if not self.job:
error = 'No job found for job_id = {0}, identifier = {1}'.format(
job_id, self.identifier)
App.logger.error(error)
raise Exception(error)
if len(job_id_parts) == 4:
part_count, part_id, book = job_id_parts[1:]
App.logger.debug(
'Multiple project, part {0} of {1}, converting book {2}'.
format(part_id, part_count, book))
multiple_project = True
else:
App.logger.debug('Single project')
part_id = None
multiple_project = False
self.job.ended_at = datetime.utcnow()
self.job.success = self.success
for message in self.log:
self.job.log_message(message)
for message in self.warnings:
self.job.warnings_message(message)
for message in self.errors:
self.job.error_message(message)
if len(self.errors):
self.job.log_message('{0} function returned with errors.'.format(
self.job.convert_module))
elif len(self.warnings):
self.job.log_message('{0} function returned with warnings.'.format(
self.job.convert_module))
else:
self.job.log_message('{0} function returned successfully.'.format(
self.job.convert_module))
if not self.success or len(self.job.errors):
self.job.success = False
self.job.status = "failed"
message = "Conversion failed"
App.logger.debug(
"Conversion failed, success: {0}, errors: {1}".format(
self.success, self.job.errors))
elif len(self.job.warnings) > 0:
self.job.success = True
self.job.status = "warnings"
message = "Conversion successful with warnings"
else:
self.job.success = True
self.job.status = "success"
message = "Conversion successful"
self.job.message = message
self.job.log_message(message)
self.job.log_message('Finished job {0} at {1}'.format(
self.job.job_id, self.job.ended_at.strftime("%Y-%m-%dT%H:%M:%SZ")))
s3_commit_key = 'u/{0}/{1}/{2}'.format(self.job.user_name,
self.job.repo_name,
self.job.commit_id)
upload_key = s3_commit_key
if multiple_project:
upload_key += "/" + part_id
App.logger.debug('Callback for commit {0}...'.format(s3_commit_key))
# Download the ZIP file of the converted files
converted_zip_url = self.job.output
converted_zip_file = os.path.join(self.temp_dir,
converted_zip_url.rpartition('/')[2])
remove(converted_zip_file) # make sure old file not present
download_success = True
App.logger.debug('Downloading converted zip file from {0}...'.format(
converted_zip_url))
try:
download_file(converted_zip_url, converted_zip_file)
except:
download_success = False # if multiple project we note fail and move on
if not multiple_project:
remove_tree(self.temp_dir) # cleanup
if self.job.errors is None:
self.job.errors = []
self.job.errors.append("Missing converted file: " +
converted_zip_url)
finally:
App.logger.debug('download finished, success={0}'.format(
str(download_success)))
self.job.update()
if download_success:
# Unzip the archive
unzip_dir = self.unzip_converted_files(converted_zip_file)
# Upload all files to the cdn_bucket with the key of <user>/<repo_name>/<commit> of the repo
self.upload_converted_files(upload_key, unzip_dir)
if multiple_project:
# Now download the existing build_log.json file, update it and upload it back to S3 as convert_log
build_log_json = self.update_convert_log(s3_commit_key,
part_id + "/")
# mark current part as finished
self.cdn_upload_contents({}, s3_commit_key + '/' + part_id +
'/finished')
else: # single part conversion
# Now download the existing build_log.json file, update it and upload it back to S3 as convert_log
build_log_json = self.update_convert_log(s3_commit_key)
self.cdn_upload_contents({}, s3_commit_key +
'/finished') # flag finished
results = ClientLinterCallback.deploy_if_conversion_finished(
s3_commit_key, self.identifier)
if results:
self.all_parts_completed = True
build_log_json = results
remove_tree(self.temp_dir) # cleanup
return build_log_json
def process_callback(self):
job_id_parts = self.identifier.split('/')
job_id = job_id_parts[0]
self.job = TxJob.get(job_id)
if not self.job:
error = 'No job found for job_id = {0}, identifier = {0}'.format(job_id, self.identifier)
App.logger.error(error)
raise Exception(error)
if len(job_id_parts) == 4:
part_count, part_id, book = job_id_parts[1:]
App.logger.debug('Multiple project, part {0} of {1}, converting book {2}'.
format(part_id, part_count, book))
multiple_project = True
else:
App.logger.debug('Single project')
part_id = None
multiple_project = False
self.job.ended_at = datetime.utcnow()
self.job.success = self.success
for message in self.log:
self.job.log_message(message)
for message in self.warnings:
self.job.warnings_message(message)
for message in self.errors:
self.job.error_message(message)
if len(self.errors):
self.job.log_message('{0} function returned with errors.'.format(self.job.convert_module))
elif len(self.warnings):
self.job.log_message('{0} function returned with warnings.'.format(self.job.convert_module))
else:
self.job.log_message('{0} function returned successfully.'.format(self.job.convert_module))
if not self.success or len(self.job.errors):
self.job.success = False
self.job.status = "failed"
message = "Conversion failed"
App.logger.debug("Conversion failed, success: {0}, errors: {1}".format(self.success, self.job.errors))
elif len(self.job.warnings) > 0:
self.job.success = True
self.job.status = "warnings"
message = "Conversion successful with warnings"
else:
self.job.success = True
self.job.status = "success"
message = "Conversion successful"
self.job.message = message
self.job.log_message(message)
self.job.log_message('Finished job {0} at {1}'.format(self.job.job_id, self.job.ended_at.strftime("%Y-%m-%dT%H:%M:%SZ")))
s3_commit_key = 'u/{0}/{1}/{2}'.format(self.job.user_name, self.job.repo_name, self.job.commit_id)
upload_key = s3_commit_key
if multiple_project:
upload_key += "/" + part_id
App.logger.debug('Callback for commit {0}...'.format(s3_commit_key))
# Download the ZIP file of the converted files
converted_zip_url = self.job.output
converted_zip_file = os.path.join(self.temp_dir, converted_zip_url.rpartition('/')[2])
remove(converted_zip_file) # make sure old file not present
download_success = True
App.logger.debug('Downloading converted zip file from {0}...'.format(converted_zip_url))
try:
download_file(converted_zip_url, converted_zip_file)
except:
download_success = False # if multiple project we note fail and move on
if not multiple_project:
remove_tree(self.temp_dir) # cleanup
if self.job.errors is None:
self.job.errors = []
self.job.errors.append("Missing converted file: " + converted_zip_url)
finally:
App.logger.debug('download finished, success={0}'.format(str(download_success)))
self.job.update()
if download_success:
# Unzip the archive
unzip_dir = self.unzip_converted_files(converted_zip_file)
# Upload all files to the cdn_bucket with the key of <user>/<repo_name>/<commit> of the repo
self.upload_converted_files(upload_key, unzip_dir)
if multiple_project:
# Now download the existing build_log.json file, update it and upload it back to S3 as convert_log
build_log_json = self.update_convert_log(s3_commit_key, part_id + "/")
# mark current part as finished
self.cdn_upload_contents({}, s3_commit_key + '/' + part_id + '/finished')
else: # single part conversion
# Now download the existing build_log.json file, update it and upload it back to S3 as convert_log
build_log_json = self.update_convert_log(s3_commit_key)
self.cdn_upload_contents({}, s3_commit_key + '/finished') # flag finished
results = ClientLinterCallback.deploy_if_conversion_finished(s3_commit_key, self.identifier)
if results:
self.all_parts_completed = True
build_log_json = results
remove_tree(self.temp_dir) # cleanup
return build_log_json
def test_delete_job(self):
job = TxJob.get(self.items['job1']['job_id'])
self.assertIsNotNone(job)
job.delete()
job = TxJob.get(self.items['job1']['job_id'])
self.assertIsNone(job)
def test_update_job(self):
job = TxJob.get(self.items['job3']['job_id'])
job.status = 'finished'
job.update()
job = TxJob.get(self.items['job3']['job_id'])
self.assertEqual(job.status, 'finished')
def test_load_job(self):
# Test loading by just giving it the job_id in the constructor
job = TxJob.get('job1')
self.assertEqual(job.identifier, self.items['job1']['identifier'])