def test_callbackSimpleJob_LintNotFinished(self): # given self.unzip_resource_files("id_mat_ulb.zip", convert_finished=True) identifier = self.lint_callback_data['identifier'] # when results = ClientLinterCallback.deploy_if_conversion_finished(self.results_key, identifier) # then self.assertIsNone(results)
def test_callbackMultpleJob_last_job_LintNotFinished(self): # given self.results_key = 'u/tx-manager-test-data/en-ulb/22f3d09f7a' self.lint_callback_data['s3_results_key'] = self.results_key + '/3' self.lint_callback_data['identifier'] = '1234567890/4/3/05-DEU.usfm' self.unzip_resource_files("en_ulb.zip", convert_finished=True) lint_log_path = self.get_source_path(file_name='lint_log.json') file_utils.remove(lint_log_path) identifier = self.lint_callback_data['identifier'] # when results = ClientLinterCallback.deploy_if_conversion_finished(self.results_key, identifier) # then self.assertIsNone(results)
def test_callbackMultpleJob_first_job_BuildLogMissing(self): # given self.results_key = 'u/tx-manager-test-data/en-ulb/22f3d09f7a' self.lint_callback_data['s3_results_key'] = self.results_key + '/0' self.lint_callback_data['identifier'] = '1234567890/4/0/01-GEN.usfm' self.unzip_resource_files("en_ulb.zip", convert_finished=False) build_log_path = self.get_source_path('build_log.json') file_utils.remove(build_log_path) self.finish_convert(self.source_folder) identifier = self.lint_callback_data['identifier'] # when results = ClientLinterCallback.deploy_if_conversion_finished(self.results_key, identifier) # then self.assertIsNone(results)
def test_callbackMultpleJob_last_job_BuildLogMissing(self): # given self.results_key = 'u/tx-manager-test-data/en-ulb/22f3d09f7a' self.lint_callback_data['s3_results_key'] = self.results_key + '/3' self.lint_callback_data['identifier'] = '1234567890/4/3/05-DEU.usfm' self.unzip_resource_files("en_ulb.zip", convert_finished=False) build_log_path = self.get_source_path('build_log.json') file_utils.remove(build_log_path) self.finish_convert(self.source_folder) identifier = self.lint_callback_data['identifier'] # when results = ClientLinterCallback.deploy_if_conversion_finished( self.results_key, identifier) # then self.assertIsNone(results)
def process_callback(self): job_id_parts = self.identifier.split('/') job_id = job_id_parts[0] self.job = TxJob.get(job_id) if not self.job: error = 'No job found for job_id = {0}, identifier = {1}'.format( job_id, self.identifier) App.logger.error(error) raise Exception(error) if len(job_id_parts) == 4: part_count, part_id, book = job_id_parts[1:] App.logger.debug( 'Multiple project, part {0} of {1}, converting book {2}'. format(part_id, part_count, book)) multiple_project = True else: App.logger.debug('Single project') part_id = None multiple_project = False self.job.ended_at = datetime.utcnow() self.job.success = self.success for message in self.log: self.job.log_message(message) for message in self.warnings: self.job.warnings_message(message) for message in self.errors: self.job.error_message(message) if len(self.errors): self.job.log_message('{0} function returned with errors.'.format( self.job.convert_module)) elif len(self.warnings): self.job.log_message('{0} function returned with warnings.'.format( self.job.convert_module)) else: self.job.log_message('{0} function returned successfully.'.format( self.job.convert_module)) if not self.success or len(self.job.errors): self.job.success = False self.job.status = "failed" message = "Conversion failed" App.logger.debug( "Conversion failed, success: {0}, errors: {1}".format( self.success, self.job.errors)) elif len(self.job.warnings) > 0: self.job.success = True self.job.status = "warnings" message = "Conversion successful with warnings" else: self.job.success = True self.job.status = "success" message = "Conversion successful" self.job.message = message self.job.log_message(message) self.job.log_message('Finished job {0} at {1}'.format( self.job.job_id, self.job.ended_at.strftime("%Y-%m-%dT%H:%M:%SZ"))) s3_commit_key = 'u/{0}/{1}/{2}'.format(self.job.user_name, self.job.repo_name, self.job.commit_id) upload_key = s3_commit_key if multiple_project: upload_key += "/" + part_id App.logger.debug('Callback for commit {0}...'.format(s3_commit_key)) # Download the ZIP file of the converted files converted_zip_url = self.job.output converted_zip_file = os.path.join(self.temp_dir, converted_zip_url.rpartition('/')[2]) remove(converted_zip_file) # make sure old file not present download_success = True App.logger.debug('Downloading converted zip file from {0}...'.format( converted_zip_url)) try: download_file(converted_zip_url, converted_zip_file) except: download_success = False # if multiple project we note fail and move on if not multiple_project: remove_tree(self.temp_dir) # cleanup if self.job.errors is None: self.job.errors = [] self.job.errors.append("Missing converted file: " + converted_zip_url) finally: App.logger.debug('download finished, success={0}'.format( str(download_success))) self.job.update() if download_success: # Unzip the archive unzip_dir = self.unzip_converted_files(converted_zip_file) # Upload all files to the cdn_bucket with the key of <user>/<repo_name>/<commit> of the repo self.upload_converted_files(upload_key, unzip_dir) if multiple_project: # Now download the existing build_log.json file, update it and upload it back to S3 as convert_log build_log_json = self.update_convert_log(s3_commit_key, part_id + "/") # mark current part as finished self.cdn_upload_contents({}, s3_commit_key + '/' + part_id + '/finished') else: # single part conversion # Now download the existing build_log.json file, update it and upload it back to S3 as convert_log build_log_json = self.update_convert_log(s3_commit_key) self.cdn_upload_contents({}, s3_commit_key + '/finished') # flag finished results = ClientLinterCallback.deploy_if_conversion_finished( s3_commit_key, self.identifier) if results: self.all_parts_completed = True build_log_json = results remove_tree(self.temp_dir) # cleanup return build_log_json
def process_callback(self): job_id_parts = self.identifier.split('/') job_id = job_id_parts[0] self.job = TxJob.get(job_id) if not self.job: error = 'No job found for job_id = {0}, identifier = {0}'.format(job_id, self.identifier) App.logger.error(error) raise Exception(error) if len(job_id_parts) == 4: part_count, part_id, book = job_id_parts[1:] App.logger.debug('Multiple project, part {0} of {1}, converting book {2}'. format(part_id, part_count, book)) multiple_project = True else: App.logger.debug('Single project') part_id = None multiple_project = False self.job.ended_at = datetime.utcnow() self.job.success = self.success for message in self.log: self.job.log_message(message) for message in self.warnings: self.job.warnings_message(message) for message in self.errors: self.job.error_message(message) if len(self.errors): self.job.log_message('{0} function returned with errors.'.format(self.job.convert_module)) elif len(self.warnings): self.job.log_message('{0} function returned with warnings.'.format(self.job.convert_module)) else: self.job.log_message('{0} function returned successfully.'.format(self.job.convert_module)) if not self.success or len(self.job.errors): self.job.success = False self.job.status = "failed" message = "Conversion failed" App.logger.debug("Conversion failed, success: {0}, errors: {1}".format(self.success, self.job.errors)) elif len(self.job.warnings) > 0: self.job.success = True self.job.status = "warnings" message = "Conversion successful with warnings" else: self.job.success = True self.job.status = "success" message = "Conversion successful" self.job.message = message self.job.log_message(message) self.job.log_message('Finished job {0} at {1}'.format(self.job.job_id, self.job.ended_at.strftime("%Y-%m-%dT%H:%M:%SZ"))) s3_commit_key = 'u/{0}/{1}/{2}'.format(self.job.user_name, self.job.repo_name, self.job.commit_id) upload_key = s3_commit_key if multiple_project: upload_key += "/" + part_id App.logger.debug('Callback for commit {0}...'.format(s3_commit_key)) # Download the ZIP file of the converted files converted_zip_url = self.job.output converted_zip_file = os.path.join(self.temp_dir, converted_zip_url.rpartition('/')[2]) remove(converted_zip_file) # make sure old file not present download_success = True App.logger.debug('Downloading converted zip file from {0}...'.format(converted_zip_url)) try: download_file(converted_zip_url, converted_zip_file) except: download_success = False # if multiple project we note fail and move on if not multiple_project: remove_tree(self.temp_dir) # cleanup if self.job.errors is None: self.job.errors = [] self.job.errors.append("Missing converted file: " + converted_zip_url) finally: App.logger.debug('download finished, success={0}'.format(str(download_success))) self.job.update() if download_success: # Unzip the archive unzip_dir = self.unzip_converted_files(converted_zip_file) # Upload all files to the cdn_bucket with the key of <user>/<repo_name>/<commit> of the repo self.upload_converted_files(upload_key, unzip_dir) if multiple_project: # Now download the existing build_log.json file, update it and upload it back to S3 as convert_log build_log_json = self.update_convert_log(s3_commit_key, part_id + "/") # mark current part as finished self.cdn_upload_contents({}, s3_commit_key + '/' + part_id + '/finished') else: # single part conversion # Now download the existing build_log.json file, update it and upload it back to S3 as convert_log build_log_json = self.update_convert_log(s3_commit_key) self.cdn_upload_contents({}, s3_commit_key + '/finished') # flag finished results = ClientLinterCallback.deploy_if_conversion_finished(s3_commit_key, self.identifier) if results: self.all_parts_completed = True build_log_json = results remove_tree(self.temp_dir) # cleanup return build_log_json