def test_tn_part(self): """ Runs the converter and verifies the output """ # given file_name = 'en_tn.zip' part = '01-GEN.md' # when self.doTransformTn(file_name, part=part) # then self.assertTrue(os.path.isfile(self.out_zip_file), "There was no output zip file produced.") self.assertIsNotNone(self.return_val, "There was no return value.") self.out_dir = tempfile.mkdtemp(prefix='tw_') unzip(self.out_zip_file, self.out_dir) remove(self.out_zip_file) files_to_verify = ['01-GEN.html', 'manifest.yaml', 'index.json'] for dir in BOOK_NUMBERS: book = '{0}-{1}'.format(BOOK_NUMBERS[dir], dir.upper()) file = '{0}.html'.format(book) file_path = os.path.join(self.out_dir, file) if file not in files_to_verify: self.assertFalse(os.path.isfile(file_path), 'file should not be converted: {0}'.format(file)) for file_to_verify in files_to_verify: file_path = os.path.join(self.out_dir, file_to_verify) self.assertTrue(os.path.isfile(file_path), 'file not found: {0}' .format(file_to_verify))
def test_tq(self): """ Runs the converter and verifies the output """ # given file_name = 'en_tq.zip' # when self.doTransformTq(file_name) # then self.assertTrue(os.path.isfile(self.out_zip_file), "There was no output zip file produced.") self.assertIsNotNone(self.return_val, "There was no return value.") self.out_dir = tempfile.mkdtemp(prefix='tw_') unzip(self.out_zip_file, self.out_dir) remove(self.out_zip_file) files_to_verify = ['manifest.yaml', 'index.json'] for book in BOOK_NUMBERS: html_file = '{0}-{1}.html'.format(BOOK_NUMBERS[book], book.upper()) files_to_verify.append(html_file) for file_to_verify in files_to_verify: file_path = os.path.join(self.out_dir, file_to_verify) self.assertTrue(os.path.isfile(file_path), 'file not found: {0}'.format(file_to_verify))
def test_tq(self): """ Runs the converter and verifies the output """ # given file_name = 'en_tq.zip' # when self.doTransformTq(file_name) # then self.assertTrue(os.path.isfile(self.out_zip_file), "There was no output zip file produced.") self.assertIsNotNone(self.return_val, "There was no return value.") self.out_dir = tempfile.mkdtemp(prefix='tw_') unzip(self.out_zip_file, self.out_dir) remove(self.out_zip_file) files_to_verify = ['manifest.yaml', 'index.json'] for book in BOOK_NUMBERS: html_file = '{0}-{1}.html'.format(BOOK_NUMBERS[book], book.upper()) files_to_verify.append(html_file) for file_to_verify in files_to_verify: file_path = os.path.join(self.out_dir, file_to_verify) self.assertTrue(os.path.isfile(file_path), 'file not found: {0}' .format(file_to_verify))
def test_tw(self): """ Runs the converter and verifies the output """ # given file_name = 'en_tw.zip' # when self.doTransformTw(file_name) # then self.assertTrue(os.path.isfile(self.out_zip_file), "There was no output zip file produced.") self.assertIsNotNone(self.return_val, "There was no return value.") self.out_dir = tempfile.mkdtemp(prefix='tw_') unzip(self.out_zip_file, self.out_dir) remove(self.out_zip_file) files_to_verify = [ 'index.json', 'kt.html', 'names.html', 'other.html', 'config.yaml', 'manifest.yaml' ] for file_to_verify in files_to_verify: file_path = os.path.join(self.out_dir, file_to_verify) self.assertTrue(os.path.isfile(file_path), 'file not found: {0}'.format(file_to_verify))
def test_tn_part(self): """ Runs the converter and verifies the output """ # given file_name = 'en_tn.zip' part = '01-GEN.md' # when self.doTransformTn(file_name, part=part) # then self.assertTrue(os.path.isfile(self.out_zip_file), "There was no output zip file produced.") self.assertIsNotNone(self.return_val, "There was no return value.") self.out_dir = tempfile.mkdtemp(prefix='tw_') unzip(self.out_zip_file, self.out_dir) remove(self.out_zip_file) files_to_verify = ['01-GEN.html', 'manifest.yaml', 'index.json'] for dir in BOOK_NUMBERS: book = '{0}-{1}'.format(BOOK_NUMBERS[dir], dir.upper()) file = '{0}.html'.format(book) file_path = os.path.join(self.out_dir, file) if file not in files_to_verify: self.assertFalse( os.path.isfile(file_path), 'file should not be converted: {0}'.format(file)) for file_to_verify in files_to_verify: file_path = os.path.join(self.out_dir, file_to_verify) self.assertTrue(os.path.isfile(file_path), 'file not found: {0}'.format(file_to_verify))
def run(self): """ Call the converters """ success = False try: if not self.input_zip_file or not os.path.exists( self.input_zip_file): # No input zip file yet, so we need to download the archive self.download_archive() # unzip the input archive App.logger.debug("Unzipping {0} to {1}".format( self.input_zip_file, self.files_dir)) unzip(self.input_zip_file, self.files_dir) # convert method called App.logger.debug("Converting files...") if self.convert(): App.logger.debug("Was able to convert {0}".format( self.resource)) # zip the output dir to the output archive App.logger.debug("Adding files in {0} to {1}".format( self.output_dir, self.output_zip_file)) add_contents_to_zip(self.output_zip_file, self.output_dir) remove_tree(self.output_dir) # upload the output archive either to cdn_bucket or to a file (no cdn_bucket) App.logger.debug("Uploading archive to {0}/{1}".format( self.cdn_bucket, self.cdn_file)) self.upload_archive() remove(self.output_zip_file) App.logger.debug("Uploaded") success = True else: self.log.error('Resource {0} currently not supported.'.format( self.resource)) except Exception as e: self.log.error('Conversion process ended abnormally: {0}'.format( e.message)) App.logger.error('{0}: {1}'.format(str(e), traceback.format_exc())) results = { 'identifier': self.identifier, 'success': success and len(self.log.logs['error']) == 0, 'info': self.log.logs['info'], 'warnings': self.log.logs['warning'], 'errors': self.log.logs['error'] } if self.callback is not None: self.callback_results = results self.do_callback(self.callback, self.callback_results) App.logger.debug(results) return results
def test_callbackSimpleJob_convert_not_finished(self): # given self.unzip_resource_files("id_mat_ulb.zip", convert_finished=True) finished_path = self.get_source_path('finished') file_utils.remove(finished_path) self.expected_log_count = 1 self.expected_status = None self.expected_all_parts_completed = False linter_cb = self.mock_client_linter_callback() # when results = linter_cb.process_callback() # then self.validate_results(results, linter_cb)
def test_callbackMultpleJob_last_job_LintNotFinished(self): # given self.results_key = 'u/tx-manager-test-data/en-ulb/22f3d09f7a' self.lint_callback_data['s3_results_key'] = self.results_key + '/3' self.lint_callback_data['identifier'] = '1234567890/4/3/05-DEU.usfm' self.unzip_resource_files("en_ulb.zip", convert_finished=True) lint_log_path = self.get_source_path(file_name='lint_log.json') file_utils.remove(lint_log_path) identifier = self.lint_callback_data['identifier'] # when results = ClientLinterCallback.deploy_if_conversion_finished(self.results_key, identifier) # then self.assertIsNone(results)
def verifyTransform(self, tx, missing_chapters=None): if not missing_chapters: missing_chapters = [] self.assertTrue(os.path.isfile(self.out_zip_file), "There was no output zip file produced.") self.assertIsNotNone(self.return_val, "There was no return value.") self.out_dir = tempfile.mkdtemp(prefix='obs_') unzip(self.out_zip_file, self.out_dir) remove(self.out_zip_file) files_to_verify = [] files_missing = [] for i in range(1, 51): file_name = str(i).zfill(2) + '.html' if not i in missing_chapters: files_to_verify.append(file_name) else: files_missing.append(file_name) for file_to_verify in files_to_verify: file_path = os.path.join(self.out_dir, file_to_verify) contents = self.getContents(file_path) self.assertIsNotNone( contents, 'OBS HTML body contents not found: {0}'.format( os.path.basename(file_path))) for file_to_verify in files_missing: file_path = os.path.join(self.out_dir, file_to_verify) contents = self.getContents(file_path) self.assertIsNone( contents, 'OBS HTML body contents present, but should not be: {0}'. format(os.path.basename(file_path))) self.assertEqual(self.return_val['success'], self.expected_success, "Mismatch in for success boolean") self.assertEqual( len(self.return_val['info']) == 0, self.expected_info_empty, "Mismatch in expected info empty") for warning in self.return_val['warnings']: App.logger.debug("Warning: " + warning) for error in self.return_val['errors']: App.logger.debug("Error: " + error) self.assertEqual(len(self.return_val['warnings']), self.expected_warnings, "Mismatch in expected warnings") self.assertEqual(len(self.return_val['errors']), self.expected_errors, "Mismatch in expected errors")
def test_callbackMultpleJob_last_job_LintNotFinished(self): # given self.results_key = 'u/tx-manager-test-data/en-ulb/22f3d09f7a' self.lint_callback_data['s3_results_key'] = self.results_key + '/3' self.lint_callback_data['identifier'] = '1234567890/4/3/05-DEU.usfm' self.unzip_resource_files("en_ulb.zip", convert_finished=True) lint_log_path = self.get_source_path(file_name='lint_log.json') file_utils.remove(lint_log_path) identifier = self.lint_callback_data['identifier'] # when results = ClientLinterCallback.deploy_if_conversion_finished( self.results_key, identifier) # then self.assertIsNone(results)
def test_callbackMultpleJob_first_job_BuildLogMissing(self): # given self.results_key = 'u/tx-manager-test-data/en-ulb/22f3d09f7a' self.lint_callback_data['s3_results_key'] = self.results_key + '/0' self.lint_callback_data['identifier'] = '1234567890/4/0/01-GEN.usfm' self.unzip_resource_files("en_ulb.zip", convert_finished=False) build_log_path = self.get_source_path('build_log.json') file_utils.remove(build_log_path) self.finish_convert(self.source_folder) identifier = self.lint_callback_data['identifier'] # when results = ClientLinterCallback.deploy_if_conversion_finished(self.results_key, identifier) # then self.assertIsNone(results)
def test_callbackMultpleJob_first_job_BuildLogMissing(self): # given self.results_key = 'u/tx-manager-test-data/en-ulb/22f3d09f7a' self.lint_callback_data['s3_results_key'] = self.results_key + '/0' self.lint_callback_data['identifier'] = '1234567890/4/0/01-GEN.usfm' self.unzip_resource_files("en_ulb.zip", convert_finished=False) build_log_path = self.get_source_path('build_log.json') file_utils.remove(build_log_path) self.finish_convert(self.source_folder) identifier = self.lint_callback_data['identifier'] # when results = ClientLinterCallback.deploy_if_conversion_finished( self.results_key, identifier) # then self.assertIsNone(results)
def test_ta(self): """ Runs the converter and verifies the output """ file_name = 'en_ta.zip' self.doTransformTa(file_name) self.assertTrue(os.path.isfile(self.out_zip_file), "There was no output zip file produced.") self.assertIsNotNone(self.return_val, "There was no return value.") self.out_dir = tempfile.mkdtemp(prefix='ta_') unzip(self.out_zip_file, self.out_dir) remove(self.out_zip_file) files_to_verify = ['checking.html', 'checking-toc.yaml', 'intro.html', 'intro-toc.yaml', 'process.html', 'process-toc.yaml', 'translate.html', 'translate-toc.yaml'] for file_to_verify in files_to_verify: file_path = os.path.join(self.out_dir, file_to_verify) self.assertTrue(os.path.isfile(file_path), 'file not found: {0}' .format(file_to_verify))
def test_ta(self): """ Runs the converter and verifies the output """ file_name = 'en_ta.zip' self.doTransformTa(file_name) self.assertTrue(os.path.isfile(self.out_zip_file), "There was no output zip file produced.") self.assertIsNotNone(self.return_val, "There was no return value.") self.out_dir = tempfile.mkdtemp(prefix='ta_') unzip(self.out_zip_file, self.out_dir) remove(self.out_zip_file) files_to_verify = [ 'checking.html', 'checking-toc.yaml', 'intro.html', 'intro-toc.yaml', 'process.html', 'process-toc.yaml', 'translate.html', 'translate-toc.yaml' ] for file_to_verify in files_to_verify: file_path = os.path.join(self.out_dir, file_to_verify) self.assertTrue(os.path.isfile(file_path), 'file not found: {0}'.format(file_to_verify))
def test_run(self): """Runs the converter and verifies the output.""" # test with the English OBS zip_file = os.path.join(self.resources_dir, 'en-obs.zip') zip_file = self.make_duplicate_zip_that_can_be_deleted(zip_file) out_zip_file = tempfile.mktemp(prefix="en-obs", suffix=".zip") with closing(Md2HtmlConverter('', 'obs', out_zip_file)) as tx: tx.input_zip_file = zip_file tx.run() # verify the output self.assertTrue(os.path.isfile(out_zip_file), "There was no output zip file produced.") self.out_dir = tempfile.mkdtemp(prefix='obs_') unzip(out_zip_file, self.out_dir) remove(out_zip_file) files_to_verify = [] for i in range(1, 51): files_to_verify.append(str(i).zfill(2) + '.html') for file_to_verify in files_to_verify: file_name = os.path.join(self.out_dir, file_to_verify) self.assertTrue(os.path.isfile(file_name), 'OBS HTML file not found: {0}'.format(file_name))
def test_tw(self): """ Runs the converter and verifies the output """ # given file_name = 'en_tw.zip' # when self.doTransformTw(file_name) # then self.assertTrue(os.path.isfile(self.out_zip_file), "There was no output zip file produced.") self.assertIsNotNone(self.return_val, "There was no return value.") self.out_dir = tempfile.mkdtemp(prefix='tw_') unzip(self.out_zip_file, self.out_dir) remove(self.out_zip_file) files_to_verify = ['index.json', 'kt.html', 'names.html', 'other.html', 'config.yaml', 'manifest.yaml'] for file_to_verify in files_to_verify: file_path = os.path.join(self.out_dir, file_to_verify) self.assertTrue(os.path.isfile(file_path), 'file not found: {0}' .format(file_to_verify))
def verifyTransform(self, tx, missing_chapters=None): if not missing_chapters: missing_chapters = [] self.assertTrue(os.path.isfile(self.out_zip_file), "There was no output zip file produced.") self.assertIsNotNone(self.return_val, "There was no return value.") self.out_dir = tempfile.mkdtemp(prefix='obs_') unzip(self.out_zip_file, self.out_dir) remove(self.out_zip_file) files_to_verify = [] files_missing = [] for i in range(1, 51): file_name = str(i).zfill(2) + '.html' if not i in missing_chapters: files_to_verify.append(file_name) else: files_missing.append(file_name) for file_to_verify in files_to_verify: file_path = os.path.join(self.out_dir, file_to_verify) contents = self.getContents(file_path) self.assertIsNotNone(contents, 'OBS HTML body contents not found: {0}'.format(os.path.basename(file_path))) for file_to_verify in files_missing: file_path = os.path.join(self.out_dir, file_to_verify) contents = self.getContents(file_path) self.assertIsNone(contents, 'OBS HTML body contents present, but should not be: {0}'.format(os.path.basename(file_path))) self.assertEqual(self.return_val['success'], self.expected_success, "Mismatch in for success boolean") self.assertEqual(len(self.return_val['info']) == 0, self.expected_info_empty, "Mismatch in expected info empty") for warning in self.return_val['warnings']: App.logger.debug("Warning: " + warning) for error in self.return_val['errors']: App.logger.debug("Error: " + error) self.assertEqual(len(self.return_val['warnings']), self.expected_warnings, "Mismatch in expected warnings") self.assertEqual(len(self.return_val['errors']), self.expected_errors, "Mismatch in expected errors")
def process_callback(self): job_id_parts = self.identifier.split('/') job_id = job_id_parts[0] self.job = TxJob.get(job_id) if not self.job: error = 'No job found for job_id = {0}, identifier = {0}'.format(job_id, self.identifier) App.logger.error(error) raise Exception(error) if len(job_id_parts) == 4: part_count, part_id, book = job_id_parts[1:] App.logger.debug('Multiple project, part {0} of {1}, converting book {2}'. format(part_id, part_count, book)) multiple_project = True else: App.logger.debug('Single project') part_id = None multiple_project = False self.job.ended_at = datetime.utcnow() self.job.success = self.success for message in self.log: self.job.log_message(message) for message in self.warnings: self.job.warnings_message(message) for message in self.errors: self.job.error_message(message) if len(self.errors): self.job.log_message('{0} function returned with errors.'.format(self.job.convert_module)) elif len(self.warnings): self.job.log_message('{0} function returned with warnings.'.format(self.job.convert_module)) else: self.job.log_message('{0} function returned successfully.'.format(self.job.convert_module)) if not self.success or len(self.job.errors): self.job.success = False self.job.status = "failed" message = "Conversion failed" App.logger.debug("Conversion failed, success: {0}, errors: {1}".format(self.success, self.job.errors)) elif len(self.job.warnings) > 0: self.job.success = True self.job.status = "warnings" message = "Conversion successful with warnings" else: self.job.success = True self.job.status = "success" message = "Conversion successful" self.job.message = message self.job.log_message(message) self.job.log_message('Finished job {0} at {1}'.format(self.job.job_id, self.job.ended_at.strftime("%Y-%m-%dT%H:%M:%SZ"))) s3_commit_key = 'u/{0}/{1}/{2}'.format(self.job.user_name, self.job.repo_name, self.job.commit_id) upload_key = s3_commit_key if multiple_project: upload_key += "/" + part_id App.logger.debug('Callback for commit {0}...'.format(s3_commit_key)) # Download the ZIP file of the converted files converted_zip_url = self.job.output converted_zip_file = os.path.join(self.temp_dir, converted_zip_url.rpartition('/')[2]) remove(converted_zip_file) # make sure old file not present download_success = True App.logger.debug('Downloading converted zip file from {0}...'.format(converted_zip_url)) try: download_file(converted_zip_url, converted_zip_file) except: download_success = False # if multiple project we note fail and move on if not multiple_project: remove_tree(self.temp_dir) # cleanup if self.job.errors is None: self.job.errors = [] self.job.errors.append("Missing converted file: " + converted_zip_url) finally: App.logger.debug('download finished, success={0}'.format(str(download_success))) self.job.update() if download_success: # Unzip the archive unzip_dir = self.unzip_converted_files(converted_zip_file) # Upload all files to the cdn_bucket with the key of <user>/<repo_name>/<commit> of the repo self.upload_converted_files(upload_key, unzip_dir) if multiple_project: # Now download the existing build_log.json file, update it and upload it back to S3 as convert_log build_log_json = self.update_convert_log(s3_commit_key, part_id + "/") # mark current part as finished self.cdn_upload_contents({}, s3_commit_key + '/' + part_id + '/finished') else: # single part conversion # Now download the existing build_log.json file, update it and upload it back to S3 as convert_log build_log_json = self.update_convert_log(s3_commit_key) self.cdn_upload_contents({}, s3_commit_key + '/finished') # flag finished results = ClientLinterCallback.deploy_if_conversion_finished(s3_commit_key, self.identifier) if results: self.all_parts_completed = True build_log_json = results remove_tree(self.temp_dir) # cleanup return build_log_json
def close(self): """delete temp files""" remove_tree(self.download_dir) remove_tree(self.files_dir) remove_tree(self.output_dir) remove(self.output_zip_file)
def tearDown(self): """Runs after each test.""" # delete temp files remove_tree(self.out_dir) remove(self.out_zip_file)
def process_callback(self): job_id_parts = self.identifier.split('/') job_id = job_id_parts[0] self.job = TxJob.get(job_id) if not self.job: error = 'No job found for job_id = {0}, identifier = {1}'.format( job_id, self.identifier) App.logger.error(error) raise Exception(error) if len(job_id_parts) == 4: part_count, part_id, book = job_id_parts[1:] App.logger.debug( 'Multiple project, part {0} of {1}, converting book {2}'. format(part_id, part_count, book)) multiple_project = True else: App.logger.debug('Single project') part_id = None multiple_project = False self.job.ended_at = datetime.utcnow() self.job.success = self.success for message in self.log: self.job.log_message(message) for message in self.warnings: self.job.warnings_message(message) for message in self.errors: self.job.error_message(message) if len(self.errors): self.job.log_message('{0} function returned with errors.'.format( self.job.convert_module)) elif len(self.warnings): self.job.log_message('{0} function returned with warnings.'.format( self.job.convert_module)) else: self.job.log_message('{0} function returned successfully.'.format( self.job.convert_module)) if not self.success or len(self.job.errors): self.job.success = False self.job.status = "failed" message = "Conversion failed" App.logger.debug( "Conversion failed, success: {0}, errors: {1}".format( self.success, self.job.errors)) elif len(self.job.warnings) > 0: self.job.success = True self.job.status = "warnings" message = "Conversion successful with warnings" else: self.job.success = True self.job.status = "success" message = "Conversion successful" self.job.message = message self.job.log_message(message) self.job.log_message('Finished job {0} at {1}'.format( self.job.job_id, self.job.ended_at.strftime("%Y-%m-%dT%H:%M:%SZ"))) s3_commit_key = 'u/{0}/{1}/{2}'.format(self.job.user_name, self.job.repo_name, self.job.commit_id) upload_key = s3_commit_key if multiple_project: upload_key += "/" + part_id App.logger.debug('Callback for commit {0}...'.format(s3_commit_key)) # Download the ZIP file of the converted files converted_zip_url = self.job.output converted_zip_file = os.path.join(self.temp_dir, converted_zip_url.rpartition('/')[2]) remove(converted_zip_file) # make sure old file not present download_success = True App.logger.debug('Downloading converted zip file from {0}...'.format( converted_zip_url)) try: download_file(converted_zip_url, converted_zip_file) except: download_success = False # if multiple project we note fail and move on if not multiple_project: remove_tree(self.temp_dir) # cleanup if self.job.errors is None: self.job.errors = [] self.job.errors.append("Missing converted file: " + converted_zip_url) finally: App.logger.debug('download finished, success={0}'.format( str(download_success))) self.job.update() if download_success: # Unzip the archive unzip_dir = self.unzip_converted_files(converted_zip_file) # Upload all files to the cdn_bucket with the key of <user>/<repo_name>/<commit> of the repo self.upload_converted_files(upload_key, unzip_dir) if multiple_project: # Now download the existing build_log.json file, update it and upload it back to S3 as convert_log build_log_json = self.update_convert_log(s3_commit_key, part_id + "/") # mark current part as finished self.cdn_upload_contents({}, s3_commit_key + '/' + part_id + '/finished') else: # single part conversion # Now download the existing build_log.json file, update it and upload it back to S3 as convert_log build_log_json = self.update_convert_log(s3_commit_key) self.cdn_upload_contents({}, s3_commit_key + '/finished') # flag finished results = ClientLinterCallback.deploy_if_conversion_finished( s3_commit_key, self.identifier) if results: self.all_parts_completed = True build_log_json = results remove_tree(self.temp_dir) # cleanup return build_log_json