def close(self) -> None: """ Delete temp files (except in debug mode) """ # print("Converter close() was called!") if prefix and debug_mode_flag: AppSettings.logger.debug( f"Converter temp folder '{self.converter_dir}' has been left on disk for debugging!" ) else: try: remove_tree(self.converter_dir) except AttributeError: pass # no such variable try: remove_file(self.output_zip_file) except AttributeError: pass # no such variable
def verifyTransform(self, tx, missing_chapters=None): if not missing_chapters: missing_chapters = [] self.assertTrue(os.path.isfile(self.out_zip_file), "There was no output zip file produced.") self.assertIsNotNone(self.return_val, "There was no return value.") self.out_dir = tempfile.mkdtemp(prefix='tX_test_obs_') unzip(self.out_zip_file, self.out_dir) remove_file(self.out_zip_file) files_to_verify = [] files_missing = [] for i in range(1, 51): file_name = str(i).zfill(2) + '.html' if not i in missing_chapters: files_to_verify.append(file_name) else: files_missing.append(file_name) for file_to_verify in files_to_verify: file_path = os.path.join(self.out_dir, file_to_verify) contents = self.getContents(file_path) self.assertIsNotNone(contents, 'OBS HTML body contents not found: {0}'.format(os.path.basename(file_path))) for file_to_verify in files_missing: file_path = os.path.join(self.out_dir, file_to_verify) contents = self.getContents(file_path) self.assertIsNone(contents, 'OBS HTML body contents present, but should not be: {0}'.format(os.path.basename(file_path))) self.assertEqual(self.return_val['success'], self.expected_success, "Mismatch in for success boolean") self.assertEqual(len(self.return_val['info']) == 0, self.expected_info_empty, "Mismatch in expected info empty") for warning in self.return_val['warnings']: AppSettings.logger.debug("Warning: " + warning) for error in self.return_val['errors']: AppSettings.logger.debug("Error: " + error) self.assertEqual(len(self.return_val['warnings']), self.expected_warnings, "Mismatch in expected warnings") self.assertEqual(len(self.return_val['errors']), self.expected_errors, "Mismatch in expected errors")
def tearDown(self): """Runs after each test.""" # delete temp files remove_tree(self.out_dir) remove_file(self.out_zip_file)
def run(self) -> Dict[str, Any]: """ Call the converters """ success = False if os.path.isdir(self.source_dir): self.files_dir = self.source_dir # TODO: This can be cleaned up later try: # if not self.input_zip_file or not os.path.exists(self.input_zip_file): # # No input zip file yet, so we need to download the archive # self.download_archive() # # unzip the input archive # AppSettings.logger.debug(f"Converter unzipping {self.input_zip_file} to {self.files_dir}") # unzip(self.input_zip_file, self.files_dir) # convert method called AppSettings.logger.debug( f"Converting files from {self.files_dir}…") if self.convert(): #AppSettings.logger.debug(f"Was able to convert {self.resource}") # Zip the output dir to the output archive #AppSettings.logger.debug(f"Converter adding files in {self.output_dir} to {self.output_zip_file}") add_contents_to_zip(self.output_zip_file, self.output_dir) # remove_tree(self.output_dir) # Done in converter.close() # Upload the output archive either to cdn_bucket or to a file (no cdn_bucket) AppSettings.logger.info( f"Converter uploading output archive to {self.cdn_file_key} …" ) if self.cdn_file_key: self.upload_archive() AppSettings.logger.debug( f"Uploaded converted files (using '{self.cdn_file_key}')." ) else: AppSettings.logger.debug( "No converted file upload requested.") remove_file(self.output_zip_file) success = True else: self.log.error( f"Resource type '{self.repo_subject}' currently not supported." ) except Exception as e: self.log.error(f"Conversion process ended abnormally: {e}") AppSettings.logger.debug( f"Converter failure: {traceback.format_exc()}") results = { 'identifier': self.identifier, 'success': success and len(self.log.logs['error']) == 0, 'info': self.log.logs['info'], 'warnings': self.log.logs['warning'], 'errors': self.log.logs['error'] } # if self.callback is not None: # self.callback_results = results # self.do_callback(self.callback, self.callback_results) # AppSettings.logger.debug(results) return results
def do_post_processing(self) -> Tuple[Optional[str], Dict[str,Any]]: AppSettings.logger.debug(f"ClientConverterCallback.do_post_processing()…") self.job.ended_at = datetime.utcnow() self.job.success = self.success for message in self.log: self.job.log_message(message) for message in self.warnings: self.job.warnings_message(message) for message in self.errors: self.job.error_message(message) if self.errors: self.job.log_message(f"{self.job.convert_module} function returned with errors.") elif self.warnings: self.job.log_message(f"{self.job.convert_module} function returned with warnings.") else: self.job.log_message(f"{self.job.convert_module} function returned successfully.") if not self.success or self.job.errors: self.job.success = False self.job.status = 'failed' message = "Conversion failed" AppSettings.logger.debug(f"Conversion failed, success: {self.success}, errors: {self.job.errors}") elif self.job.warnings: self.job.success = True self.job.status = 'warnings' message = "Conversion successful with warnings." else: self.job.success = True self.job.status = 'success' message = "Conversion successful." self.job.message = message self.job.log_message(message) self.job.log_message(f"Finished job {self.job.job_id} at {self.job.ended_at.strftime('%Y-%m-%dT%H:%M:%SZ')}") s3_commit_key = f'u/{self.job.repo_owner_username}/{self.job.repo_name}/{self.job.commit_id}' # AppSettings.logger.debug(f"Callback for commit = '{s3_commit_key}'") upload_key = s3_commit_key # Download the ZIP file of the converted files converted_zip_url = self.job.output converted_zip_file = os.path.join(self.temp_dir, converted_zip_url.rpartition('/')[2]) remove_file(converted_zip_file) # make sure old file not present download_success = True AppSettings.logger.debug(f"Downloading converted zip file from {converted_zip_url} …") try: download_file(converted_zip_url, converted_zip_file) except: download_success = False # if multiple project we note fail and move on # if not multiple_project: # if prefix and debug_mode_flag: # AppSettings.logger.debug(f"Temp folder '{self.temp_dir}' has been left on disk for debugging!") # else: # remove_tree(self.temp_dir) # cleanup if self.job.errors is None: self.job.errors = [] message = f"Missing converted file: {converted_zip_url}" AppSettings.logger.debug(message) if not self.job.errors or not self.job.errors[0].startswith("No converter "): # Missing file error is irrelevant if no conversion was attempted self.job.errors.append(message) finally: AppSettings.logger.debug(f"Download finished, success={download_success}.") # self.job.update() if download_success: # Unzip the archive unzip_dirpath = self.unzip_converted_files(converted_zip_file) # Upload all files to the cdn_bucket with the key of <user>/<repo_name>/<commit> of the repo # This is required for the print function to work self.upload_converted_files_to_CDN(upload_key, unzip_dirpath) else: unzip_dirpath = None # So we have something to return (fail later -- is that an advantage?) # TODO: Do we really need this now? # Now download the existing build_log.json file, update it and upload it back to S3 as convert_log # NOTE: Do we need this -- disabled 25Feb2019 # build_log_json = self.update_convert_log(s3_commit_key) # self.cdn_upload_contents({}, s3_commit_key + '/finished') # flag finished converter_build_log = self.make_our_build_log() # print("Got ConPP converter_build_log", converter_build_log) # NOTE: Disabled 4Mar2019 coz moved to callback.py # results = ClientLinterCallback.deploy_if_conversion_finished(s3_commit_key, self.identifier) # if results: # self.all_parts_completed = True # build_log_json = results # if prefix and debug_mode_flag: # AppSettings.logger.debug(f"Temp folder '{self.temp_dir}' has been left on disk for debugging!") # else: # remove_tree(self.temp_dir) # cleanup return unzip_dirpath, converter_build_log