def close(self) -> None:
"""
Delete temp files (except in debug mode)
"""
# print("Converter close() was called!")
if prefix and debug_mode_flag:
AppSettings.logger.debug(
f"Converter temp folder '{self.converter_dir}' has been left on disk for debugging!"
)
else:
try:
remove_tree(self.converter_dir)
except AttributeError:
pass # no such variable
try:
remove_file(self.output_zip_file)
except AttributeError:
pass # no such variable
def verifyTransform(self, tx, missing_chapters=None):
if not missing_chapters:
missing_chapters = []
self.assertTrue(os.path.isfile(self.out_zip_file), "There was no output zip file produced.")
self.assertIsNotNone(self.return_val, "There was no return value.")
self.out_dir = tempfile.mkdtemp(prefix='tX_test_obs_')
unzip(self.out_zip_file, self.out_dir)
remove_file(self.out_zip_file)
files_to_verify = []
files_missing = []
for i in range(1, 51):
file_name = str(i).zfill(2) + '.html'
if not i in missing_chapters:
files_to_verify.append(file_name)
else:
files_missing.append(file_name)
for file_to_verify in files_to_verify:
file_path = os.path.join(self.out_dir, file_to_verify)
contents = self.getContents(file_path)
self.assertIsNotNone(contents, 'OBS HTML body contents not found: {0}'.format(os.path.basename(file_path)))
for file_to_verify in files_missing:
file_path = os.path.join(self.out_dir, file_to_verify)
contents = self.getContents(file_path)
self.assertIsNone(contents, 'OBS HTML body contents present, but should not be: {0}'.format(os.path.basename(file_path)))
self.assertEqual(self.return_val['success'], self.expected_success, "Mismatch in for success boolean")
self.assertEqual(len(self.return_val['info']) == 0, self.expected_info_empty, "Mismatch in expected info empty")
for warning in self.return_val['warnings']:
AppSettings.logger.debug("Warning: " + warning)
for error in self.return_val['errors']:
AppSettings.logger.debug("Error: " + error)
self.assertEqual(len(self.return_val['warnings']), self.expected_warnings, "Mismatch in expected warnings")
self.assertEqual(len(self.return_val['errors']), self.expected_errors, "Mismatch in expected errors")
def tearDown(self):
"""Runs after each test."""
# delete temp files
remove_tree(self.out_dir)
remove_file(self.out_zip_file)
def run(self) -> Dict[str, Any]:
"""
Call the converters
"""
success = False
if os.path.isdir(self.source_dir):
self.files_dir = self.source_dir # TODO: This can be cleaned up later
try:
# if not self.input_zip_file or not os.path.exists(self.input_zip_file):
# # No input zip file yet, so we need to download the archive
# self.download_archive()
# # unzip the input archive
# AppSettings.logger.debug(f"Converter unzipping {self.input_zip_file} to {self.files_dir}")
# unzip(self.input_zip_file, self.files_dir)
# convert method called
AppSettings.logger.debug(
f"Converting files from {self.files_dir}…")
if self.convert():
#AppSettings.logger.debug(f"Was able to convert {self.resource}")
# Zip the output dir to the output archive
#AppSettings.logger.debug(f"Converter adding files in {self.output_dir} to {self.output_zip_file}")
add_contents_to_zip(self.output_zip_file, self.output_dir)
# remove_tree(self.output_dir) # Done in converter.close()
# Upload the output archive either to cdn_bucket or to a file (no cdn_bucket)
AppSettings.logger.info(
f"Converter uploading output archive to {self.cdn_file_key} …"
)
if self.cdn_file_key:
self.upload_archive()
AppSettings.logger.debug(
f"Uploaded converted files (using '{self.cdn_file_key}')."
)
else:
AppSettings.logger.debug(
"No converted file upload requested.")
remove_file(self.output_zip_file)
success = True
else:
self.log.error(
f"Resource type '{self.repo_subject}' currently not supported."
)
except Exception as e:
self.log.error(f"Conversion process ended abnormally: {e}")
AppSettings.logger.debug(
f"Converter failure: {traceback.format_exc()}")
results = {
'identifier': self.identifier,
'success': success and len(self.log.logs['error']) == 0,
'info': self.log.logs['info'],
'warnings': self.log.logs['warning'],
'errors': self.log.logs['error']
}
# if self.callback is not None:
# self.callback_results = results
# self.do_callback(self.callback, self.callback_results)
# AppSettings.logger.debug(results)
return results
def do_post_processing(self) -> Tuple[Optional[str], Dict[str,Any]]:
AppSettings.logger.debug(f"ClientConverterCallback.do_post_processing()…")
self.job.ended_at = datetime.utcnow()
self.job.success = self.success
for message in self.log:
self.job.log_message(message)
for message in self.warnings:
self.job.warnings_message(message)
for message in self.errors:
self.job.error_message(message)
if self.errors:
self.job.log_message(f"{self.job.convert_module} function returned with errors.")
elif self.warnings:
self.job.log_message(f"{self.job.convert_module} function returned with warnings.")
else:
self.job.log_message(f"{self.job.convert_module} function returned successfully.")
if not self.success or self.job.errors:
self.job.success = False
self.job.status = 'failed'
message = "Conversion failed"
AppSettings.logger.debug(f"Conversion failed, success: {self.success}, errors: {self.job.errors}")
elif self.job.warnings:
self.job.success = True
self.job.status = 'warnings'
message = "Conversion successful with warnings."
else:
self.job.success = True
self.job.status = 'success'
message = "Conversion successful."
self.job.message = message
self.job.log_message(message)
self.job.log_message(f"Finished job {self.job.job_id} at {self.job.ended_at.strftime('%Y-%m-%dT%H:%M:%SZ')}")
s3_commit_key = f'u/{self.job.repo_owner_username}/{self.job.repo_name}/{self.job.commit_id}'
# AppSettings.logger.debug(f"Callback for commit = '{s3_commit_key}'")
upload_key = s3_commit_key
# Download the ZIP file of the converted files
converted_zip_url = self.job.output
converted_zip_file = os.path.join(self.temp_dir, converted_zip_url.rpartition('/')[2])
remove_file(converted_zip_file) # make sure old file not present
download_success = True
AppSettings.logger.debug(f"Downloading converted zip file from {converted_zip_url} …")
try:
download_file(converted_zip_url, converted_zip_file)
except:
download_success = False # if multiple project we note fail and move on
# if not multiple_project:
# if prefix and debug_mode_flag:
# AppSettings.logger.debug(f"Temp folder '{self.temp_dir}' has been left on disk for debugging!")
# else:
# remove_tree(self.temp_dir) # cleanup
if self.job.errors is None:
self.job.errors = []
message = f"Missing converted file: {converted_zip_url}"
AppSettings.logger.debug(message)
if not self.job.errors or not self.job.errors[0].startswith("No converter "):
# Missing file error is irrelevant if no conversion was attempted
self.job.errors.append(message)
finally:
AppSettings.logger.debug(f"Download finished, success={download_success}.")
# self.job.update()
if download_success:
# Unzip the archive
unzip_dirpath = self.unzip_converted_files(converted_zip_file)
# Upload all files to the cdn_bucket with the key of <user>/<repo_name>/<commit> of the repo
# This is required for the print function to work
self.upload_converted_files_to_CDN(upload_key, unzip_dirpath)
else:
unzip_dirpath = None # So we have something to return (fail later -- is that an advantage?)
# TODO: Do we really need this now?
# Now download the existing build_log.json file, update it and upload it back to S3 as convert_log
# NOTE: Do we need this -- disabled 25Feb2019
# build_log_json = self.update_convert_log(s3_commit_key)
# self.cdn_upload_contents({}, s3_commit_key + '/finished') # flag finished
converter_build_log = self.make_our_build_log()
# print("Got ConPP converter_build_log", converter_build_log)
# NOTE: Disabled 4Mar2019 coz moved to callback.py
# results = ClientLinterCallback.deploy_if_conversion_finished(s3_commit_key, self.identifier)
# if results:
# self.all_parts_completed = True
# build_log_json = results
# if prefix and debug_mode_flag:
# AppSettings.logger.debug(f"Temp folder '{self.temp_dir}' has been left on disk for debugging!")
# else:
# remove_tree(self.temp_dir) # cleanup
return unzip_dirpath, converter_build_log