def write_data_to_file_and_upload_to_CDN(self, output_dir:str, s3_commit_key:str, fname:str, data:Union[str, Dict[str,Any]]) -> None: out_file = os.path.join(output_dir, fname) write_file(out_file, data) key = s3_commit_key + '/' + fname AppSettings.logger.debug(f"Uploading '{fname}' to {AppSettings.cdn_bucket_name} {key} …") AppSettings.cdn_s3_handler().upload_file(out_file, key, cache_time=0)
def test_reset_app(self): default_name = AppSettings.name AppSettings(name='test-name') AppSettings() self.assertEqual(AppSettings.name, default_name) AppSettings.name = 'test-name-2' AppSettings(name='test-name-2', reset=False) self.assertNotEqual(AppSettings.name, default_name)
def set_deployed_flags(self, project_key, part_count, skip=-1): tempf = tempfile.mktemp(prefix="temp", suffix="deployed") file_utils.write_file(tempf, ' ') for i in range(0, part_count): if i != skip: key = '{0}/{1}/deployed'.format(project_key, i) AppSettings.cdn_s3_handler().upload_file(tempf, key, cache_time=0) os.remove(tempf)
def test_db(self): AppSettings(db_connection_string='sqlite:///:memory:') AppSettings.db_create_tables([User.__table__]) user = User(name='ed', fullname='Edward Scissorhands', password='******') user.insert() user_from_db = User.get(name='ed') self.assertIsNotNone(user_from_db) self.assertEqual(user_from_db.password, '12345')
def get(cls, *args, **kwargs): """ :param args: :param kwargs: :return TxModel: """ if args: kwargs[inspect(cls).primary_key[0].name] = args[0] item = cls.query(**kwargs).first() AppSettings.db().close() return item
def test_prefix_vars(self): AppSettings(prefix='') self.assertEqual(AppSettings.cdn_bucket_name, 'cdn.door43.org') # self.assertEqual(AppSettings.api_url, 'https://api.door43.org') AppSettings(prefix='test-') self.assertEqual(AppSettings.cdn_bucket_name, 'test-cdn.door43.org') # self.assertEqual(AppSettings.api_url, 'https://test-api.door43.org') AppSettings(prefix='test2-') self.assertEqual(AppSettings.cdn_bucket_name, 'test2-cdn.door43.org') # self.assertEqual(AppSettings.api_url, 'https://test2-api.door43.org') AppSettings(prefix='') self.assertEqual(AppSettings.cdn_bucket_name, 'cdn.door43.org')
def upload_converted_files_to_CDN(s3_commit_key:str, unzip_dir:str) -> None: """ Uploads the converted (but not templated) files to the cdn.door43.org bucket NOTE: These are used from there by the Print button/function. """ AppSettings.logger.info(f"Uploading converted files from {unzip_dir} to {prefix}CDN {s3_commit_key} …") for root, _dirs, files in os.walk(unzip_dir): for filename in sorted(files): filepath = os.path.join(root, filename) key = s3_commit_key + filepath.replace(unzip_dir, '') AppSettings.logger.debug(f"Uploading {filename} to {prefix}CDN {key} …") AppSettings.cdn_s3_handler().upload_file(filepath, key, cache_time=0)
def upload_archive(self) -> None: """ Uploads self.output_zip_file """ #AppSettings.logger.debug("converter.upload_archive()") if self.cdn_file_key and os.path.isdir( os.path.dirname(self.cdn_file_key)): #AppSettings.logger.debug("converter.upload_archive() doing copy") copy(self.output_zip_file, self.cdn_file_key) elif AppSettings.cdn_s3_handler(): #AppSettings.logger.debug("converter.upload_archive() using S3 handler") AppSettings.cdn_s3_handler().upload_file(self.output_zip_file, self.cdn_file_key, cache_time=0)
def setUp(self): """Runs before each test.""" AppSettings(prefix=f'{self._testMethodName}-') AppSettings.cdn_s3_handler().create_bucket() AppSettings.door43_s3_handler().create_bucket() self.temp_dir = tempfile.mkdtemp(prefix='Door43_test_project_deployer') self.deployer = ProjectDeployer(self.temp_dir) TdLanguage.language_list = { 'aa': TdLanguage({ 'gw': False, 'ld': 'ltr', 'ang': 'Afar', 'lc': 'aa', 'ln': 'Afaraf', 'lr': 'Africa', 'pk': 6 }), 'en': TdLanguage({ 'gw': True, 'ld': 'ltr', 'ang': 'English', 'lc': 'en', 'ln': 'English', 'lr': 'Europe', 'pk': 1747 }), 'es': TdLanguage({ 'gw': True, 'ld': 'ltr', 'ang': 'Spanish', 'lc': 'es', 'ln': 'espa\xf1ol', 'lr': 'Europe', 'pk': 1776 }), 'fr': TdLanguage({ 'gw': True, 'ld': 'ltr', 'ang': 'French', 'lc': 'fr', 'ln': 'fran\xe7ais, langue fran\xe7aise', 'lr': 'Europe', 'pk': 1868 }) }
def test_construction_connection_string(self): """ Test the construction of the connection string with multiple attributes """ AppSettings(db_protocol='protocol', db_user='******', db_pass='******', db_end_point='my.endpoint.url', db_port='9999', db_name='db', db_connection_string_params='charset=utf8', auto_setup_db=False) expected = "protocol://*****:*****@my.endpoint.url:9999/db?charset=utf8" connection_str = AppSettings.construct_connection_string() self.assertEqual(connection_str, expected)
def get_templater_index(s3_commit_key:str, index_json_fname:str) -> Dict[str,Any]: index_json = AppSettings.cdn_s3_handler().get_json(s3_commit_key + '/' + index_json_fname) if not index_json: index_json['titles'] = {} index_json['chapters'] = {} index_json['book_codes'] = {} return index_json
def validate_bible_results(self, ret, build_log_key, expect_success, output_key): self.assertEqual(ret, expect_success) if expect_success: if output_key: self.assertTrue( AppSettings.door43_s3_handler().key_exists(output_key))
def test_update_manifest(self): repo_name = self.items['francis/fr_ulb']['repo_name'] user_name = self.items['francis/fr_ulb']['user_name'] tx_manifest = TxManifest.get(repo_name=repo_name, user_name=user_name) # Update by setting fields and calling update() tx_manifest.resource_id = 'udb' tx_manifest.title = 'Unlocked Dynamic Bible' tx_manifest.update() manifest_from_db = TxManifest.get(repo_name=repo_name, user_name=user_name) self.assertEqual(manifest_from_db.title, tx_manifest.title) # Update by giving a dict to update() tx_manifest.views = 5 tx_manifest.update() manifest_from_db = TxManifest.get(repo_name=repo_name, user_name=user_name) self.assertEqual(manifest_from_db.views, 5) AppSettings.db_close()
def test_setup_db_with_connection_string_parts(self): AppSettings(db_protocol='sqlite', db_user=None, db_pass=None, db_end_point=None, db_port=None, db_name=':memory:', db_connection_string_params=None) AppSettings.db_create_tables([User.__table__]) user = User(name='ed', fullname='Edward Scissorhands', password='******') user.insert() user_from_db = User.get(name='ed') self.assertIsNotNone(user_from_db) self.assertEqual(user_from_db.password, '12345') AppSettings.db_close()
def get_jobID_from_commit_buildLog(project_folder_key:str, ix:int, commit_id:str) -> Optional[str]: """ Look for build_log.json in the Door43 bucket and extract the job_id from it. NOTE: It seems like old builds also put build_log.json in the CDN bucket but the new ones don't seem to have that. Return None if anything fails. """ file_key = f'{project_folder_key}{commit_id}/build_log.json' try: file_content = AppSettings.door43_s3_handler() \ .resource.Object(bucket_name=AppSettings.door43_bucket_name, key=file_key) \ .get()['Body'].read().decode('utf-8') json_content = json.loads(file_content) return json_content['job_id'] except Exception as e: AppSettings.logger.critical(f"get_jobID_from_commit_buildLog threw an exception while getting {prefix}D43 {ix:,} '{file_key}': {e}") return None
def mock_s3_obs_project(self): zip_file = os.path.join(self.resources_dir, 'converted_projects', 'en-obs-complete.zip') out_dir = os.path.join(self.temp_dir, 'en-obs-complete') unzip(zip_file, out_dir) project_dir = os.path.join(out_dir, 'door43', 'en-obs', '12345678') self.project_files = [ f for f in os.listdir(project_dir) if os.path.isfile(os.path.join(project_dir, f)) ] self.project_key = 'u/door43/en-obs/12345678' for filename in self.project_files: AppSettings.cdn_s3_handler().upload_file( os.path.join(project_dir, filename), '{0}/{1}'.format(self.project_key, filename)) AppSettings.cdn_s3_handler().upload_file( os.path.join(out_dir, 'door43', 'en-obs', 'project.json'), 'u/door43/en-obs/project.json') AppSettings.door43_s3_handler().upload_file( os.path.join(self.resources_dir, 'templates', 'project-page.html'), 'templates/project-page.html')
def mock_s3_bible_project(self, test_file_name, project_key, multi_part=False): converted_proj_dir = os.path.join(self.resources_dir, 'converted_projects') test_file_base = test_file_name.split('.zip')[0] zip_file = os.path.join(converted_proj_dir, test_file_name) out_dir = os.path.join(self.temp_dir, test_file_base) unzip(zip_file, out_dir) project_dir = os.path.join(out_dir, test_file_base) + os.path.sep self.project_files = file_utils.get_files(out_dir) self.project_key = project_key for filename in self.project_files: sub_path = filename.split(project_dir)[1].replace( os.path.sep, '/') # Make sure it is a bucket path AppSettings.cdn_s3_handler().upload_file( filename, '{0}/{1}'.format(project_key, sub_path)) if multi_part: # copy files from cdn to door43 base_name = os.path.basename(filename) if '.html' in base_name: with open(filename, 'r') as f: soup = BeautifulSoup(f, 'html.parser') # add nav tag new_tag = soup.new_tag('div', id='right-sidebar') soup.body.append(new_tag) html = str(soup) file_utils.write_file( filename, html.encode('ascii', 'xmlcharrefreplace')) AppSettings.door43_s3_handler().upload_file( filename, '{0}/{1}'.format(project_key, base_name)) # u, user, repo = project_key AppSettings.door43_s3_handler().upload_file( os.path.join(self.resources_dir, 'templates', 'project-page.html'), 'templates/project-page.html')
def job(queued_json_payload:Dict[str,Any]) -> None: """ This function is called by the rq package to process a job in the queue(s). (Don't rename this function.) The job is removed from the queue before the job is started, but if the job throws an exception or times out (timeout specified in enqueue process) then the job gets added to the 'failed' queue. """ AppSettings.logger.info("Door43-Job-Handler received a callback" + (" (in debug mode)" if debug_mode_flag else "")) start_time = time.time() stats_client.incr(f'{callback_stats_prefix}.jobs.attempted') current_job = get_current_job() #print(f"Current job: {current_job}") # Mostly just displays the job number and payload #print("id",current_job.id) # Displays job number #print("origin",current_job.origin) # Displays queue name #print("meta",current_job.meta) # Empty dict # AppSettings.logger.info(f"Updating queue statistics…") our_queue= Queue(callback_queue_name, connection=current_job.connection) len_our_queue = len(our_queue) # Should normally sit at zero here # AppSettings.logger.debug(f"Queue '{callback_queue_name}' length={len_our_queue}") stats_client.gauge(f'"{door43_stats_prefix}.enqueue-job.callback.queue.length.current', len_our_queue) AppSettings.logger.info(f"Updated stats for '{door43_stats_prefix}.enqueue-job.callback.queue.length.current' to {len_our_queue}") #print(f"Got a job from {current_job.origin} queue: {queued_json_payload}") #print(f"\nGot job {current_job.id} from {current_job.origin} queue") #queue_prefix = 'dev-' if current_job.origin.startswith('dev-') else '' #assert queue_prefix == prefix try: job_descriptive_name, door43_webhook_received_at = \ process_callback_job(prefix, queued_json_payload, current_job.connection) except Exception as e: # Catch most exceptions here so we can log them to CloudWatch prefixed_name = f"{prefix}Door43_Callback" AppSettings.logger.critical(f"{prefixed_name} threw an exception while processing: {queued_json_payload}") AppSettings.logger.critical(f"{e}: {traceback.format_exc()}") AppSettings.close_logger() # Ensure queued logs are uploaded to AWS CloudWatch # Now attempt to log it to an additional, separate FAILED log import logging from boto3 import Session from watchtower import CloudWatchLogHandler logger2 = logging.getLogger(prefixed_name) test_mode_flag = os.getenv('TEST_MODE', '') travis_flag = os.getenv('TRAVIS_BRANCH', '') log_group_name = f"FAILED_{'' if test_mode_flag or travis_flag else prefix}tX" \ f"{'_DEBUG' if debug_mode_flag else ''}" \ f"{'_TEST' if test_mode_flag else ''}" \ f"{'_TravisCI' if travis_flag else ''}" aws_access_key_id = os.environ['AWS_ACCESS_KEY_ID'] boto3_session = Session(aws_access_key_id=aws_access_key_id, aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'], region_name='us-west-2') failure_watchtower_log_handler = CloudWatchLogHandler(boto3_session=boto3_session, use_queues=False, log_group=log_group_name, stream_name=prefixed_name) logger2.addHandler(failure_watchtower_log_handler) logger2.setLevel(logging.DEBUG) logger2.info(f"Logging to AWS CloudWatch group '{log_group_name}' using key '…{aws_access_key_id[-2:]}'.") logger2.critical(f"{prefixed_name} threw an exception while processing: {queued_json_payload}") logger2.critical(f"{e}: {traceback.format_exc()}") failure_watchtower_log_handler.close() # NOTE: following line removed as stats recording used too much disk space # stats_client.gauge(user_projects_invoked_string, 1) # Mark as 'failed' stats_client.gauge(project_types_invoked_string, 1) # Mark as 'failed' raise e # We raise the exception again so it goes into the failed queue elapsed_milliseconds = round((time.time() - start_time) * 1000) stats_client.timing(f'{callback_stats_prefix}.job.duration', elapsed_milliseconds) if elapsed_milliseconds < 2000: AppSettings.logger.info(f"{prefix}Door43 callback handling for {job_descriptive_name} completed in {elapsed_milliseconds:,} milliseconds.") else: AppSettings.logger.info(f"{prefix}Door43 callback handling for {job_descriptive_name} completed in {round(time.time() - start_time)} seconds.") # Calculate total elapsed time for the job total_elapsed_time = datetime.utcnow() - \ datetime.strptime(door43_webhook_received_at, '%Y-%m-%dT%H:%M:%SZ') AppSettings.logger.info(f"{prefix}Door43 total job for {job_descriptive_name} completed in {round(total_elapsed_time.total_seconds())} seconds.") stats_client.timing(f'{job_handler_stats_prefix}.total.job.duration', round(total_elapsed_time.total_seconds() * 1000)) # NOTE: following line removed as stats recording used too much disk space # stats_client.gauge(user_projects_invoked_string, 0) # Mark as 'succeeded' stats_client.gauge(project_types_invoked_string, 0) # Mark as 'succeeded' stats_client.incr(f'{callback_stats_prefix}.jobs.succeeded') AppSettings.close_logger() # Ensure queued logs are uploaded to AWS CloudWatch
def test_init(self): gogs_url = 'https://my.gogs.org' AppSettings(gogs_url=gogs_url) self.assertEqual(AppSettings.gogs_url, gogs_url)
def insert(self): AppSettings.db().add(self) AppSettings.db().commit() AppSettings.db().close()
def job(queued_json_payload: Dict[str, Any]) -> None: """ This function is called by the rq package to process a job in the queue(s). (Don't rename this function.) The job is removed from the queue before the job is started, but if the job throws an exception or times out (timeout specified in enqueue process) then the job gets added to the 'failed' queue. """ AppSettings.logger.debug("tX JobHandler received a job" + (" (in debug mode)" if debug_mode_flag else "")) start_time = time() stats_client.incr('jobs.HTML.attempted') AppSettings.logger.info(f"Clearing /tmp folder…") empty_folder( '/tmp/', only_prefix='tX_') # Stops failed jobs from accumulating in /tmp # AppSettings.logger.info(f"Updating queue statistics…") our_queue = Queue(webhook_queue_name, connection=get_current_job().connection) len_our_queue = len(our_queue) # Should normally sit at zero here # AppSettings.logger.debug(f"Queue '{webhook_queue_name}' length={len_our_queue}") stats_client.gauge(f'{webhook_queue_name}.queue.length.current', len_our_queue) AppSettings.logger.info( f"Updated stats for '{tx_stats_prefix}.enqueue-job.queue.length.current' to {len_our_queue}" ) try: job_descriptive_name = process_tx_job(prefix, queued_json_payload) except Exception as e: # Catch most exceptions here so we can log them to CloudWatch prefixed_name = f"{prefix}tX_HTML_Job_Handler" AppSettings.logger.critical( f"{prefixed_name} threw an exception while processing: {queued_json_payload}" ) AppSettings.logger.critical(f"{e}: {traceback.format_exc()}") AppSettings.close_logger( ) # Ensure queued logs are uploaded to AWS CloudWatch # Now attempt to log it to an additional, separate FAILED log logger2 = logging.getLogger(prefixed_name) test_mode_flag = os.getenv('TEST_MODE', '') travis_flag = os.getenv('TRAVIS_BRANCH', '') log_group_name = f"FAILED_{'' if test_mode_flag or travis_flag else prefix}tX" \ f"{'_DEBUG' if debug_mode_flag else ''}" \ f"{'_TEST' if test_mode_flag else ''}" \ f"{'_TravisCI' if travis_flag else ''}" aws_access_key_id = os.environ['AWS_ACCESS_KEY_ID'] aws_secret_access_key = os.environ['AWS_SECRET_ACCESS_KEY'] boto3_client = boto3.client( "logs", aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, region_name='us-west-2') failure_watchtower_log_handler = watchtower.CloudWatchLogHandler( boto3_client=boto3_client, use_queues=False, log_group_name=log_group_name, stream_name=prefixed_name) logger2.addHandler(failure_watchtower_log_handler) logger2.setLevel(logging.DEBUG) logger2.info( f"Logging to AWS CloudWatch group '{log_group_name}' using key '…{aws_access_key_id[-2:]}'." ) logger2.critical( f"{prefixed_name} threw an exception while processing: {queued_json_payload}" ) logger2.critical(f"{e}: {traceback.format_exc()}") failure_watchtower_log_handler.close() raise e # We raise the exception again so it goes into the failed queue elapsed_milliseconds = round((time() - start_time) * 1000) stats_client.timing(f'job.HTML.duration', elapsed_milliseconds) if elapsed_milliseconds < 2000: AppSettings.logger.info( f"{prefix}tX job handling for {job_descriptive_name} completed in {elapsed_milliseconds:,} milliseconds." ) else: AppSettings.logger.info( f"{prefix}tX job handling for {job_descriptive_name} completed in {round(time() - start_time)} seconds." ) stats_client.incr('jobs.HTML.completed') AppSettings.close_logger( ) # Ensure queued logs are uploaded to AWS CloudWatch
def update(self): AppSettings.db().merge(self) AppSettings.db().commit() AppSettings.db().close()
def delete(self): AppSettings.db().delete(self) AppSettings.db().commit() AppSettings.db().close()
def query(cls, **kwargs): items = AppSettings.db().query(cls).filter_by(**kwargs) return items
def do_post_processing(self) -> Dict[str, Any]: AppSettings.logger.debug(f"ClientLinterCallback.do_post_processing()…") if not self.identifier: error = 'No identifier found' AppSettings.logger.error(error) raise Exception(error) # if not self.s3_results_key: # error = f"No s3_results_key found for identifier = {self.identifier}" # AppSettings.logger.error(error) # raise Exception(error) id_parts = self.identifier.split('/') self.multipart = len(id_parts) > 3 if self.multipart: raise Exception("Unsupported") # NOTE: Disabled 4Mar2019 coz unused # part_count, part_id, book = id_parts[1:4] # AppSettings.logger.debug('Multiple project, part {0} of {1}, linted book {2}'. # format(part_id, part_count, book)) # s3__master_results_key = '/'.join(self.s3_results_key.split('/')[:-1]) else: AppSettings.logger.debug('Single project') # NOTE: Disabled 4Mar2019 coz unused # s3__master_results_key = self.s3_results_key build_log = { 'identifier': self.identifier, 'success': self.success, 'multipart_project': self.multipart, 'log': self.log, 'warnings': self.warnings, 'errors': self.errors, # 's3_commit_key': self.s3_results_key } if not self.success: msg = "Linter failed for identifier: " + self.identifier build_log['warnings'].append(msg) AppSettings.logger.error(msg) else: AppSettings.logger.debug( f"Linter {self.identifier} had success with" f" {len(self.warnings)} warnings: {', '.join(self.warnings[:5])} …" ) has_warnings = len(build_log['warnings']) > 0 if has_warnings: msg = f"Linter {self.identifier} has Warnings!" build_log['log'].append(msg) else: msg = f"Linter {self.identifier} completed with no warnings" build_log['log'].append(msg) # NOTE: Do we need this -- disabled 25Feb2019 # ClientLinterCallback.upload_build_log(build_log, 'lint_log.json', self.temp_dir, self.s3_results_key) # NOTE: Do we need this -- disabled 4Mar2019 since linting is always done first # results = ClientLinterCallback.deploy_if_conversion_finished(s3__master_results_key, self.identifier) # if results: # self.all_parts_completed = True # build_log = results # if prefix and debug_mode_flag: # AppSettings.logger.debug(f"Temp folder '{self.temp_dir}' has been left on disk for debugging!") # else: # remove_tree(self.temp_dir) # cleanup AppSettings.db_close() return build_log # end of do_post_processing() # end of ClientLinterCallback class
def mock_s3_tn_project(self, part): zip_file = os.path.join(self.resources_dir, 'converted_projects', 'en_tn_converted.zip') out_dir = os.path.join(self.temp_dir, 'en_tn_converted') unzip(zip_file, out_dir) src_dir = os.path.join(out_dir, 'en_tn_converted') self.project_files = [ f for f in os.listdir(src_dir) if os.path.isfile(os.path.join(src_dir, f)) ] self.project_key = 'u/door43/en_tn/12345678' build_log = file_utils.load_json_object( os.path.join(src_dir, 'build_log.json')) build_log['part'] = part file_utils.write_file(os.path.join(src_dir, 'build_log.json'), build_log) AppSettings.cdn_s3_handler().upload_file( os.path.join(src_dir, 'build_log.json'), '{0}/{1}/build_log.json'.format(self.project_key, part)) AppSettings.cdn_s3_handler().upload_file( os.path.join(src_dir, 'index.json'), '{0}/{1}/index.json'.format(self.project_key, part)) AppSettings.cdn_s3_handler().upload_file( os.path.join(src_dir, 'build_log.json'), '{0}/{1}/finished'.format(self.project_key, part)) AppSettings.cdn_s3_handler().upload_file( os.path.join(src_dir, '01-GEN.html'), '{0}/{1}/01-GEN.html'.format(self.project_key, part)) AppSettings.cdn_s3_handler().upload_file( os.path.join(src_dir, 'project.json'), 'u/door43/en_tq/project.json') AppSettings.door43_s3_handler().upload_file( os.path.join(self.resources_dir, 'templates', 'project-page.html'), 'templates/project-page.html')
def test_s3_handler(self): self.assertIsNotNone(AppSettings.cdn_s3_handler())
'TSV_Translation_Notes', 'tn', 'other', ), 'html'), ('usfm2html', Usfm2HtmlConverter, ('usfm', ), ( 'Bible', 'Aligned_Bible', 'Greek_New_Testament', 'Hebrew_Old_Testament', 'bible', 'reg', 'other', ), 'html'), ) AppSettings(prefix=prefix) if prefix not in ('', 'dev-'): AppSettings.logger.critical( f"Unexpected prefix: '{prefix}' — expected '' or 'dev-'") tx_stats_prefix = f"tx.{'dev' if prefix else 'prod'}" job_handler_stats_prefix = f"{tx_stats_prefix}.tx-job-handler" # Get the Graphite URL from the environment, otherwise use a local test instance graphite_url = os.getenv('GRAPHITE_HOSTNAME', 'localhost') stats_client = StatsClient(host=graphite_url, port=8125, prefix=job_handler_stats_prefix) def get_linter_module(glm_job: Dict[str, Any]) -> Tuple[Optional[str], Any]: """
def setUp(self): """Runs before each test.""" AppSettings(prefix='{0}-'.format(self._testMethodName)) self.temp_dir = tempfile.mkdtemp(prefix='tX_test_Md2HtmlConverter') self.out_dir = '' self.out_zip_file = ''
def deploy_revision_to_door43(self, build_log:Dict[str,Any]) -> bool: """ Deploys a single revision of a project to door43.org Templates the converted files then uploads them and the build log to the S3 bucket and creates a 'deployed' file there too. :param dict build_log: :return bool: """ start = time.time() AppSettings.logger.debug(f"Deploying, build log: {json.dumps(build_log)[:256]} …") assert 'multiple' not in build_log assert 'part' not in build_log user = build_log['repo_owner_username'] # was 'repo_owner' repo_name = build_log['repo_name'] commit_id = build_log['commit_id'] # Hashes should already be reduced to 10 characters s3_commit_key = f'u/{user}/{repo_name}/{commit_id}' s3_repo_key = f'u/{user}/{repo_name}' source_dir = tempfile.mkdtemp(prefix='source_', dir=self.temp_dir) template_dir = tempfile.mkdtemp(prefix='template_', dir=self.temp_dir) output_dir = tempfile.mkdtemp(prefix='output_', dir=self.temp_dir) # Do the templating first resource_type = build_log['resource_type'] template_key = 'templates/project-page.html' template_file = os.path.join(template_dir, 'project-page.html') AppSettings.logger.info(f"Downloading project page template from {AppSettings.door43_bucket_name} '{template_key}' to {template_file} …") AppSettings.door43_s3_handler().download_file(template_key, template_file) source_dir, success = self.template_converted_files(build_log, output_dir, repo_name, resource_type, s3_commit_key, source_dir, start, template_file) build_log['warnings'].extend(self.error_messages) if not success: AppSettings.logger.critical("Templating failed—returning False") return False ####################### # # Now do the deploy # ####################### # Copy all other files over that don't already exist in output_dir, like css files # Copying from source_dir to output_dir (both are folders inside main temp folder) for filename in sorted(glob(os.path.join(source_dir, '*'))): output_file = os.path.join(output_dir, os.path.basename(filename)) if not os.path.exists(output_file) and not os.path.isdir(filename): copyfile(filename, output_file) # Save master build_log.json build_log['ended_at'] = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') file_utils.write_file(os.path.join(output_dir, 'build_log.json'), build_log) AppSettings.logger.debug(f"Final build_log.json: {json.dumps(build_log)[:256]} …") # Clear out the door43.org bucket's commit dir AppSettings.logger.info(f"Deleting all files in the website bucket directory: {AppSettings.door43_bucket_name}/{s3_commit_key} …") AppSettings.door43_s3_handler().bucket.objects.filter(Prefix=s3_commit_key).delete() # Upload all files to the S3 door43.org bucket AppSettings.logger.info(f"Uploading all files to the website bucket directory: {AppSettings.door43_bucket_name}/{s3_commit_key} …") has_index_file = False for root, _dirs, files in os.walk(output_dir): for filename in sorted(files): filepath = os.path.join(root, filename) if os.path.isdir(filepath): continue key = s3_commit_key + filepath.replace(output_dir, '').replace(os.path.sep, '/') AppSettings.logger.debug(f"Uploading {filename} to {AppSettings.door43_bucket_name} bucket {key} …") AppSettings.door43_s3_handler().upload_file(filepath, key, cache_time=0) redirect_to_file = "index.html" html_files = get_sorted_Bible_html_filepath_list(output_dir) if len(html_files) > 0 and os.path.join(output_dir, "index.html") not in html_files: redirect_to_file = html_files[0].replace(output_dir, "").lstrip("/") # Now we place json files and redirect index.html for the whole repo to this index.html file AppSettings.logger.info("Copying json files and setting up redirect…") try: AppSettings.door43_s3_handler().copy(from_key=f'{s3_repo_key}/project.json', from_bucket=AppSettings.cdn_bucket_name) AppSettings.door43_s3_handler().copy(from_key=f'{s3_commit_key}/manifest.json', to_key=f'{s3_repo_key}/manifest.json') master_exists = AppSettings.door43_s3_handler().object_exists(f'{s3_repo_key}/master/index.html') main_exists = AppSettings.door43_s3_handler().object_exists(f'{s3_repo_key}/main/index.html') if commit_id == 'master' or commit_id == 'main' or (not master_exists and not main_exists): AppSettings.door43_s3_handler().redirect(key=s3_repo_key, location=f"/{s3_commit_key}/{redirect_to_file}") AppSettings.door43_s3_handler().redirect(key=s3_repo_key + '/index.html', location=f"/{s3_commit_key}/{redirect_to_file}") AppSettings.door43_s3_handler().redirect(key=s3_commit_key, location=f"/{s3_commit_key}/{redirect_to_file}") if not has_index_file: AppSettings.door43_s3_handler().redirect(key=f"{s3_commit_key}/index.html", location=f"/{s3_commit_key}/{redirect_to_file}") self.write_data_to_file_and_upload_to_CDN(output_dir, s3_commit_key, fname='deployed', data=' ') # flag that deploy has finished except Exception as e: AppSettings.logger.critical(f"Deployer threw an exception: {e}: {traceback.format_exc()}") elapsed_seconds = int(time.time() - start) AppSettings.logger.debug(f"Deploy completed in {elapsed_seconds} seconds.") self.close() return True