def multipart_master_merge(self, s3_commit_key, resource_type,
download_key, output_dir, source_dir, start,
template_file):
prefix = download_key + '/'
App.door43_s3_handler().download_dir(
prefix, source_dir) # get previous templated files
source_dir = os.path.join(source_dir, download_key)
files = sorted(glob(os.path.join(source_dir, '*.*')))
for f in files:
App.logger.debug("Downloaded: " + f)
fname = os.path.join(source_dir, 'index.html')
if os.path.isfile(fname):
os.remove(fname) # remove index if already exists
elapsed_seconds = int(time.time() - start)
App.logger.debug("deploy download completed in " +
str(elapsed_seconds) + " seconds")
templater = init_template(resource_type, source_dir, output_dir,
template_file)
# restore index from previous passes
index_json = self.get_templater_index(s3_commit_key, 'index.json')
templater.titles = index_json['titles']
templater.chapters = index_json['chapters']
templater.book_codes = index_json['book_codes']
templater.already_converted = templater.files # do not reconvert files
# merge the source files with the template
try:
self.run_templater(templater)
success = True
except Exception as e:
App.logger.error(
"Error multi-part applying template {0} to resource type {1}".
format(template_file, resource_type))
self.close()
success = False
return source_dir, success
def multipart_master_merge(self, s3_commit_key, resource_type, download_key, output_dir, source_dir, start,
template_file):
prefix = download_key + '/'
App.door43_s3_handler().download_dir(prefix, source_dir) # get previous templated files
source_dir = os.path.join(source_dir, download_key)
files = sorted(glob(os.path.join(source_dir, '*.*')))
for f in files:
App.logger.debug("Downloaded: " + f)
fname = os.path.join(source_dir, 'index.html')
if os.path.isfile(fname):
os.remove(fname) # remove index if already exists
elapsed_seconds = int(time.time() - start)
App.logger.debug("deploy download completed in " + str(elapsed_seconds) + " seconds")
templater = init_template(resource_type, source_dir, output_dir, template_file)
# restore index from previous passes
index_json = self.get_templater_index(s3_commit_key, 'index.json')
templater.titles = index_json['titles']
templater.chapters = index_json['chapters']
templater.book_codes = index_json['book_codes']
templater.already_converted = templater.files # do not reconvert files
# merge the source files with the template
try:
self.run_templater(templater)
success = True
except Exception as e:
App.logger.error("Error multi-part applying template {0} to resource type {1}".format(template_file,
resource_type))
self.close()
success = False
return source_dir, success
def doTemplater(self, resource_type, test_folder_name, alreadyProcessed=False):
template_file = os.path.join(self.resources_dir, 'templates', 'project-page.html')
self.out_dir = tempfile.mkdtemp(prefix='output_')
if not alreadyProcessed:
return do_template(resource_type, test_folder_name, self.out_dir, template_file)
# we pre-process to get title and chapter info
template_pre = init_template(resource_type, test_folder_name, self.out_dir, template_file)
template_pre.run()
# copy pre-processed data and run again
template = init_template(resource_type, self.out_dir, self.out_dir, template_file)
template.already_converted = template.files
template.book_codes = template_pre.book_codes
template.chapters = template_pre.chapters
template.titles = template_pre.titles
return template.run()
def template_converted_files(self, build_log, download_key, output_dir, repo_name, resource_type, s3_commit_key,
source_dir, start, template_file):
App.cdn_s3_handler().download_dir(download_key + '/', source_dir)
source_dir = os.path.join(source_dir, download_key.replace('/', os.path.sep))
elapsed_seconds = int(time.time() - start)
App.logger.debug("deploy download completed in " + str(elapsed_seconds) + " seconds")
html_files = sorted(glob(os.path.join(source_dir, '*.html')))
if len(html_files) < 1:
content = ''
if len(build_log['errors']) > 0:
content += """
<div style="text-align:center;margin-bottom:20px">
<i class="fa fa-times-circle-o" style="font-size: 250px;font-weight: 300;color: red"></i>
<br/>
<h2>Critical!</h2>
<h3>Here is what went wrong with this build:</h3>
</div>
"""
content += '<div><ul><li>' + '</li><li>'.join(build_log['errors']) + '</li></ul></div>'
else:
content += '<h1 class="conversion-requested">{0}</h1>'.format(build_log['message'])
content += '<p><i>No content is available to show for {0} yet.</i></p>'.format(repo_name)
html = """
<html lang="en">
<head>
<title>{0}</title>
</head>
<body>
<div id="content">{1}</div>
</body>
</html>""".format(repo_name, content)
repo_index_file = os.path.join(source_dir, 'index.html')
write_file(repo_index_file, html)
# merge the source files with the template
templater = init_template(resource_type, source_dir, output_dir, template_file)
try:
self.run_templater(templater)
success = True
except Exception as e:
App.logger.error("Error applying template {0} to resource type {1}:".format(template_file, resource_type))
App.logger.error(e.message)
App.logger.error('{0}: {1}'.format(str(e), traceback.format_exc()))
self.close()
success = False
if success:
# update index of templated files
index_json_fname = 'index.json'
index_json = self.get_templater_index(s3_commit_key, index_json_fname)
App.logger.debug("initial 'index.json': " + json.dumps(index_json)[:256])
self.update_index_key(index_json, templater, 'titles')
self.update_index_key(index_json, templater, 'chapters')
self.update_index_key(index_json, templater, 'book_codes')
App.logger.debug("final 'index.json': " + json.dumps(index_json)[:256])
self.write_data_to_file(output_dir, s3_commit_key, index_json_fname, index_json)
return source_dir, success
def doTemplater(self,
resource_type,
test_folder_name,
alreadyProcessed=False):
template_file = os.path.join(self.resources_dir, 'templates',
'project-page.html')
self.out_dir = tempfile.mkdtemp(prefix='output_')
if not alreadyProcessed:
return do_template(resource_type, test_folder_name, self.out_dir,
template_file)
# we pre-process to get title and chapter info
template_pre = init_template(resource_type, test_folder_name,
self.out_dir, template_file)
template_pre.run()
# copy pre-processed data and run again
template = init_template(resource_type, self.out_dir, self.out_dir,
template_file)
template.already_converted = template.files
template.book_codes = template_pre.book_codes
template.chapters = template_pre.chapters
template.titles = template_pre.titles
return template.run()
def template_converted_files(self, build_log, download_key, output_dir,
repo_name, resource_type, s3_commit_key,
source_dir, start, template_file):
App.cdn_s3_handler().download_dir(download_key + '/', source_dir)
source_dir = os.path.join(source_dir,
download_key.replace('/', os.path.sep))
elapsed_seconds = int(time.time() - start)
App.logger.debug("deploy download completed in " +
str(elapsed_seconds) + " seconds")
html_files = sorted(glob(os.path.join(source_dir, '*.html')))
if len(html_files) < 1:
content = ''
if len(build_log['errors']) > 0:
content += """
<div style="text-align:center;margin-bottom:20px">
<i class="fa fa-times-circle-o" style="font-size: 250px;font-weight: 300;color: red"></i>
<br/>
<h2>Critical!</h2>
<h3>Here is what went wrong with this build:</h3>
</div>
"""
content += '<div><ul><li>' + '</li><li>'.join(
build_log['errors']) + '</li></ul></div>'
else:
content += '<h1 class="conversion-requested">{0}</h1>'.format(
build_log['message'])
content += '<p><i>No content is available to show for {0} yet.</i></p>'.format(
repo_name)
html = """
<html lang="en">
<head>
<title>{0}</title>
</head>
<body>
<div id="content">{1}</div>
</body>
</html>""".format(repo_name, content)
repo_index_file = os.path.join(source_dir, 'index.html')
write_file(repo_index_file, html)
# merge the source files with the template
templater = init_template(resource_type, source_dir, output_dir,
template_file)
try:
self.run_templater(templater)
success = True
except Exception as e:
App.logger.error(
"Error applying template {0} to resource type {1}:".format(
template_file, resource_type))
App.logger.error(e.message)
App.logger.error('{0}: {1}'.format(str(e), traceback.format_exc()))
self.close()
success = False
if success:
# update index of templated files
index_json_fname = 'index.json'
index_json = self.get_templater_index(s3_commit_key,
index_json_fname)
App.logger.debug("initial 'index.json': " +
json.dumps(index_json)[:256])
self.update_index_key(index_json, templater, 'titles')
self.update_index_key(index_json, templater, 'chapters')
self.update_index_key(index_json, templater, 'book_codes')
App.logger.debug("final 'index.json': " +
json.dumps(index_json)[:256])
self.write_data_to_file(output_dir, s3_commit_key,
index_json_fname, index_json)
return source_dir, success