def test_ceb_psa_text_ulb_L3(self):
""" Populates the ResourceContainer object and verifies the output."""
# test with the English OBS
zip_file = os.path.join(self.resources_dir, 'ceb_psa_text_ulb_L3.zip')
self.out_dir = tempfile.mkdtemp(prefix='repo_')
unzip(zip_file, self.out_dir)
repo_dir = os.path.join(self.out_dir, 'ceb_psa_text_ulb_l3')
rc = RC(directory=repo_dir)
rc.as_dict()
json = load_json_object(os.path.join(repo_dir, 'manifest.json'))
self.assertEqual(rc.resource.identifier, json['resource']['id'])
self.assertEqual(rc.resource.type, 'book')
self.assertEqual(rc.resource.format, 'text/{0}'.format(json['format']))
self.assertEqual(rc.resource.file_ext, json['format'])
self.assertEqual(rc.resource.conformsto, 'pre-rc')
self.assertEqual(rc.resource.modified, datetime.utcnow().strftime("%Y-%m-%d"))
chapters = rc.projects[0].chapters()
idx = 1
for chapter in chapters:
if chapter.isnumeric():
self.assertEqual(int(chapter), idx)
idx += 1
self.assertEqual(len(chapters), 151)
chunks = rc.projects[0].chunks('01')
self.assertEqual(len(chunks), 5)
def test_multiple_projects(self):
""" Populates the ResourceContainer object and verifies the output."""
# test with the English OBS
zip_file = os.path.join(self.resources_dir, 'en-ta-multiple-projects.zip')
self.out_dir = tempfile.mkdtemp(prefix='repo_')
unzip(zip_file, self.out_dir)
repo_dir = os.path.join(self.out_dir, 'en_ta')
rc = RC(directory=repo_dir)
rc.as_dict()
yaml = load_yaml_object(os.path.join(repo_dir, 'manifest.yaml'))
self.assertEqual(rc.resource.identifier, yaml['dublin_core']['identifier'])
self.assertEqual(rc.resource.type, yaml['dublin_core']['type'])
self.assertEqual(rc.resource.format, yaml['dublin_core']['format'])
self.assertEqual(rc.resource.file_ext, 'md')
self.assertEqual(rc.resource.conformsto, yaml['dublin_core']['conformsto'])
self.assertEqual(rc.resource.modified, yaml['dublin_core']['modified'])
self.assertEqual(len(rc.project_ids), 4)
self.assertEqual(rc.project_count, 4)
chapters = rc.project('checking').chapters()
self.assertEqual(len(chapters), 44)
chunks = rc.project('checking').chunks('level1')
self.assertEqual(chunks, ['01.md', 'sub-title.md', 'title.md'])
self.assertTrue('acceptable' in rc.project('checking').config())
self.assertTrue('title' in rc.project('checking').toc())
self.assertTrue(rc.project('checking').toc()['title'], 'Table of Contents')
def test_en_obs_manifest_yaml(self):
""" Populates the ResourceContainer object and verifies the output."""
# test with the English OBS
zip_file = os.path.join(self.resources_dir, 'en-obs-manifest-yaml.zip')
self.out_dir = tempfile.mkdtemp(prefix='repo_')
unzip(zip_file, self.out_dir)
repo_dir = os.path.join(self.out_dir, 'en_obs')
rc = RC(directory=repo_dir, repo_name='en_obs')
rc_dic = rc.as_dict()
yaml = load_yaml_object(os.path.join(repo_dir, 'manifest.yaml'))
self.assertDictEqual(yaml, rc_dic)
chapters = rc.projects[0].chapters()
self.assertEqual(len(chapters), 2)
chunks = rc.project().chunks('front')
self.assertEqual(chunks, ['intro.md', 'title.md'])
def print_project(self, project_id):
"""
:param string project_id:
:return string:
"""
self.project_id = project_id
if len(project_id.split('/')) != 3:
raise Exception('Project not found.')
user_name, repo_name, commit_id = project_id.split('/')
source_path = 'u/{0}'.format(project_id)
print_all_key = '{0}/print_all.html'.format(source_path)
print_all_file = tempfile.mktemp(prefix='print_all_')
if not App.cdn_s3_handler().key_exists(print_all_key):
files_dir = tempfile.mkdtemp(prefix='files_')
App.cdn_s3_handler().download_dir(source_path, files_dir)
project_dir = os.path.join(files_dir,
source_path.replace('/', os.path.sep))
if not os.path.isdir(project_dir):
raise Exception('Project not found.')
rc = RC(project_dir, repo_name)
with codecs.open(print_all_file, 'w', 'utf-8-sig') as print_all:
print_all.write("""
<html lang="{0}" dir="{1}">
<head>
<meta charset="UTF-8"/>
<title>{2}: {3}</title>
<style type="text/css">
body > div {{
page-break-after: always;
}}
</style>
</head>
<body onLoad="window.print()">
<h1>{2}: {3}</h1>
""".format(rc.resource.language.identifier, rc.resource.language.direction,
rc.resource.language.title, rc.resource.title))
for fname in sorted(glob(os.path.join(project_dir, '*.html')),
key=self.front_to_back):
with codecs.open(fname, 'r', 'utf-8-sig') as f:
soup = BeautifulSoup(f, 'html.parser')
# get the body of the raw html file
content = soup.div
if not content:
content = BeautifulSoup(
'<div>No content</div>',
'html.parser').find('div').extract()
content['id'] = os.path.basename(fname)
print_all.write(unicode(content))
print_all.write("""
</body>
</html>
""")
App.cdn_s3_handler().upload_file(print_all_file,
print_all_key,
cache_time=0)
html = read_file(print_all_file)
else:
html = App.cdn_s3_handler().get_file_contents(print_all_key)
return html
def test_EnUlbValid(self):
out_dir = self.unzip_resource('en_ulb.zip')
expected_warnings = 0
start = time.time()
rc = RC(out_dir)
linter = UsfmLinter(source_dir=out_dir, rc=rc)
linter.run()
elapsed_seconds = int(time.time() - start)
App.logger.debug("Checking time was " + str(elapsed_seconds) +
" seconds")
self.verify_results_counts(expected_warnings, linter)
def test_en_obs_package_json(self):
""" Populates the ResourceContainer object and verifies the output."""
# test with the English OBS
zip_file = os.path.join(self.resources_dir, 'en-obs-package-json.zip')
self.out_dir = tempfile.mkdtemp(prefix='repo_')
unzip(zip_file, self.out_dir)
repo_dir = os.path.join(self.out_dir, 'en-obs')
rc = RC(directory=repo_dir)
rc.as_dict()
json = load_json_object(os.path.join(repo_dir, 'package.json'))
self.assertEqual(rc.resource.identifier, json['resource']['slug'])
self.assertEqual(rc.resource.type, 'book')
self.assertEqual(rc.resource.format, json['content_mime_type'])
self.assertEqual(rc.resource.file_ext, 'md')
self.assertEqual(rc.resource.conformsto, 'pre-rc')
self.assertEqual(rc.resource.issued, json['resource']['status']['pub_date'])
chapters = rc.projects[0].chapters()
self.assertEqual(len(chapters), 2)
chunks = rc.project().chunks('_back')
self.assertEqual(chunks, ['back-matter.md'])
def test_bible_from_tx_pre_rc(self):
""" Populates the ResourceContainer object and verifies the output."""
# test with the English OBS
zip_file = os.path.join(self.resources_dir, 'id_mat_text_ulb-ts.zip')
self.out_dir = tempfile.mkdtemp(prefix='repo_')
unzip(zip_file, self.out_dir)
repo_dir = os.path.join(self.out_dir, 'id_mat_text_ulb-ts')
rc = RC(directory=repo_dir)
rc.as_dict()
json = load_json_object(os.path.join(repo_dir, 'manifest.json'))
self.assertEqual(rc.resource.identifier, json['resource']['id'])
self.assertEqual(rc.resource.type, 'book')
self.assertEqual(rc.resource.format, 'text/{0}'.format(json['format']))
self.assertEqual(rc.resource.file_ext, json['format'])
self.assertEqual(rc.resource.conformsto, 'pre-rc')
self.assertEqual(rc.resource.modified, datetime.utcnow().strftime("%Y-%m-%d"))
chapters = rc.projects[0].chapters()
self.assertEqual(len(chapters), 29)
chunks = rc.projects[0].chunks('01')
self.assertEqual(len(chunks), 11)
def run(self):
# get the resource container
self.rc = RC(self.source_dir)
with open(self.template_file) as template_file:
self.template_html = template_file.read()
soup = BeautifulSoup(self.template_html, 'html.parser')
soup.body['class'] = soup.body.get('class',
[]) + [self.resource_type]
if self.classes:
soup.body['class'] = soup.body.get('class', []) + self.classes
self.template_html = unicode(soup)
self.apply_template()
return True
def extractFiles(cls, file_name, repo_name):
file_path = os.path.join(TestTnPreprocessor.resources_dir, file_name)
# 1) unzip the repo files
temp_dir = tempfile.mkdtemp(prefix='repo_')
unzip(file_path, temp_dir)
repo_dir = os.path.join(temp_dir, repo_name)
if not os.path.isdir(repo_dir):
repo_dir = file_path
# 2) Get the resource container
rc = RC(repo_dir)
return rc, repo_dir, temp_dir
def test_EnUlbValidSubset(self):
check_files = [
'19-PSA.usfm', '22-SNG.usfm', '24-JER.usfm', '25-LAM.usfm',
'35-HAB.usfm'
]
out_dir = self.unzip_resource_only('en_ulb.zip', check_files)
expected_warnings = 0
start = time.time()
rc = RC(out_dir)
linter = UsfmLinter(source_dir=out_dir, rc=rc)
linter.run()
elapsed_seconds = int(time.time() - start)
App.logger.debug("Checking time was " + str(elapsed_seconds) +
" seconds")
self.verify_results_counts(expected_warnings, linter)
def run(self):
"""
Run common handling for all linters,and then calls the lint() function
"""
success = False
try:
# Download file if a source_zip_url was given
if self.source_zip_url:
App.logger.debug("Linting url: " + self.source_zip_url)
self.download_archive()
# unzip the input archive if a source_zip_file exists
if self.source_zip_file:
App.logger.debug("Linting zip: " + self.source_zip_file)
self.unzip_archive()
# lint files
if self.source_dir:
self.rc = RC(directory=self.source_dir)
App.logger.debug("Linting '{0}' files...".format(
self.source_dir))
success = self.lint()
App.logger.debug("...finished.")
except Exception as e:
message = 'Linting process ended abnormally: {0}'.format(e.message)
App.logger.error(message)
self.log.warnings.append(message)
App.logger.error('{0}: {1}'.format(str(e), traceback.format_exc()))
warnings = self.log.warnings
if len(
warnings
) > 200: # sanity check so we don't overflow callback size limits
warnings = warnings[0:199]
msg = 'Warnings truncated for {0}'.format(self.s3_results_key)
App.logger.debug(msg)
warnings.append(msg)
results = {
'identifier': self.identifier,
'success': success,
'warnings': warnings,
's3_results_key': self.s3_results_key
}
if self.callback is not None:
self.callback_results = results
self.do_callback(self.callback, self.callback_results)
App.logger.debug("Linter results: " + str(results))
return results
def test_bible_no_manifest(self):
""" Populates the ResourceContainer object and verifies the output."""
# test with the English OBS
zip_file = os.path.join(self.resources_dir, 'bible-no-manifest.zip')
self.out_dir = tempfile.mkdtemp(prefix='repo_')
unzip(zip_file, self.out_dir)
repo_dir = os.path.join(self.out_dir, 'en_ulb')
rc = RC(directory=repo_dir)
rc.as_dict()
self.assertEqual(rc.resource.identifier, 'ulb')
self.assertEqual(rc.resource.type, 'bundle')
self.assertEqual(rc.resource.format, 'text/usfm')
self.assertEqual(rc.resource.file_ext, 'usfm')
self.assertEqual(rc.resource.conformsto, 'pre-rc')
self.assertEqual(rc.resource.modified, datetime.utcnow().strftime("%Y-%m-%d"))
chapters = rc.project().chapters()
self.assertEqual(len(chapters), 0)
self.assertEqual(len(rc.project().usfm_files()), 8)
def process_webhook(self):
# Check that we got commit data
if not self.commit_data:
raise Exception('No commit data from DCS was found in the Payload')
# Check that the user token is valid
if not App.gogs_user_token:
raise Exception('DCS user token not given in Payload.')
user = App.gogs_handler().get_user(App.gogs_user_token)
if not user:
raise Exception('Invalid DCS user token given in Payload')
# Check that the URL to the DCS repo is valid
if not self.commit_data['repository']['html_url'].startswith(App.gogs_url):
raise Exception('Repos can only belong to {0} to use this webhook client.'.format(App.gogs_url))
# Check that commit is on repo's default branch, else quit
try:
commit_branch = self.commit_data['ref'].split('/')[2]
except IndexError:
raise Exception('Could not determine commit branch, exiting.')
except KeyError:
Exception('This does not appear to be a push, exiting.')
if commit_branch != self.commit_data['repository']['default_branch']:
raise Exception('Commit branch: {0} is not the default branch, exiting.'.format(commit_branch))
# Get the commit_id, commit_url
commit_id = self.commit_data['after']
commit = None
for commit in self.commit_data['commits']:
if commit['id'] == commit_id:
break
commit_id = commit_id[:10] # Only use the short form
commit_url = commit['url']
# Gather other details from the commit that we will note for the job(s)
user_name = self.commit_data['repository']['owner']['username']
repo_name = self.commit_data['repository']['name']
compare_url = self.commit_data['compare_url']
commit_message = commit['message']
if 'pusher' in self.commit_data:
pusher = self.commit_data['pusher']
else:
pusher = {'username': commit['author']['username']}
pusher_username = pusher['username']
# Download and unzip the repo files
repo_dir = self.get_repo_files(commit_url, repo_name)
# Get the resource container
rc = RC(repo_dir, repo_name)
# Save manifest to manifest table
manifest_data = {
'repo_name': repo_name,
'user_name': user_name,
'lang_code': rc.resource.language.identifier,
'resource_id': rc.resource.identifier,
'resource_type': rc.resource.type,
'title': rc.resource.title,
'manifest': json.dumps(rc.as_dict()),
'last_updated': datetime.utcnow()
}
# First see if manifest already exists in DB and update it if it is
tx_manifest = TxManifest.get(repo_name=repo_name, user_name=user_name)
if tx_manifest:
for key, value in manifest_data.iteritems():
setattr(tx_manifest, key, value)
App.logger.debug('Updating manifest in manifest table: {0}'.format(manifest_data))
tx_manifest.update()
else:
tx_manifest = TxManifest(**manifest_data)
App.logger.debug('Inserting manifest into manifest table: {0}'.format(tx_manifest))
tx_manifest.insert()
# Preprocess the files
preprocess_dir = tempfile.mkdtemp(dir=self.base_temp_dir, prefix='preprocess_')
results, preprocessor = do_preprocess(rc, repo_dir, preprocess_dir)
# Zip up the massaged files
zip_filepath = tempfile.mktemp(dir=self.base_temp_dir, suffix='.zip')
App.logger.debug('Zipping files from {0} to {1}...'.format(preprocess_dir, zip_filepath))
add_contents_to_zip(zip_filepath, preprocess_dir)
App.logger.debug('finished.')
# Upload zipped file to the S3 bucket
file_key = self.upload_zip_file(commit_id, zip_filepath)
job = TxJob()
job.job_id = self.get_unique_job_id()
job.identifier = job.job_id
job.user_name = user_name
job.repo_name = repo_name
job.commit_id = commit_id
job.manifests_id = tx_manifest.id
job.created_at = datetime.utcnow()
job.user = user.username # Username of the token, not necessarily the repo's owner
job.input_format = rc.resource.file_ext
job.resource_type = rc.resource.identifier
job.source = self.source_url_base + "/" + file_key
job.cdn_bucket = App.cdn_bucket
job.cdn_file = 'tx/job/{0}.zip'.format(job.job_id)
job.output = 'https://{0}/{1}'.format(App.cdn_bucket, job.cdn_file)
job.callback = App.api_url + '/client/callback'
job.output_format = 'html'
job.links = {
"href": "{0}/tx/job/{1}".format(App.api_url, job.job_id),
"rel": "self",
"method": "GET"
}
job.success = False
converter = self.get_converter_module(job)
linter = self.get_linter_module(job)
if converter:
job.convert_module = converter.name
job.started_at = datetime.utcnow()
job.expires_at = job.started_at + timedelta(days=1)
job.eta = job.started_at + timedelta(minutes=5)
job.status = 'started'
job.message = 'Conversion started...'
job.log_message('Started job for {0}/{1}/{2}'.format(job.user_name, job.repo_name, job.commit_id))
else:
job.error_message('No converter was found to convert {0} from {1} to {2}'.format(job.resource_type,
job.input_format,
job.output_format))
job.message = 'No converter found'
job.status = 'failed'
if linter:
job.lint_module = linter.name
else:
App.logger.debug('No linter was found to lint {0}'.format(job.resource_type))
job.insert()
# Get S3 bucket/dir ready
s3_commit_key = 'u/{0}/{1}/{2}'.format(job.user_name, job.repo_name, job.commit_id)
self.clear_commit_directory_in_cdn(s3_commit_key)
# Create a build log
build_log_json = self.create_build_log(commit_id, commit_message, commit_url, compare_url, job,
pusher_username, repo_name, user_name)
# Upload an initial build_log
self.upload_build_log_to_s3(build_log_json, s3_commit_key)
# Update the project.json file
self.update_project_json(commit_id, job, repo_name, user_name)
# Convert and lint
if converter:
if not preprocessor.is_multiple_jobs():
self.send_request_to_converter(job, converter)
if linter:
extra_payload = {
's3_results_key': s3_commit_key
}
self.send_request_to_linter(job, linter, commit_url, extra_payload=extra_payload)
else:
# -----------------------------
# multiple book project
# -----------------------------
books = preprocessor.get_book_list()
App.logger.debug('Splitting job into separate parts for books: ' + ','.join(books))
book_count = len(books)
build_log_json['multiple'] = True
build_log_json['build_logs'] = []
for i in range(0, len(books)):
book = books[i]
App.logger.debug('Adding job for {0}, part {1} of {2}'.format(book, i, book_count))
# Send job request to tx-manager
if i == 0:
book_job = job # use the original job created above for the first book
book_job.identifier = '{0}/{1}/{2}/{3}'.format(job.job_id, book_count, i, book)
else:
book_job = job.clone() # copy the original job for this book's job
book_job.job_id = self.get_unique_job_id()
book_job.identifier = '{0}/{1}/{2}/{3}'.format(book_job.job_id, book_count, i, book)
book_job.cdn_file = 'tx/job/{0}.zip'.format(book_job.job_id)
book_job.output = 'https://{0}/{1}'.format(App.cdn_bucket, book_job.cdn_file)
book_job.links = {
"href": "{0}/tx/job/{1}".format(App.api_url, book_job.job_id),
"rel": "self",
"method": "GET"
}
book_job.insert()
book_job.source = self.build_multipart_source(file_key, book)
book_job.update()
book_build_log = self.create_build_log(commit_id, commit_message, commit_url, compare_url, book_job,
pusher_username, repo_name, user_name)
if len(book) > 0:
part = str(i)
book_build_log['book'] = book
book_build_log['part'] = part
build_log_json['build_logs'].append(book_build_log)
self.upload_build_log_to_s3(book_build_log, s3_commit_key, str(i) + "/")
self.send_request_to_converter(book_job, converter)
if linter:
extra_payload = {
'single_file': book,
's3_results_key': '{0}/{1}'.format(s3_commit_key, i)
}
self.send_request_to_linter(book_job, linter, commit_url, extra_payload)
remove_tree(self.base_temp_dir) # cleanup
return build_log_json
def process_webhook(self):
# Check that we got commit data
if not self.commit_data:
raise Exception('No commit data from DCS was found in the Payload')
# Check that the user token is valid
if not App.gogs_user_token:
raise Exception('DCS user token not given in Payload.')
user = App.gogs_handler().get_user(App.gogs_user_token)
if not user:
raise Exception('Invalid DCS user token given in Payload')
# Check that the URL to the DCS repo is valid
if not self.commit_data['repository']['html_url'].startswith(App.gogs_url):
raise Exception('Repos can only belong to {0} to use this webhook client.'.format(App.gogs_url))
# Check that commit is on repo's default branch, else quit
try:
commit_branch = self.commit_data['ref'].split('/')[2]
except IndexError:
raise Exception('Could not determine commit branch, exiting.')
except KeyError:
Exception('This does not appear to be a push, exiting.')
if commit_branch != self.commit_data['repository']['default_branch']:
raise Exception('Commit branch: {0} is not the default branch, exiting.'.format(commit_branch))
# Get the commit_id, commit_url
commit_id = self.commit_data['after']
commit = None
for commit in self.commit_data['commits']:
if commit['id'] == commit_id:
break
commit_id = commit_id[:10] # Only use the short form
commit_url = commit['url']
# Gather other details from the commit that we will note for the job(s)
user_name = self.commit_data['repository']['owner']['username']
repo_name = self.commit_data['repository']['name']
compare_url = self.commit_data['compare_url']
commit_message = commit['message']
if 'pusher' in self.commit_data:
pusher = self.commit_data['pusher']
else:
pusher = {'username': commit['author']['username']}
pusher_username = pusher['username']
# Download and unzip the repo files
repo_dir = self.get_repo_files(commit_url, repo_name)
# Get the resource container
rc = RC(repo_dir, repo_name)
# Save manifest to manifest table
manifest_data = {
'repo_name': repo_name,
'user_name': user_name,
'lang_code': rc.resource.language.identifier,
'resource_id': rc.resource.identifier,
'resource_type': rc.resource.type,
'title': rc.resource.title,
'manifest': json.dumps(rc.as_dict()),
'last_updated': datetime.utcnow()
}
# First see if manifest already exists in DB and update it if it is
tx_manifest = TxManifest.get(repo_name=repo_name, user_name=user_name)
if tx_manifest:
for key, value in manifest_data.iteritems():
setattr(tx_manifest, key, value)
App.logger.debug('Updating manifest in manifest table: {0}'.format(manifest_data))
tx_manifest.update()
else:
tx_manifest = TxManifest(**manifest_data)
App.logger.debug('Inserting manifest into manifest table: {0}'.format(tx_manifest))
tx_manifest.insert()
# Preprocess the files
preprocess_dir = tempfile.mkdtemp(dir=self.base_temp_dir, prefix='preprocess_')
results, preprocessor = do_preprocess(rc, repo_dir, preprocess_dir)
# Zip up the massaged files
zip_filepath = tempfile.mktemp(dir=self.base_temp_dir, suffix='.zip')
App.logger.debug('Zipping files from {0} to {1}...'.format(preprocess_dir, zip_filepath))
add_contents_to_zip(zip_filepath, preprocess_dir)
App.logger.debug('finished.')
# Upload zipped file to the S3 bucket
file_key = self.upload_zip_file(commit_id, zip_filepath)
job = TxJob()
job.job_id = self.get_unique_job_id()
job.identifier = job.job_id
job.user_name = user_name
job.repo_name = repo_name
job.commit_id = commit_id
job.manifests_id = tx_manifest.id
job.created_at = datetime.utcnow()
job.user = user.username # Username of the token, not necessarily the repo's owner
job.input_format = rc.resource.file_ext
job.resource_type = rc.resource.identifier
job.source = self.source_url_base + "/" + file_key
job.cdn_bucket = App.cdn_bucket
job.cdn_file = 'tx/job/{0}.zip'.format(job.job_id)
job.output = 'http://{0}.s3-{1}.amazonaws.com/{2}'.format(App.cdn_bucket, App.aws_region_name, job.cdn_file)
job.callback = App.api_url + '/client/callback'
job.output_format = 'html'
job.links = {
"href": "{0}/tx/job/{1}".format(App.api_url, job.job_id),
"rel": "self",
"method": "GET"
}
job.success = False
converter = self.get_converter_module(job)
linter = self.get_linter_module(job)
if converter:
job.convert_module = converter.name
job.started_at = datetime.utcnow()
job.expires_at = job.started_at + timedelta(days=1)
job.eta = job.started_at + timedelta(minutes=5)
job.status = 'started'
job.message = 'Conversion started...'
job.log_message('Started job for {0}/{1}/{2}'.format(job.user_name, job.repo_name, job.commit_id))
else:
job.error_message('No converter was found to convert {0} from {1} to {2}'.format(job.resource_type,
job.input_format,
job.output_format))
job.message = 'No converter found'
job.status = 'failed'
if linter:
job.lint_module = linter.name
else:
App.logger.debug('No linter was found to lint {0}'.format(job.resource_type))
job.insert()
# Get S3 bucket/dir ready
s3_commit_key = 'u/{0}/{1}/{2}'.format(job.user_name, job.repo_name, job.commit_id)
self.clear_commit_directory_in_cdn(s3_commit_key)
# Create a build log
build_log_json = self.create_build_log(commit_id, commit_message, commit_url, compare_url, job,
pusher_username, repo_name, user_name)
# Upload an initial build_log
self.upload_build_log_to_s3(build_log_json, s3_commit_key)
# Update the project.json file
self.update_project_json(commit_id, job, repo_name, user_name)
# Convert and lint
if converter:
if not preprocessor.is_multiple_jobs():
self.send_request_to_converter(job, converter)
if linter:
extra_payload = {
's3_results_key': s3_commit_key
}
self.send_request_to_linter(job, linter, commit_url, extra_payload=extra_payload)
else:
# -----------------------------
# multiple book project
# -----------------------------
books = preprocessor.get_book_list()
App.logger.debug('Splitting job into separate parts for books: ' + ','.join(books))
book_count = len(books)
build_log_json['multiple'] = True
build_log_json['build_logs'] = []
for i in range(0, len(books)):
book = books[i]
App.logger.debug('Adding job for {0}, part {1} of {2}'.format(book, i, book_count))
# Send job request to tx-manager
if i == 0:
book_job = job # use the original job created above for the first book
book_job.identifier = '{0}/{1}/{2}/{3}'.format(job.job_id, book_count, i, book)
else:
book_job = job.clone() # copy the original job for this book's job
book_job.job_id = self.get_unique_job_id()
book_job.identifier = '{0}/{1}/{2}/{3}'.format(book_job.job_id, book_count, i, book)
book_job.cdn_file = 'tx/job/{0}.zip'.format(book_job.job_id)
book_job.output = 'http://{0}.s3-{1}.amazonaws.com/{2}'.format(App.cdn_bucket, App.aws_region_name, book_job.cdn_file)
book_job.links = {
"href": "{0}/tx/job/{1}".format(App.api_url, book_job.job_id),
"rel": "self",
"method": "GET"
}
book_job.insert()
book_job.source = self.build_multipart_source(file_key, book)
book_job.update()
book_build_log = self.create_build_log(commit_id, commit_message, commit_url, compare_url, book_job,
pusher_username, repo_name, user_name)
if len(book) > 0:
part = str(i)
book_build_log['book'] = book
book_build_log['part'] = part
build_log_json['build_logs'].append(book_build_log)
self.upload_build_log_to_s3(book_build_log, s3_commit_key, str(i) + "/")
self.send_request_to_converter(book_job, converter)
if linter:
extra_payload = {
'single_file': book,
's3_results_key': '{0}/{1}'.format(s3_commit_key, i)
}
self.send_request_to_linter(book_job, linter, commit_url, extra_payload)
remove_tree(self.base_temp_dir) # cleanup
return build_log_json
def test_fix_links(self):
# given
rc = RC(os.path.join(self.resources_dir, 'manifests', 'tw'))
repo_name = 'Door43'
current_category = 'names'
tw = TwPreprocessor(rc, tempfile.gettempdir(), tempfile.gettempdir())
tw.repo_name = repo_name
content = "This has links to the same category: (See also: [titus](../names/titus.md), [timothy](../names/timothy.md)"
expected = "This has links to the same category: (See also: [titus](#titus), [timothy](#timothy)"
# when
converted = tw.fix_links(content, current_category)
# then
self.assertEqual(converted, expected)
# given
content = """This has links to other categories:
(See also:[lamb](../kt/lamb.md), [license](../other/license.md)"""
expected = """This has links to other categories:
(See also:[lamb](kt.html#lamb), [license](other.html#license)"""
# when
converted = tw.fix_links(content, current_category)
# then
self.assertEqual(converted, expected)
# given
content = """This has links to the same category and others:
(See also: [titus](../names/titus.md), [timothy](../names/timothy.md), [lamb](../kt/lamb.md),
[license](../other/license.md)"""
expected = """This has links to the same category and others:
(See also: [titus](#titus), [timothy](#timothy), [lamb](kt.html#lamb),
[license](other.html#license)"""
# when
converted = tw.fix_links(content, current_category)
# then
self.assertEqual(converted, expected)
# given
content = """This link should NOT be converted: [webpage](http://example.com/somewhere/outthere) """
expected = """This link should NOT be converted: [webpage](http://example.com/somewhere/outthere) """
# when
converted = tw.fix_links(content, current_category)
# then
self.assertEqual(converted, expected)
# given
content = """This [link](rc://en/tn/help/ezr/09/01) is a rc link that should go to
ezr/09/01.md in the en_tn repo"""
expected = """This [link](https://git.door43.org/Door43/en_tn/src/master/ezr/09/01.md) is a rc link that should go to
ezr/09/01.md in the en_tn repo"""
# when
converted = tw.fix_links(content, current_category)
# then
self.assertEqual(converted, expected)
# given
content = """This url should be made into a link: http://example.com/somewhere/outthere and so should www.example.com/asdf.html?id=5&view=dashboard#report."""
expected = """This url should be made into a link: [http://example.com/somewhere/outthere](http://example.com/somewhere/outthere) and so should [www.example.com/asdf.html?id=5&view=dashboard#report](http://www.example.com/asdf.html?id=5&view=dashboard#report)."""
# when
converted = tw.fix_links(content, current_category)
# then
self.assertEqual(converted, expected)
def test_fix_links(self):
rc = RC(os.path.join(self.resources_dir, 'manifests', 'ta'))
ta = TaPreprocessor(rc, tempfile.gettempdir(), tempfile.gettempdir())
content = "This has [links](../section1/01.md) to the same [manual](../section2/01.md)"
expected = "This has [links](#section1) to the same [manual](#section2)"
converted = ta.fix_links(content)
self.assertEqual(converted, expected)
content = """This has links to
[other](../../checking/section1/01.md) [manuals](../../translate/section2/01.md)"""
expected = """This has links to
[other](04-checking.html#section1) [manuals](03-translate.html#section2)"""
converted = ta.fix_links(content)
self.assertEqual(converted, expected)
content = """This has links to both this [manual](../section1/01.md),
this [page](section2) and [another manual](../../process/section3/01.md)."""
expected = """This has links to both this [manual](#section1),
this [page](#section2) and [another manual](02-process.html#section3)."""
converted = ta.fix_links(content)
self.assertEqual(converted, expected)
content = """This link should NOT be converted: [webpage](http://example.com/somewhere/outthere) """
expected = """This link should NOT be converted: [webpage](http://example.com/somewhere/outthere) """
converted = ta.fix_links(content)
self.assertEqual(converted, expected)
content = """This [link](rc://en/tw/dict/bible/other/dream) is a rc link that should go to
other/dream.md in the en_tw repo"""
expected = """This [link](https://git.door43.org/Door43/en_tw/src/master/bible/other/dream.md) is a rc link that should go to
other/dream.md in the en_tw repo"""
converted = ta.fix_links(content)
self.assertEqual(converted, expected)
content = """This url should be made into a link: http://example.com/somewhere/outthere and so should www.example.com/asdf.html?id=5&view=dashboard#report."""
expected = """This url should be made into a link: [http://example.com/somewhere/outthere](http://example.com/somewhere/outthere) and so should [www.example.com/asdf.html?id=5&view=dashboard#report](http://www.example.com/asdf.html?id=5&view=dashboard#report)."""
converted = ta.fix_links(content)
self.assertEqual(converted, expected)
# Tests https://git.door43.org/Door43/en_ta/raw/master/translate/translate-source-text/01.md
content = """
### Factors to Consider for a Source Text
When choosing a source text, there are a number of factors that must be considered:
* **[Statement of Faith](../../intro/statement-of-faith/01.md)** - Is the text in line with the Statement of Faith?
* **[Translation Guidelines](../../intro/translation-guidelines/01.md)** - Is the text in line with the Translation Guidelines?
* **Language** - Is the text in a suitable language that translators and checkers understand well?
* **[Copyrights, Licensing, and Source Texts](../translate-source-licensing/01.md)** - Is the text released under a license that gives sufficient legal freedom?
* **[Source Texts and Version Numbers](../translate-source-version/01.md)** - Is the text the latest, most updated version?
* **[The Original and Source Languages](../translate-original/01.md)** - Does the translation team understand the difference between source languages and original languages?
* **[Original Manuscripts](../translate-manuscripts/01.md)** - Does the translation team understand about Original Manuscripts and [Textual Variants](../translate-textvariants/01.md)?
It is important the the leaders of the churches in the language group agree that the source text is a good one. The Open Bible Stories are available in many source languages on http://ufw.io/stories/. There are also translations of the Bible there to be used as sources for translation in English, and soon other languages, as well.
"""
expected = """
### Factors to Consider for a Source Text
When choosing a source text, there are a number of factors that must be considered:
* **[Statement of Faith](01-intro.html#statement-of-faith)** - Is the text in line with the Statement of Faith?
* **[Translation Guidelines](01-intro.html#translation-guidelines)** - Is the text in line with the Translation Guidelines?
* **Language** - Is the text in a suitable language that translators and checkers understand well?
* **[Copyrights, Licensing, and Source Texts](#translate-source-licensing)** - Is the text released under a license that gives sufficient legal freedom?
* **[Source Texts and Version Numbers](#translate-source-version)** - Is the text the latest, most updated version?
* **[The Original and Source Languages](#translate-original)** - Does the translation team understand the difference between source languages and original languages?
* **[Original Manuscripts](#translate-manuscripts)** - Does the translation team understand about Original Manuscripts and [Textual Variants](#translate-textvariants)?
It is important the the leaders of the churches in the language group agree that the source text is a good one. The Open Bible Stories are available in many source languages on [http://ufw.io/stories/](http://ufw.io/stories/). There are also translations of the Bible there to be used as sources for translation in English, and soon other languages, as well.
"""
converted = ta.fix_links(content)
self.assertEqual(converted, expected)