def create_pdf(lang_code, checking_level, version):
global download_dir, unfoldingWord_dir
# Create PDF via ConTeXt
try:
print_ok('BEGINNING: ', 'PDF generation.')
out_dir = os.path.join(download_dir, 'make_pdf')
make_dir(out_dir)
# generate a tex file
print('Generating tex file...', end=' ')
tex_file = os.path.join(out_dir, '{0}.tex'.format(lang_code))
# make sure it doesn't already exist
if os.path.isfile(tex_file):
os.remove(tex_file)
with OBSTexExport(lang_code, tex_file, 0, '360px',
checking_level) as tex:
tex.run()
print('finished.')
# run context
print_notice('Running context - this may take several minutes.')
# noinspection PyTypeChecker
trackers = ','.join([
'afm.loading', 'fonts.missing', 'fonts.warnings', 'fonts.names',
'fonts.specifications', 'fonts.scaling', 'system.dump'
])
cmd = 'context --paranoid --batchmode --trackers={0} "{1}"'.format(
trackers, tex_file)
try:
subprocess.check_call(cmd,
shell=True,
stderr=subprocess.STDOUT,
cwd=out_dir)
except subprocess.CalledProcessError as e:
if e.message:
raise e
print('Finished running context.')
print('Copying PDF to API...', end=' ')
version = version.replace('.', '_')
if version[0:1] != 'v':
version = 'v' + version
pdf_file = os.path.join(unfoldingWord_dir, lang_code,
'obs-{0}-{1}.pdf'.format(lang_code, version))
shutil.copyfile(os.path.join(out_dir, '{0}.pdf'.format(lang_code)),
pdf_file)
print('finished.')
finally:
print_ok('FINISHED:', 'generating PDF.')
def create_pdf(lang_code, checking_level, version):
global download_dir, unfoldingWord_dir
# Create PDF via ConTeXt
try:
print_ok('BEGINNING: ', 'PDF generation.')
out_dir = os.path.join(download_dir, 'make_pdf')
make_dir(out_dir)
# generate a tex file
print('Generating tex file...', end=' ')
tex_file = os.path.join(out_dir, '{0}.tex'.format(lang_code))
# make sure it doesn't already exist
if os.path.isfile(tex_file):
os.remove(tex_file)
with OBSTexExport(lang_code, tex_file, 0, '360px', checking_level) as tex:
tex.run()
print('finished.')
# run context
print_notice('Running context - this may take several minutes.')
# noinspection PyTypeChecker
trackers = ','.join(['afm.loading', 'fonts.missing', 'fonts.warnings', 'fonts.names',
'fonts.specifications', 'fonts.scaling', 'system.dump'])
cmd = 'context --paranoid --batchmode --trackers={0} "{1}"'.format(trackers, tex_file)
try:
subprocess.check_call(cmd, shell=True, stderr=subprocess.STDOUT, cwd=out_dir)
except subprocess.CalledProcessError as e:
if e.message:
raise e
print('Finished running context.')
print('Copying PDF to API...', end=' ')
version = version.replace('.', '_')
if version[0:1] != 'v':
version = 'v' + version
pdf_file = os.path.join(unfoldingWord_dir, lang_code, 'obs-{0}-{1}.pdf'.format(lang_code, version))
shutil.copyfile(os.path.join(out_dir, '{0}.pdf'.format(lang_code)), pdf_file)
print('finished.')
finally:
print_ok('FINISHED:', 'generating PDF.')
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('-r',
'--gitrepo',
dest='gitrepo',
default=False,
required=True,
help='Git repository where the source can be found.')
parser.add_argument('-o',
'--outdir',
dest='outdir',
default=False,
required=True,
help='The output directory for markdown files.')
parser.add_argument('-q',
'--quiet',
dest='quiet',
action='store_true',
help='Minimize console output.')
args = parser.parse_args(sys.argv[1:])
# call with closing to be sure the temp files get cleaned up
with closing(TransformOBS(args.gitrepo, args.outdir, args.quiet)) as tx:
tx.run()
print_ok('ALL FINISHED: ', 'Please check the output directory.')
'version': str(version)
}
for chp in tw_book_dict:
tw_book_dict[chp].sort(key=lambda x: x['id'])
entry = {'id': chp, 'frames': tw_book_dict[chp]}
tw_cat['chapters'].append(entry)
tw_cat['chapters'].sort(key=lambda x: x['id'])
write_json(filepath, tw_cat)
if __name__ == '__main__':
print()
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('-v',
'--version',
dest='version',
default=False,
required=True,
help='The version number.')
args = parser.parse_args(sys.argv[1:])
today = ''.join(str(datetime.date.today()).rsplit('-')[0:3])
print_ok('STARTING: ', 'publishing tN from Dokuwiki.')
run_tn(args.version, 'en', today)
print_ok('ALL FINISHED: ', 'publishing tN from Dokuwiki.')
print_notice('Don\'t forget to notify the interested parties.')
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('-r', '--gitrepo', dest='gitrepo', default=False,
required=True, help='Git repository where the source can be found.')
parser.add_argument('-t', '--tag', dest='tag', default='master',
required=False, help='Branch or tag to use as the source. Default is master.')
parser.add_argument('-d', '--domain', dest='domain', choices=['udb', 'ulb', 'pdb'],
required=True, help='ulb, udb or pdb')
args = parser.parse_args(sys.argv[1:])
# prompt user to update meta.json
print_notice('Check meta.json in the git repository and update the information if needed.')
prompt('Press Enter to continue when ready...')
try:
print_ok('STARTING: ', 'importing USFM repository.')
main(args.gitrepo, args.tag, args.domain)
print_ok('ALL FINISHED: ', 'importing USFM repository.')
print_notice('Don\'t forget to notify the interested parties.')
finally:
# delete temp files
if os.path.isdir(download_dir):
shutil.rmtree(download_dir, ignore_errors=True)
def import_obs(lang_data, git_repo, door43_url, no_pdf):
global download_dir, root, api_dir
lang_code = lang_data['lc']
# pre-flight checklist
link_source = '/var/www/vhosts/api.unfoldingword.org/httpdocs/obs/jpg/1/en'
if not os.path.isdir(link_source):
print_error('Image source directory not found: {0}.'.format(link_source))
sys.exit(1)
if git_repo[-1:] != '/':
git_repo += '/'
if no_pdf:
tools_dir = None
else:
tools_dir = '/var/www/vhosts/door43.org/tools'
if not os.path.isdir(tools_dir):
tools_dir = os.path.expanduser('~/Projects/tools')
# prompt if tools not found
if not os.path.isdir(tools_dir):
tools_dir = None
print_notice('The tools directory was not found. The PDF cannot be generated.')
resp = prompt('Do you want to continue without generating a PDF? [Y|n]: ')
if resp != '' and resp != 'Y' and resp != 'y':
sys.exit(0)
if git_repo:
if git_repo[-1:] == '/':
git_repo = git_repo[:-1]
download_dir = '/tmp/{0}'.format(git_repo.rpartition('/')[2])
make_dir(download_dir)
# make sure OBS is initialized on Dokuwiki
test_dir = root.format(lang_code, '')
if not os.path.isdir(test_dir):
print_warning('It seems OBS has not been initialized on Door43.org for {0}'.format(lang_code))
sys.exit(1)
elif door43_url:
print_error('URL not yet implemented.')
return
else:
print_error('Source not provided.')
return
# get the source files from the git repository
if 'github' in git_repo:
# https://github.com/unfoldingWord/obs-ru
# https://raw.githubusercontent.com/unfoldingWord/obs-ru/master/obs-ru.json
raw_url = git_repo.replace('github.com', 'raw.githubusercontent.com')
elif 'git.door43.org' in git_repo:
raw_url = join_url_parts(git_repo, 'raw')
else:
# this is to keep IntelliJ happy, is should have been caught in sub main
return
# download needed files from the repository
file_suffix = '-{0}.json'.format(lang_code.lower())
files_to_download = [
join_url_parts(raw_url, 'master/obs' + file_suffix),
join_url_parts(raw_url, 'master/status' + file_suffix)
]
for file_to_download in files_to_download:
downloaded_file = os.path.join(download_dir, file_to_download.rpartition('/')[2])
try:
print('Downloading {0}...'.format(file_to_download), end=' ')
download_file(file_to_download, downloaded_file)
finally:
print('finished.')
# read the files from the git repository
file_suffix = '-{0}.json'.format(lang_code.lower())
obs_obj = None
status_obj = None
# front_matter_found = False
# back_matter_found = False
try:
print('Examining the files...', end=' ')
for root_path, dirs, files in os.walk(download_dir):
if not len(files):
continue
for git_file in files:
if git_file == 'obs' + file_suffix:
obs_obj = OBS(os.path.join(root_path, git_file))
elif git_file == 'status' + file_suffix:
status_obj = OBSStatus(os.path.join(root_path, git_file))
# elif 'front-matter' in git_file:
# front_matter_found = True
# elif 'back-matter' in git_file:
# back_matter_found = True
finally:
print('finished.')
# check data integrity
if not obs_obj.verify_all():
sys.exit(1)
if not status_obj:
print_error('The file "status{0}" was not found in the git repository.'.format(file_suffix))
sys.exit(1)
# create Dokuwiki pages
print_ok('Begin: ', 'creating Dokuwiki pages.')
for chapter in obs_obj.chapters:
chapter_title = '====== {0} ======'.format(chapter['title'])
chapter_ref = '//{0}//'.format(chapter['ref'])
chapter_body = ''
chapter_num = chapter['number'].zfill(2)
for frame in chapter['frames']:
chapter_body += '{{{{{0}?direct&}}}}\n\n{1}\n\n'.format(frame['img'], frame['text'])
file_name = root.format(lang_code, chapter_num + '.txt')
print(' Writing {0}'.format(file_name))
with codecs.open(file_name, 'w', 'utf-8-sig') as out_file:
out_file.write('{0}\n\n{1}{2}\n'.format(chapter_title, chapter_body, chapter_ref))
print_ok('Finished: ', 'creating Dokuwiki pages.')
# Create image symlinks on api.unfoldingword.org
try:
print('Creating symlink to images directory...', end=' ')
link_name = '/var/www/vhosts/api.unfoldingword.org/httpdocs/obs/jpg/1/{0}'.format(lang_code.lower())
if not os.path.isfile(link_name) and not os.path.isdir(link_name) and not os.path.islink(link_name):
os.symlink(link_source, link_name)
finally:
print('finished.')
# Create PDF via ConTeXt
if not no_pdf and tools_dir and os.path.isdir(tools_dir):
try:
print_ok('Beginning: ', 'PDF generation.')
script_file = os.path.join(tools_dir, 'obs/book/pdf_export.sh')
out_dir = api_dir.format(lang_code)
make_dir(out_dir)
process = subprocess.Popen([script_file,
'-l', lang_code,
'-c', status_obj.checking_level,
'-v', status_obj.version,
'-o', out_dir],
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# wait for the process to terminate
out, err = process.communicate()
exit_code = process.returncode
out = out.strip().decode('utf-8')
err = err.strip().decode('utf-8')
# the error message may be in stdout
if exit_code != 0:
if not err:
err = out
out = None
if err:
print_error(err, 2)
if out:
print(' ' + out)
print(' PDF subprocess finished with exit code {0}'.format(exit_code))
finally:
print_ok('Finished:', 'generating PDF.')
'These USFM errors must be corrected before publishing can continue.'
)
sys.exit(1)
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('-d',
'--directory',
dest='directory',
default=False,
required=True,
help='The directory to check.')
parser.add_argument(
'-v',
'--versification',
dest='versification',
default='ufw',
required=False,
help=
'Versification system - current options are "ufw" (unfoldingWord) and "rsc" (Russian)'
)
args = parser.parse_args(sys.argv[1:])
print_ok('STARTING: ', 'validating USFM files.')
main(args.directory, args.versification)
print_ok('ALL FINISHED: ', 'validating USFM files.')
def main(git_repo, tag, no_pdf):
global download_dir
# clean up the git repo url
if git_repo[-4:] == '.git':
git_repo = git_repo[:-4]
if git_repo[-1:] == '/':
git_repo = git_repo[:-1]
# initialize some variables
today = ''.join(str(datetime.date.today()).rsplit('-')[0:3]) # str(datetime.date.today())
download_dir = '/tmp/{0}'.format(git_repo.rpartition('/')[2])
make_dir(download_dir)
downloaded_file = '{0}/{1}.zip'.format(download_dir, git_repo.rpartition('/')[2])
file_to_download = join_url_parts(git_repo, 'archive/' + tag + '.zip')
manifest = None
status = None # type: OBSStatus
content_dir = None
# download the repository
try:
print('Downloading {0}...'.format(file_to_download), end=' ')
if not os.path.isfile(downloaded_file):
download_file(file_to_download, downloaded_file)
finally:
print('finished.')
try:
print('Unzipping...'.format(downloaded_file), end=' ')
unzip(downloaded_file, download_dir)
finally:
print('finished.')
# examine the repository
for root, dirs, files in os.walk(download_dir):
if 'manifest.json' in files:
# read the manifest
try:
print('Reading the manifest...', end=' ')
content_dir = root
manifest = load_json_object(os.path.join(root, 'manifest.json'))
finally:
print('finished.')
if 'status.json' in files:
# read the meta data
try:
print('Reading the status...', end=' ')
content_dir = root
status = OBSStatus(os.path.join(root, 'status.json'))
finally:
print('finished.')
# if we have everything, exit the loop
if content_dir and manifest and status:
break
# check for valid repository structure
if not manifest:
print_error('Did not find manifest.json in {}'.format(git_repo))
sys.exit(1)
if not status:
print_error('Did not find status.json in {}'.format(git_repo))
sys.exit(1)
print('Initializing OBS object...', end=' ')
lang = manifest['target_language']['id']
obs_obj = OBS()
obs_obj.date_modified = today
obs_obj.direction = manifest['target_language']['direction']
obs_obj.language = lang
print('finished')
obs_obj.chapters = load_obs_chapters(content_dir)
obs_obj.chapters.sort(key=lambda c: c['number'])
if not obs_obj.verify_all():
print_error('Quality check did not pass.')
sys.exit(1)
print('Loading languages...', end=' ')
lang_dict = OBS.load_lang_strings()
print('finished.')
print('Loading the catalog...', end=' ')
export_dir = '/var/www/vhosts/door43.org/httpdocs/exports'
# uw_cat_path = os.path.join(unfoldingWord_dir, 'obs-catalog.json')
# uw_catalog = load_json_object(uw_cat_path, [])
# uw_cat_langs = [x['language'] for x in uw_catalog]
cat_path = os.path.join(export_dir, 'obs-catalog.json')
catalog = load_json_object(cat_path, [])
print('finished')
print('Getting already published languages...', end=' ')
json_lang_file_path = os.path.join(export_dir, lang, 'obs', 'obs-{0}.json'.format(lang))
# prev_json_lang = load_json_object(json_lang_file_path, {})
if lang not in lang_dict:
print("Configuration for language {0} missing.".format(lang))
sys.exit(1)
print('finished.')
updated = update_language_catalog(lang, obs_obj.direction, status, today, lang_dict, catalog)
print('Writing the OBS file to the exports directory...', end=' ')
cur_json = json.dumps(obs_obj, sort_keys=True, cls=OBSEncoder)
if updated:
([x for x in catalog if x['language'] == lang][0]['date_modified']) = today
write_file(json_lang_file_path.replace('.txt', '.json'), cur_json)
print('finished.')
export_to_api(lang, status, today, cur_json)
cat_json = json.dumps(catalog, sort_keys=True, cls=OBSEncoder)
write_file(cat_path, cat_json)
# update the catalog
print_ok('STARTING: ', 'updating the catalogs.')
update_catalog()
print_ok('FINISHED: ', 'updating the catalogs.')
if no_pdf:
return
create_pdf(lang, status.checking_level, status.version)
chunk_url = '{0}/{1}/chunks.json'.format(base_url, book[0])
chunk_defs = json.loads(get_url(chunk_url))
for chunk in chunk_defs:
# is this is a different chapter?
if chap_num != chunk['chp']:
chap_num = chunk['chp']
if current:
file_contents.append(current)
current = {'chapter': int(chunk['chp']), 'first_verses': []}
current['first_verses'].append(int(chunk['firstvs']))
if current:
file_contents.append(current)
# format the output
file_contents_str = json.dumps(file_contents, sort_keys=True, indent=2)
file_contents_str = regex1.sub(r'\1\2', file_contents_str)
file_contents_str = regex2.sub(r'\1 \2', file_contents_str)
file_contents_str = regex3.sub(r'\1\2', file_contents_str)
file_contents_str = regex4.sub(r'\1', file_contents_str)
write_file(os.path.join(output_dir, '{0}.json'.format(book[0])), file_contents_str)
if __name__ == '__main__':
import_now()
print_ok('FINISHED: ', 'Done importing chunk definitions.')
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('-l', '--lang', dest="lang", default=False,
required=True, help="Language code of resource.")
parser.add_argument('-e', '--export', dest="uwexport", default=False,
action='store_true', help="Export to unfoldingWord.")
parser.add_argument('-t', '--testexport', dest="testexport", default=False,
action='store_true', help="Test export to unfoldingWord.")
parser.add_argument('-p', '--nopdf', dest='nopdf', action='store_true', help='Do not produce a PDF.')
args = parser.parse_args(sys.argv[1:])
lang = args.lang
uw_export = args.uwexport
test_export = args.testexport
no_pdf = args.nopdf
print_ok('STARTING: ', 'importing OBS from Dokuwiki')
# pre-flight checklist
link_source = '/var/www/vhosts/api.unfoldingword.org/httpdocs/obs/jpg/1/en'
if not os.path.isdir(link_source):
print_error('Image source directory not found: {0}.'.format(link_source))
sys.exit(1)
if no_pdf:
tools_dir = None
else:
tools_dir = '/var/www/vhosts/door43.org/tools'
if not os.path.isdir(tools_dir):
tools_dir = os.path.expanduser('~/Projects/tools')
# prompt if tools not found
chunk_defs = json.loads(get_url(chunk_url))
for chunk in chunk_defs:
# is this is a different chapter?
if chap_num != chunk['chp']:
chap_num = chunk['chp']
if current:
file_contents.append(current)
current = {'chapter': int(chunk['chp']), 'first_verses': []}
current['first_verses'].append(int(chunk['firstvs']))
if current:
file_contents.append(current)
# format the output
file_contents_str = json.dumps(file_contents, sort_keys=True, indent=2)
file_contents_str = regex1.sub(r'\1\2', file_contents_str)
file_contents_str = regex2.sub(r'\1 \2', file_contents_str)
file_contents_str = regex3.sub(r'\1\2', file_contents_str)
file_contents_str = regex4.sub(r'\1', file_contents_str)
write_file(os.path.join(output_dir, '{0}.json'.format(book[0])),
file_contents_str)
if __name__ == '__main__':
import_now()
print_ok('FINISHED: ', 'Done importing chunk definitions.')
if errors_found:
print_error(
'These USFM errors must be corrected before publishing can continue.'
)
sys.exit(1)
# rebuild source for tS
print()
print('Publishing to the API...')
with api_publish(api_directory) as api:
api.run()
print('Finished publishing to the API.')
# update the catalog
print()
print('Updating the catalogs...', end=' ')
update_catalog()
print('finished.')
if __name__ == '__main__':
print_ok('STARTING: ', 're-chunking all Bibles.')
source_directories = get_source_directories()
for source_directory in source_directories:
rechunk_this_one(source_directory)
print_ok('ALL FINISHED: ', 're-chunking all Bibles.')
default=False,
action='store_true',
help="Test export to unfoldingWord.")
parser.add_argument('-p',
'--nopdf',
dest='nopdf',
action='store_true',
help='Do not produce a PDF.')
args = parser.parse_args(sys.argv[1:])
lang = args.lang
uw_export = args.uwexport
test_export = args.testexport
no_pdf = args.nopdf
print_ok('STARTING: ', 'importing OBS from Dokuwiki')
# pre-flight checklist
link_source = '/var/www/vhosts/api.unfoldingword.org/httpdocs/obs/jpg/1/en'
if not os.path.isdir(link_source):
print_error(
'Image source directory not found: {0}.'.format(link_source))
sys.exit(1)
if no_pdf:
tools_dir = None
else:
tools_dir = '/var/www/vhosts/door43.org/tools'
if not os.path.isdir(tools_dir):
tools_dir = os.path.expanduser('~/Projects/tools')
#!/usr/bin/env python2
# -*- coding: utf8 -*-
#
# Copyright (c) 2016 unfoldingWord
# http://creativecommons.org/licenses/MIT/
# See LICENSE file for details.
#
# Contributors:
# Phil Hopper <*****@*****.**>
#
from __future__ import print_function, unicode_literals
from general_tools.print_utils import print_ok
from uw.update_catalog import update_catalog
if __name__ == '__main__':
print()
print_ok('STARTING: ', 'updating the catalogs.')
update_catalog()
print_ok('FINISHED: ', 'updating the catalogs.')
print('finished.')
if errors_found:
print_error('These USFM errors must be corrected before publishing can continue.')
sys.exit(1)
# rebuild source for tS
print()
print('Publishing to the API...')
with api_publish(api_directory) as api:
api.run()
print('Finished publishing to the API.')
# update the catalog
print()
print('Updating the catalogs...', end=' ')
update_catalog()
print('finished.')
if __name__ == '__main__':
print_ok('STARTING: ', 're-chunking all Bibles.')
source_directories = get_source_directories()
for source_directory in source_directories:
rechunk_this_one(source_directory)
print_ok('ALL FINISHED: ', 're-chunking all Bibles.')
def main(git_repo, tag, no_pdf):
global download_dir
# clean up the git repo url
if git_repo[-4:] == '.git':
git_repo = git_repo[:-4]
if git_repo[-1:] == '/':
git_repo = git_repo[:-1]
# initialize some variables
today = ''.join(str(datetime.date.today()).rsplit(
str('-'))[0:3]) # str(datetime.date.today())
download_dir = '/tmp/{0}'.format(git_repo.rpartition('/')[2])
make_dir(download_dir)
downloaded_file = '{0}/{1}.zip'.format(download_dir,
git_repo.rpartition('/')[2])
file_to_download = join_url_parts(git_repo, 'archive/{0}.zip'.format(tag))
manifest = None
status = None # type: OBSStatus
content_dir = None
# download the repository
try:
print('Downloading {0}...'.format(file_to_download), end=' ')
if not os.path.isfile(downloaded_file):
download_file(file_to_download, downloaded_file)
finally:
print('finished.')
try:
print('Unzipping...'.format(downloaded_file), end=' ')
unzip(downloaded_file, download_dir)
finally:
print('finished.')
# examine the repository
for root, dirs, files in os.walk(download_dir):
if 'manifest.json' in files:
# read the manifest
try:
print('Reading the manifest...', end=' ')
content_dir = root
manifest = load_json_object(os.path.join(
root, 'manifest.json'))
status = OBSStatus.from_manifest(manifest)
finally:
print('finished.')
if 'content' in dirs:
content_dir = os.path.join(root, 'content')
# if we have everything, exit the loop
if content_dir and manifest and status:
break
# check for valid repository structure
if not manifest:
print_error('Did not find manifest.json in {}'.format(git_repo))
sys.exit(1)
print('Initializing OBS object...', end=' ')
lang = manifest['language']['slug']
obs_obj = OBS()
obs_obj.date_modified = today
obs_obj.direction = manifest['language']['dir']
obs_obj.language = lang
print('finished')
obs_obj.chapters = load_obs_chapters(content_dir)
obs_obj.chapters.sort(key=lambda c: int(c['number']))
if not obs_obj.verify_all():
print_error('Quality check did not pass.')
sys.exit(1)
print('Loading languages...', end=' ')
lang_dict = OBS.load_lang_strings()
print('finished.')
print('Loading the catalog...', end=' ')
export_dir = '/var/www/vhosts/door43.org/httpdocs/exports'
cat_path = os.path.join(export_dir, 'obs-catalog.json')
catalog = load_json_object(cat_path, [])
print('finished')
print('Getting already published languages...', end=' ')
json_lang_file_path = os.path.join(export_dir, lang, 'obs',
'obs-{0}.json'.format(lang))
if lang not in lang_dict:
print("Configuration for language {0} missing.".format(lang))
sys.exit(1)
print('finished.')
updated = update_language_catalog(lang, obs_obj.direction, status, today,
lang_dict, catalog)
print('Writing the OBS file to the exports directory...', end=' ')
cur_json = json.dumps(obs_obj, sort_keys=True, cls=OBSEncoder)
if updated:
([x for x in catalog
if x['language'] == lang][0]['date_modified']) = today
# noinspection PyTypeChecker
write_file(json_lang_file_path.replace('.txt', '.json'), cur_json)
print('finished.')
export_to_api(lang, status, today, cur_json)
cat_json = json.dumps(catalog, sort_keys=True, cls=OBSEncoder)
write_file(cat_path, cat_json)
# update the catalog
print_ok('STARTING: ', 'updating the catalogs.')
update_catalog()
print_ok('FINISHED: ', 'updating the catalogs.')
if no_pdf:
return
create_pdf(lang, status.checking_level, status.version)
'Currently only github and git.door43.org repositories are supported.'
)
sys.exit(0)
# prompt user to update status.txt
print_notice(
'Check status-{0}.json in the git repository and update the information if needed.'
.format(args.lang))
prompt('Press Enter to continue when ready...')
print()
# do the import
import_obs(this_lang, args.gitrepo, args.url, args.nopdf)
# update the catalog
print_ok('STARTING: ', 'updating the catalogs.')
update_catalog()
print_ok('FINISHED: ', 'updating the catalogs.')
# update OBS in-progress
print_ok('STARTING: ', 'updating OBS in progress.')
ObsInProgress.run()
print_ok('FINISHED: ', 'updating OBS in progress.')
print_ok('ALL FINISHED: ',
'Please check door43.org and api.unfoldingword.org.')
print_notice('Don\'t forget to notify the interested parties.')
finally:
# delete temp files
if os.path.isdir(download_dir):
print('finished.')
return chapters
if __name__ == '__main__':
print()
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('-r', '--gitrepo', dest='gitrepo', default=False,
required=True, help='Git repository where the source can be found.')
parser.add_argument('-t', '--tag', dest='tag', default='master',
required=False, help='Branch or tag to use as the source. Default is master.')
parser.add_argument('-p', '--nopdf', dest='nopdf', action='store_true', help='Do not produce a PDF.')
args = parser.parse_args(sys.argv[1:])
# prompt user to update status.json
print_notice('Check status.json in the git repository and update the information if needed.')
prompt('Press Enter to continue when ready...')
try:
print_ok('STARTING: ', 'publishing OBS repository.')
main(args.gitrepo, args.tag, args.nopdf)
print_ok('ALL FINISHED: ', 'publishing OBS repository.')
print_notice('Don\'t forget to notify the interested parties.')
finally:
# delete temp files
if os.path.isdir(download_dir):
shutil.rmtree(download_dir, ignore_errors=True)
def import_obs(lang_data, git_repo, door43_url, no_pdf):
global download_dir, root, api_dir
lang_code = lang_data['lc']
# pre-flight checklist
link_source = '/var/www/vhosts/api.unfoldingword.org/httpdocs/obs/jpg/1/en'
if not os.path.isdir(link_source):
print_error(
'Image source directory not found: {0}.'.format(link_source))
sys.exit(1)
if git_repo[-1:] != '/':
git_repo += '/'
if no_pdf:
tools_dir = None
else:
tools_dir = '/var/www/vhosts/door43.org/tools'
if not os.path.isdir(tools_dir):
tools_dir = os.path.expanduser('~/Projects/tools')
# prompt if tools not found
if not os.path.isdir(tools_dir):
tools_dir = None
print_notice(
'The tools directory was not found. The PDF cannot be generated.'
)
resp = prompt(
'Do you want to continue without generating a PDF? [Y|n]: ')
if resp != '' and resp != 'Y' and resp != 'y':
sys.exit(0)
if git_repo:
if git_repo[-1:] == '/':
git_repo = git_repo[:-1]
download_dir = '/tmp/{0}'.format(git_repo.rpartition('/')[2])
make_dir(download_dir)
# make sure OBS is initialized on Dokuwiki
test_dir = root.format(lang_code, '')
if not os.path.isdir(test_dir):
print_warning(
'It seems OBS has not been initialized on Door43.org for {0}'.
format(lang_code))
sys.exit(1)
elif door43_url:
print_error('URL not yet implemented.')
return
else:
print_error('Source not provided.')
return
# get the source files from the git repository
if 'github' in git_repo:
# https://github.com/unfoldingWord/obs-ru
# https://raw.githubusercontent.com/unfoldingWord/obs-ru/master/obs-ru.json
raw_url = git_repo.replace('github.com', 'raw.githubusercontent.com')
elif 'git.door43.org' in git_repo:
raw_url = join_url_parts(git_repo, 'raw')
else:
# this is to keep IntelliJ happy, is should have been caught in sub main
return
# download needed files from the repository
file_suffix = '-{0}.json'.format(lang_code.lower())
files_to_download = [
join_url_parts(raw_url, 'master/obs' + file_suffix),
join_url_parts(raw_url, 'master/status' + file_suffix)
]
for file_to_download in files_to_download:
downloaded_file = os.path.join(download_dir,
file_to_download.rpartition('/')[2])
try:
print('Downloading {0}...'.format(file_to_download), end=' ')
download_file(file_to_download, downloaded_file)
finally:
print('finished.')
# read the files from the git repository
file_suffix = '-{0}.json'.format(lang_code.lower())
obs_obj = None
status_obj = None
# front_matter_found = False
# back_matter_found = False
try:
print('Examining the files...', end=' ')
for root_path, dirs, files in os.walk(download_dir):
if not len(files):
continue
for git_file in files:
if git_file == 'obs' + file_suffix:
obs_obj = OBS(os.path.join(root_path, git_file))
elif git_file == 'status' + file_suffix:
status_obj = OBSStatus(os.path.join(root_path, git_file))
# elif 'front-matter' in git_file:
# front_matter_found = True
# elif 'back-matter' in git_file:
# back_matter_found = True
finally:
print('finished.')
# check data integrity
if not obs_obj.verify_all():
sys.exit(1)
if not status_obj:
print_error(
'The file "status{0}" was not found in the git repository.'.format(
file_suffix))
sys.exit(1)
# create Dokuwiki pages
print_ok('Begin: ', 'creating Dokuwiki pages.')
for chapter in obs_obj.chapters:
chapter_title = '====== {0} ======'.format(chapter['title'])
chapter_ref = '//{0}//'.format(chapter['ref'])
chapter_body = ''
chapter_num = chapter['number'].zfill(2)
for frame in chapter['frames']:
chapter_body += '{{{{{0}?direct&}}}}\n\n{1}\n\n'.format(
frame['img'], frame['text'])
file_name = root.format(lang_code, chapter_num + '.txt')
print(' Writing {0}'.format(file_name))
with codecs.open(file_name, 'w', 'utf-8-sig') as out_file:
out_file.write('{0}\n\n{1}{2}\n'.format(chapter_title,
chapter_body, chapter_ref))
print_ok('Finished: ', 'creating Dokuwiki pages.')
# Create image symlinks on api.unfoldingword.org
try:
print('Creating symlink to images directory...', end=' ')
link_name = '/var/www/vhosts/api.unfoldingword.org/httpdocs/obs/jpg/1/{0}'.format(
lang_code.lower())
if not os.path.isfile(link_name) and not os.path.isdir(
link_name) and not os.path.islink(link_name):
os.symlink(link_source, link_name)
finally:
print('finished.')
# Create PDF via ConTeXt
if not no_pdf and tools_dir and os.path.isdir(tools_dir):
try:
print_ok('Beginning: ', 'PDF generation.')
script_file = os.path.join(tools_dir, 'obs/book/pdf_export.sh')
out_dir = api_dir.format(lang_code)
make_dir(out_dir)
process = subprocess.Popen([
script_file, '-l', lang_code, '-c', status_obj.checking_level,
'-v', status_obj.version, '-o', out_dir
],
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# wait for the process to terminate
out, err = process.communicate()
exit_code = process.returncode
out = out.strip().decode('utf-8')
err = err.strip().decode('utf-8')
# the error message may be in stdout
if exit_code != 0:
if not err:
err = out
out = None
if err:
print_error(err, 2)
if out:
print(' ' + out)
print(' PDF subprocess finished with exit code {0}'.format(
exit_code))
finally:
print_ok('Finished:', 'generating PDF.')
print_error('{0}: {1}'.format(ioe.strerror, ioe.filename))
self.errors.append(ioe)
except Exception as e:
print_error(e.message)
self.errors.append(e)
finally:
print('finished.')
except Exception as e:
print_error(e.message)
self.errors.append(e)
if __name__ == '__main__':
print()
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('-r', '--gitrepo', dest='gitrepo', default=False,
required=True, help='Git repository where the source can be found.')
parser.add_argument('-o', '--outdir', dest='outdir', default=False,
required=True, help='The output directory for markdown files.')
args = parser.parse_args(sys.argv[1:])
# call with closing to be sure the temp files get cleaned up
with closing(TransformOBS(args.gitrepo, args.outdir)) as tx:
tx.run()
print_ok('ALL FINISHED: ', 'Please check the output directory.')
book_text = s5_re.sub('', book_text)
# get the usfm for the book
book.set_usfm(book_text)
# do basic checks
book.verify_chapters_and_verses(True)
if book.validation_errors:
errors_found = True
print('finished.')
# stop if errors were found
if errors_found:
print_error('These USFM errors must be corrected before publishing can continue.')
sys.exit(1)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('-d', '--directory', dest='directory', default=False, required=True,
help='The directory to check.')
parser.add_argument('-v', '--versification', dest='versification', default='ufw', required=False,
help='Versification system - current options are "ufw" (unfoldingWord) and "rsc" (Russian)')
args = parser.parse_args(sys.argv[1:])
print_ok('STARTING: ', 'validating USFM files.')
main(args.directory, args.versification)
print_ok('ALL FINISHED: ', 'validating USFM files.')
dest='gitrepo',
default=False,
required=True,
help='Git repository where the source can be found.')
parser.add_argument(
'-t',
'--tag',
dest='tag',
default='master',
required=False,
help='Branch or tag to use as the source. Default is master.')
args = parser.parse_args(sys.argv[1:])
# prompt user to update meta.json
print_notice(
'Check meta.yaml in the git repository and update the information if needed.'
)
prompt('Press Enter to continue when ready...')
try:
print_ok('STARTING: ', 'publishing TA repository.')
main(args.gitrepo, args.tag)
print_ok('ALL FINISHED: ', 'publishing TA repository.')
print_notice('Don\'t forget to notify the interested parties.')
finally:
# delete temp files
if os.path.isdir(download_dir):
shutil.rmtree(download_dir, ignore_errors=True)
if 'github' not in args.gitrepo and 'git.door43.org' not in args.gitrepo:
print_warning('Currently only github and git.door43.org repositories are supported.')
sys.exit(0)
# prompt user to update status.txt
print_notice(
'Check status-{0}.json in the git repository and update the information if needed.'.format(args.lang))
prompt('Press Enter to continue when ready...')
print()
# do the import
import_obs(this_lang, args.gitrepo, args.url, args.nopdf)
# update the catalog
print_ok('STARTING: ', 'updating the catalogs.')
update_catalog()
print_ok('FINISHED: ', 'updating the catalogs.')
# update OBS in-progress
print_ok('STARTING: ', 'updating OBS in progress.')
ObsInProgress.run()
print_ok('FINISHED: ', 'updating OBS in progress.')
print_ok('ALL FINISHED: ', 'Please check door43.org and api.unfoldingword.org.')
print_notice('Don\'t forget to notify the interested parties.')
finally:
# delete temp files
if os.path.isdir(download_dir):
shutil.rmtree(download_dir, ignore_errors=True)
