def create_pdf(lang_code, checking_level, version):
global download_dir, unfoldingWord_dir
# Create PDF via ConTeXt
try:
print_ok('BEGINNING: ', 'PDF generation.')
out_dir = os.path.join(download_dir, 'make_pdf')
make_dir(out_dir)
# generate a tex file
print('Generating tex file...', end=' ')
tex_file = os.path.join(out_dir, '{0}.tex'.format(lang_code))
# make sure it doesn't already exist
if os.path.isfile(tex_file):
os.remove(tex_file)
with OBSTexExport(lang_code, tex_file, 0, '360px',
checking_level) as tex:
tex.run()
print('finished.')
# run context
print_notice('Running context - this may take several minutes.')
# noinspection PyTypeChecker
trackers = ','.join([
'afm.loading', 'fonts.missing', 'fonts.warnings', 'fonts.names',
'fonts.specifications', 'fonts.scaling', 'system.dump'
])
cmd = 'context --paranoid --batchmode --trackers={0} "{1}"'.format(
trackers, tex_file)
try:
subprocess.check_call(cmd,
shell=True,
stderr=subprocess.STDOUT,
cwd=out_dir)
except subprocess.CalledProcessError as e:
if e.message:
raise e
print('Finished running context.')
print('Copying PDF to API...', end=' ')
version = version.replace('.', '_')
if version[0:1] != 'v':
version = 'v' + version
pdf_file = os.path.join(unfoldingWord_dir, lang_code,
'obs-{0}-{1}.pdf'.format(lang_code, version))
shutil.copyfile(os.path.join(out_dir, '{0}.pdf'.format(lang_code)),
pdf_file)
print('finished.')
finally:
print_ok('FINISHED:', 'generating PDF.')
def create_pdf(lang_code, checking_level, version):
global download_dir, unfoldingWord_dir
# Create PDF via ConTeXt
try:
print_ok('BEGINNING: ', 'PDF generation.')
out_dir = os.path.join(download_dir, 'make_pdf')
make_dir(out_dir)
# generate a tex file
print('Generating tex file...', end=' ')
tex_file = os.path.join(out_dir, '{0}.tex'.format(lang_code))
# make sure it doesn't already exist
if os.path.isfile(tex_file):
os.remove(tex_file)
with OBSTexExport(lang_code, tex_file, 0, '360px', checking_level) as tex:
tex.run()
print('finished.')
# run context
print_notice('Running context - this may take several minutes.')
# noinspection PyTypeChecker
trackers = ','.join(['afm.loading', 'fonts.missing', 'fonts.warnings', 'fonts.names',
'fonts.specifications', 'fonts.scaling', 'system.dump'])
cmd = 'context --paranoid --batchmode --trackers={0} "{1}"'.format(trackers, tex_file)
try:
subprocess.check_call(cmd, shell=True, stderr=subprocess.STDOUT, cwd=out_dir)
except subprocess.CalledProcessError as e:
if e.message:
raise e
print('Finished running context.')
print('Copying PDF to API...', end=' ')
version = version.replace('.', '_')
if version[0:1] != 'v':
version = 'v' + version
pdf_file = os.path.join(unfoldingWord_dir, lang_code, 'obs-{0}-{1}.pdf'.format(lang_code, version))
shutil.copyfile(os.path.join(out_dir, '{0}.pdf'.format(lang_code)), pdf_file)
print('finished.')
finally:
print_ok('FINISHED:', 'generating PDF.')
dest='gitrepo',
default=False,
required=True,
help='Git repository where the source can be found.')
parser.add_argument(
'-t',
'--tag',
dest='tag',
default='master',
required=False,
help='Branch or tag to use as the source. Default is master.')
args = parser.parse_args(sys.argv[1:])
# prompt user to update meta.json
print_notice(
'Check meta.yaml in the git repository and update the information if needed.'
)
prompt('Press Enter to continue when ready...')
try:
print_ok('STARTING: ', 'publishing TA repository.')
main(args.gitrepo, args.tag)
print_ok('ALL FINISHED: ', 'publishing TA repository.')
print_notice('Don\'t forget to notify the interested parties.')
finally:
# delete temp files
if os.path.isdir(download_dir):
shutil.rmtree(download_dir, ignore_errors=True)
print('finished.')
return chapters
if __name__ == '__main__':
print()
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('-r', '--gitrepo', dest='gitrepo', default=False,
required=True, help='Git repository where the source can be found.')
parser.add_argument('-t', '--tag', dest='tag', default='master',
required=False, help='Branch or tag to use as the source. Default is master.')
parser.add_argument('-p', '--nopdf', dest='nopdf', action='store_true', help='Do not produce a PDF.')
args = parser.parse_args(sys.argv[1:])
# prompt user to update status.json
print_notice('Check status.json in the git repository and update the information if needed.')
prompt('Press Enter to continue when ready...')
try:
print_ok('STARTING: ', 'publishing OBS repository.')
main(args.gitrepo, args.tag, args.nopdf)
print_ok('ALL FINISHED: ', 'publishing OBS repository.')
print_notice('Don\'t forget to notify the interested parties.')
finally:
# delete temp files
if os.path.isdir(download_dir):
shutil.rmtree(download_dir, ignore_errors=True)
'--gitrepo',
dest='gitrepo',
default=False,
required=True,
help='Git repository where the source can be found.')
parser.add_argument(
'-t',
'--tag',
dest='tag',
default='master',
required=False,
help='Branch or tag to use as the source. Default is master.')
parser.add_argument('-p',
'--nopdf',
dest='nopdf',
action='store_true',
help='Do not produce a PDF.')
args = parser.parse_args(sys.argv[1:])
try:
print_ok('STARTING: ', 'publishing OBS repository.')
main(args.gitrepo, args.tag, args.nopdf)
print_ok('ALL FINISHED: ', 'publishing OBS repository.')
print_notice('Don\'t forget to notify the interested parties.')
finally:
# delete temp files
if os.path.isdir(download_dir):
shutil.rmtree(download_dir, ignore_errors=True)
def main(git_repo, tag, domain):
global download_dir, out_template
# clean up the git repo url
if git_repo[-4:] == '.git':
git_repo = git_repo[:-4]
if git_repo[-1:] == '/':
git_repo = git_repo[:-1]
# initialize some variables
today = ''.join(str(datetime.date.today()).rsplit('-')[0:3])
download_dir = '/tmp/{0}'.format(git_repo.rpartition('/')[2])
make_dir(download_dir)
downloaded_file = '{0}/{1}.zip'.format(download_dir, git_repo.rpartition('/')[2])
file_to_download = join_url_parts(git_repo, 'archive/' + tag + '.zip')
books_published = {}
metadata_obj = None
usfm_dir = None
# download the repository
try:
print('Downloading {0}...'.format(file_to_download), end=' ')
if not os.path.isfile(downloaded_file):
download_file(file_to_download, downloaded_file)
finally:
print('finished.')
try:
print('Unzipping...'.format(downloaded_file), end=' ')
unzip(downloaded_file, download_dir)
finally:
print('finished.')
# examine the repository
for root, dirs, files in os.walk(download_dir):
if 'meta.json' in files:
# read the metadata
try:
print('Reading the metadata...', end=' ')
metadata_obj = BibleMetaData(os.path.join(root, 'meta.json'))
finally:
print('finished.')
if 'usfm' in dirs:
usfm_dir = os.path.join(root, 'usfm')
# if we have everything, exit the loop
if usfm_dir and metadata_obj:
break
# check for valid repository structure
if not metadata_obj:
print_error('Did not find meta.json in {}'.format(git_repo))
sys.exit(1)
if not usfm_dir:
print_error('Did not find the usfm directory in {}'.format(git_repo))
sys.exit(1)
# get the versification data
vrs = Bible.get_versification(metadata_obj.versification) # type: list<Book>
out_dir = out_template.format(domain, metadata_obj.slug, metadata_obj.lang)
# walk through the usfm files
usfm_files = glob(os.path.join(usfm_dir, '*.usfm'))
errors_found = False
for usfm_file in usfm_files:
# read the file
with codecs.open(usfm_file, 'r', 'utf-8') as in_file:
book_text = in_file.read()
# get the book id
book_search = id_re.search(book_text)
if not book_search:
print_error('Book id not found in {}'.format(usfm_file))
sys.exit(1)
book_id = book_search.group(1)
print('Beginning {}...'.format(book_id), end=' ')
# get book versification info
book = next((b for b in vrs if b.book_id == book_id), None)
if not book:
print_error('Book versification data was not found for "{}"'.format(book_id))
sys.exit(1)
# remove \s5 lines
book_text = s5_re.sub('', book_text)
# get the usfm for the book
book.set_usfm(book_text)
# do basic checks
book.verify_usfm_tags()
book.verify_chapters_and_verses(True)
if book.validation_errors:
errors_found = True
# get chunks for this book
Bible.chunk_book(metadata_obj.versification, book)
book.apply_chunks()
# produces something like '01-GEN.usfm'
book_file_name = '{0}-{1}.usfm'.format(str(book.number).zfill(2), book.book_id)
print('Writing ' + book_file_name + '...', end=' ')
write_file('{0}/{1}'.format(out_dir, book_file_name), book.usfm)
meta = ['Bible: OT']
if book.number > 39:
meta = ['Bible: NT']
books_published[book.book_id.lower()] = {'name': book.name,
'meta': meta,
'sort': str(book.number).zfill(2),
'desc': ''
}
print('finished.')
# stop if errors were found
if errors_found:
print_error('These USFM errors must be corrected before publishing can continue.')
sys.exit(1)
print('Writing status.json...', end=' ')
status = {"slug": '{0}'.format(metadata_obj.slug.lower()),
"name": metadata_obj.name,
"lang": metadata_obj.lang,
"date_modified": today,
"books_published": books_published,
"status": {"checking_entity": metadata_obj.checking_entity,
"checking_level": metadata_obj.checking_level,
"comments": metadata_obj.comments,
"contributors": metadata_obj.contributors,
"publish_date": today,
"source_text": metadata_obj.source_text,
"source_text_version": metadata_obj.source_text_version,
"version": metadata_obj.version
}
}
write_file('{0}/status.json'.format(out_dir), status, indent=2)
print('finished.')
print()
print('Publishing to the API...')
with api_publish(out_dir) as api:
api.run()
print('Finished publishing to the API.')
# update the catalog
print()
print('Updating the catalogs...', end=' ')
update_catalog()
print('finished.')
print_notice('Check {0} and do a git push'.format(out_dir))
def main(git_repo, tag, domain):
global download_dir, out_template
# clean up the git repo url
if git_repo[-4:] == '.git':
git_repo = git_repo[:-4]
if git_repo[-1:] == '/':
git_repo = git_repo[:-1]
# initialize some variables
today = ''.join(str(datetime.date.today()).rsplit('-')[0:3]) # str(datetime.date.today())
download_dir = '/tmp/{0}'.format(git_repo.rpartition('/')[2])
make_dir(download_dir)
downloaded_file = '{0}/{1}.zip'.format(download_dir, git_repo.rpartition('/')[2])
file_to_download = join_url_parts(git_repo, 'archive/' + tag + '.zip')
manifest = None
metadata_obj = None
content_dir = ''
usfm_file = None
# download the repository
try:
print('Downloading {0}...'.format(file_to_download), end=' ')
if not os.path.isfile(downloaded_file):
download_file(file_to_download, downloaded_file)
finally:
print('finished.')
try:
print('Unzipping...'.format(downloaded_file), end=' ')
unzip(downloaded_file, download_dir)
finally:
print('finished.')
# examine the repository
for root, dirs, files in os.walk(download_dir):
if 'manifest.json' in files:
# read the manifest
try:
print('Reading the manifest...', end=' ')
manifest = load_json_object(os.path.join(root, 'manifest.json'))
content_dir = root
# look for the usfm file for the whole book
found_usfm = glob(os.path.join(content_dir, '*.usfm'))
if len(found_usfm) == 1:
usfm_file = os.path.join(content_dir, found_usfm[0])
finally:
print('finished.')
if 'meta.json' in files:
# read the metadata
try:
print('Reading the metadata...', end=' ')
metadata_obj = BibleMetaData(os.path.join(root, 'meta.json'))
finally:
print('finished.')
# if we have everything, exit the loop
if manifest and metadata_obj:
break
# check for valid repository structure
if not manifest:
print_error('Did not find manifest.json in {}'.format(git_repo))
sys.exit(1)
if not metadata_obj:
print_error('Did not find meta.json in {}'.format(git_repo))
sys.exit(1)
# get the versification data
print('Getting versification info...', end=' ')
vrs = Bible.get_versification(metadata_obj.versification) # type: list<Book>
# get the book object for this repository
book = next((b for b in vrs if b.book_id.lower() == manifest['project']['id']), None) # type: Book
if not book:
print_error('Book versification data was not found for "{}"'.format(manifest['project']['id']))
sys.exit(1)
print('finished')
if usfm_file:
read_unified_file(book, usfm_file)
else:
read_chunked_files(book, content_dir, metadata_obj)
# do basic checks
print('Running USFM checks...', end=' ')
book.verify_chapters_and_verses(True)
if book.validation_errors:
print_error('These USFM errors must be corrected before publishing can continue.')
sys.exit(1)
else:
print('finished.')
# insert paragraph markers
print('Inserting paragraph markers...', end=' ')
Bible.insert_paragraph_markers(book)
print('finished.')
# get chunks for this book
print('Chunking the text...', end=' ')
Bible.chunk_book(metadata_obj.versification, book)
book.apply_chunks()
print('finished.')
# save the output
out_dir = out_template.format(domain, metadata_obj.slug)
# produces something like '01-GEN.usfm'
book_file_name = '{0}-{1}.usfm'.format(str(book.number).zfill(2), book.book_id)
print('Writing ' + book_file_name + '...', end=' ')
write_file('{0}/{1}'.format(out_dir, book_file_name), book.usfm)
print('finished.')
# look for an existing status.json file
print('Updating the status for {0}...'.format(metadata_obj.lang), end=' ')
status_file = '{0}/status.json'.format(out_dir)
if os.path.isfile(status_file):
status = BibleStatus(status_file)
else:
status = BibleStatus()
status.update_from_meta_data(metadata_obj)
# add this book to the list of "books_published"
status.add_book_published(book)
# update the "date_modified"
status.date_modified = today
print('finished.')
# save the status.json file
print('Writing status.json...', end=' ')
status_json = json.dumps(status, sort_keys=True, indent=2, cls=BibleEncoder)
write_file(status_file, status_json)
print('finished')
# let the API know it is there
print('Publishing to the API...')
with api_publish(out_dir) as api:
api.run()
print('Finished publishing to the API.')
# update the catalog
print()
print('Updating the catalogs...', end=' ')
update_catalog()
print('finished.')
print_notice('Check {0} and do a git push'.format(out_dir))
obs_chapter.frames.sort(key=lambda f: f['id'])
# add this chapter to the OBS object
chapters.append(obs_chapter)
print('finished.')
return chapters
if __name__ == '__main__':
print('')
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('-r', '--gitrepo', dest='gitrepo', default=False,
required=True, help='Git repository where the source can be found.')
parser.add_argument('-t', '--tag', dest='tag', default='master',
required=False, help='Branch or tag to use as the source. Default is master.')
parser.add_argument('-p', '--nopdf', dest='nopdf', action='store_true', help='Do not produce a PDF.')
args = parser.parse_args(sys.argv[1:])
try:
print_ok('STARTING: ', 'publishing OBS repository.')
main(args.gitrepo, args.tag, args.nopdf)
print_ok('ALL FINISHED: ', 'publishing OBS repository.')
print_notice('Don\'t forget to notify the interested parties.')
finally:
# delete temp files
if os.path.isdir(download_dir):
shutil.rmtree(download_dir, ignore_errors=True)
this_lang = next(l for l in langs if l['lc'] == args.lang)
if not this_lang:
print_error('Information for language "{0}" was not found.'.format(
args.lang))
sys.exit(1)
if 'github' not in args.gitrepo and 'git.door43.org' not in args.gitrepo:
print_warning(
'Currently only github and git.door43.org repositories are supported.'
)
sys.exit(0)
# prompt user to update status.txt
print_notice(
'Check status-{0}.json in the git repository and update the information if needed.'
.format(args.lang))
prompt('Press Enter to continue when ready...')
print()
# do the import
import_obs(this_lang, args.gitrepo, args.url, args.nopdf)
# update the catalog
print_ok('STARTING: ', 'updating the catalogs.')
update_catalog()
print_ok('FINISHED: ', 'updating the catalogs.')
# update OBS in-progress
print_ok('STARTING: ', 'updating OBS in progress.')
ObsInProgress.run()
def import_obs(lang_data, git_repo, door43_url, no_pdf):
global download_dir, root, api_dir
lang_code = lang_data['lc']
# pre-flight checklist
link_source = '/var/www/vhosts/api.unfoldingword.org/httpdocs/obs/jpg/1/en'
if not os.path.isdir(link_source):
print_error(
'Image source directory not found: {0}.'.format(link_source))
sys.exit(1)
if git_repo[-1:] != '/':
git_repo += '/'
if no_pdf:
tools_dir = None
else:
tools_dir = '/var/www/vhosts/door43.org/tools'
if not os.path.isdir(tools_dir):
tools_dir = os.path.expanduser('~/Projects/tools')
# prompt if tools not found
if not os.path.isdir(tools_dir):
tools_dir = None
print_notice(
'The tools directory was not found. The PDF cannot be generated.'
)
resp = prompt(
'Do you want to continue without generating a PDF? [Y|n]: ')
if resp != '' and resp != 'Y' and resp != 'y':
sys.exit(0)
if git_repo:
if git_repo[-1:] == '/':
git_repo = git_repo[:-1]
download_dir = '/tmp/{0}'.format(git_repo.rpartition('/')[2])
make_dir(download_dir)
# make sure OBS is initialized on Dokuwiki
test_dir = root.format(lang_code, '')
if not os.path.isdir(test_dir):
print_warning(
'It seems OBS has not been initialized on Door43.org for {0}'.
format(lang_code))
sys.exit(1)
elif door43_url:
print_error('URL not yet implemented.')
return
else:
print_error('Source not provided.')
return
# get the source files from the git repository
if 'github' in git_repo:
# https://github.com/unfoldingWord/obs-ru
# https://raw.githubusercontent.com/unfoldingWord/obs-ru/master/obs-ru.json
raw_url = git_repo.replace('github.com', 'raw.githubusercontent.com')
elif 'git.door43.org' in git_repo:
raw_url = join_url_parts(git_repo, 'raw')
else:
# this is to keep IntelliJ happy, is should have been caught in sub main
return
# download needed files from the repository
file_suffix = '-{0}.json'.format(lang_code.lower())
files_to_download = [
join_url_parts(raw_url, 'master/obs' + file_suffix),
join_url_parts(raw_url, 'master/status' + file_suffix)
]
for file_to_download in files_to_download:
downloaded_file = os.path.join(download_dir,
file_to_download.rpartition('/')[2])
try:
print('Downloading {0}...'.format(file_to_download), end=' ')
download_file(file_to_download, downloaded_file)
finally:
print('finished.')
# read the files from the git repository
file_suffix = '-{0}.json'.format(lang_code.lower())
obs_obj = None
status_obj = None
# front_matter_found = False
# back_matter_found = False
try:
print('Examining the files...', end=' ')
for root_path, dirs, files in os.walk(download_dir):
if not len(files):
continue
for git_file in files:
if git_file == 'obs' + file_suffix:
obs_obj = OBS(os.path.join(root_path, git_file))
elif git_file == 'status' + file_suffix:
status_obj = OBSStatus(os.path.join(root_path, git_file))
# elif 'front-matter' in git_file:
# front_matter_found = True
# elif 'back-matter' in git_file:
# back_matter_found = True
finally:
print('finished.')
# check data integrity
if not obs_obj.verify_all():
sys.exit(1)
if not status_obj:
print_error(
'The file "status{0}" was not found in the git repository.'.format(
file_suffix))
sys.exit(1)
# create Dokuwiki pages
print_ok('Begin: ', 'creating Dokuwiki pages.')
for chapter in obs_obj.chapters:
chapter_title = '====== {0} ======'.format(chapter['title'])
chapter_ref = '//{0}//'.format(chapter['ref'])
chapter_body = ''
chapter_num = chapter['number'].zfill(2)
for frame in chapter['frames']:
chapter_body += '{{{{{0}?direct&}}}}\n\n{1}\n\n'.format(
frame['img'], frame['text'])
file_name = root.format(lang_code, chapter_num + '.txt')
print(' Writing {0}'.format(file_name))
with codecs.open(file_name, 'w', 'utf-8-sig') as out_file:
out_file.write('{0}\n\n{1}{2}\n'.format(chapter_title,
chapter_body, chapter_ref))
print_ok('Finished: ', 'creating Dokuwiki pages.')
# Create image symlinks on api.unfoldingword.org
try:
print('Creating symlink to images directory...', end=' ')
link_name = '/var/www/vhosts/api.unfoldingword.org/httpdocs/obs/jpg/1/{0}'.format(
lang_code.lower())
if not os.path.isfile(link_name) and not os.path.isdir(
link_name) and not os.path.islink(link_name):
os.symlink(link_source, link_name)
finally:
print('finished.')
# Create PDF via ConTeXt
if not no_pdf and tools_dir and os.path.isdir(tools_dir):
try:
print_ok('Beginning: ', 'PDF generation.')
script_file = os.path.join(tools_dir, 'obs/book/pdf_export.sh')
out_dir = api_dir.format(lang_code)
make_dir(out_dir)
process = subprocess.Popen([
script_file, '-l', lang_code, '-c', status_obj.checking_level,
'-v', status_obj.version, '-o', out_dir
],
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# wait for the process to terminate
out, err = process.communicate()
exit_code = process.returncode
out = out.strip().decode('utf-8')
err = err.strip().decode('utf-8')
# the error message may be in stdout
if exit_code != 0:
if not err:
err = out
out = None
if err:
print_error(err, 2)
if out:
print(' ' + out)
print(' PDF subprocess finished with exit code {0}'.format(
exit_code))
finally:
print_ok('Finished:', 'generating PDF.')
link_source = '/var/www/vhosts/api.unfoldingword.org/httpdocs/obs/jpg/1/en'
if not os.path.isdir(link_source):
print_error('Image source directory not found: {0}.'.format(link_source))
sys.exit(1)
if no_pdf:
tools_dir = None
else:
tools_dir = '/var/www/vhosts/door43.org/tools'
if not os.path.isdir(tools_dir):
tools_dir = os.path.expanduser('~/Projects/tools')
# prompt if tools not found
if not os.path.isdir(tools_dir):
tools_dir = None
print_notice('The tools directory was not found. The PDF cannot be generated.')
resp = prompt('Do you want to continue without generating a PDF? [Y|n]: ')
if resp != '' and resp != 'Y' and resp != 'y':
sys.exit(0)
today = ''.join(str(datetime.date.today()).rsplit('-')[0:3])
print('Loading languages...', end=' ')
lang_dict = OBS.load_lang_strings()
print('finished.')
print('Loading the catalog...', end=' ')
uw_cat_path = os.path.join(unfoldingWord_dir, 'obs-catalog.json')
uw_catalog = load_json_object(uw_cat_path, [])
uw_cat_langs = [x['language'] for x in uw_catalog]
cat_path = os.path.join(export_dir, 'obs-catalog.json')
def rechunk_this_one(api_directory):
global id_re, s5_re
print_notice('Processing {}'.format(api_directory))
# read the status.json file
with codecs.open(os.path.join(api_directory, 'status.json'), 'r',
'utf-8-sig') as in_file:
status = json.loads(in_file.read())
# determine versification
if status['lang'] == 'ru':
versification = 'rsc'
elif status['lang'] == 'hi' or status['lang'] == 'sr-Latn' or status[
'lang'] == 'hu' or status['lang'] == 'ta':
versification = 'ufw-odx'
elif status['lang'] == 'bn':
versification = 'ufw-bn'
elif status['lang'] == 'ar':
versification = 'avd'
elif status['lang'] == 'kn':
versification = 'ufw-rev'
else:
versification = 'ufw'
versification_data = Bible.get_versification(
versification) # type: list<Book>
# remove all .sig files
for f in os.listdir(api_directory):
if f.endswith('.sig'):
os.remove(os.path.join(api_directory, f))
# rechunk files in this directory
usfm_files = glob(os.path.join(api_directory, '*.usfm'))
errors_found = False
for usfm_file in usfm_files:
if usfm_file.endswith('LICENSE.usfm'):
continue
# read the file
with codecs.open(usfm_file, 'r', 'utf-8') as in_file:
book_text = in_file.read()
# get the book id
book_search = id_re.search(book_text)
if not book_search:
print_error('Book id not found in {}'.format(usfm_file))
sys.exit(1)
book_id = book_search.group(1)
print('Beginning {}...'.format(book_id), end=' ')
# get book versification info
book = next((b for b in versification_data if b.book_id == book_id),
None)
if not book:
print_error(
'Book versification data was not found for "{}"'.format(
book_id))
sys.exit(1)
# remove \s5 lines
book_text = s5_re.sub('', book_text)
# get the usfm for the book
book.set_usfm(book_text)
# do basic checks
book.verify_chapters_and_verses(True)
if book.validation_errors:
errors_found = True
# get chunks for this book
Bible.chunk_book(versification, book)
book.apply_chunks()
# produces something like '01-GEN.usfm'
book_file_name = '{0}-{1}.usfm'.format(
str(book.number).zfill(2), book.book_id)
print('Writing ' + book_file_name + '...', end=' ')
write_file(usfm_file, book.usfm)
print('finished.')
if errors_found:
print_error(
'These USFM errors must be corrected before publishing can continue.'
)
sys.exit(1)
# rebuild source for tS
print()
print('Publishing to the API...')
with api_publish(api_directory) as api:
api.run()
print('Finished publishing to the API.')
# update the catalog
print()
print('Updating the catalogs...', end=' ')
update_catalog()
print('finished.')
print_error(
'Image source directory not found: {0}.'.format(link_source))
sys.exit(1)
if no_pdf:
tools_dir = None
else:
tools_dir = '/var/www/vhosts/door43.org/tools'
if not os.path.isdir(tools_dir):
tools_dir = os.path.expanduser('~/Projects/tools')
# prompt if tools not found
if not os.path.isdir(tools_dir):
tools_dir = None
print_notice(
'The tools directory was not found. The PDF cannot be generated.'
)
resp = prompt(
'Do you want to continue without generating a PDF? [Y|n]: ')
if resp != '' and resp != 'Y' and resp != 'y':
sys.exit(0)
today = ''.join(str(datetime.date.today()).rsplit('-')[0:3])
print('Loading languages...', end=' ')
lang_dict = OBS.load_lang_strings()
print('finished.')
print('Loading the catalog...', end=' ')
uw_cat_path = os.path.join(unfoldingWord_dir, 'obs-catalog.json')
uw_catalog = load_json_object(uw_cat_path, [])
def rechunk_this_one(api_directory):
global id_re, s5_re
print_notice('Processing {}'.format(api_directory))
# read the status.json file
with codecs.open(os.path.join(api_directory, 'status.json'), 'r', 'utf-8-sig') as in_file:
status = json.loads(in_file.read())
# determine versification
if status['lang'] == 'ru':
versification = 'rsc'
elif status['lang'] == 'hi' or status['lang'] == 'sr-Latn' or status['lang'] == 'hu' or status['lang'] == 'ta':
versification = 'ufw-odx'
elif status['lang'] == 'bn':
versification = 'ufw-bn'
elif status['lang'] == 'ar':
versification = 'avd'
elif status['lang'] == 'kn':
versification = 'ufw-rev'
else:
versification = 'ufw'
versification_data = Bible.get_versification(versification) # type: list<Book>
# remove all .sig files
for f in os.listdir(api_directory):
if f.endswith('.sig'):
os.remove(os.path.join(api_directory, f))
# rechunk files in this directory
usfm_files = glob(os.path.join(api_directory, '*.usfm'))
errors_found = False
for usfm_file in usfm_files:
if usfm_file.endswith('LICENSE.usfm'):
continue
# read the file
with codecs.open(usfm_file, 'r', 'utf-8') as in_file:
book_text = in_file.read()
# get the book id
book_search = id_re.search(book_text)
if not book_search:
print_error('Book id not found in {}'.format(usfm_file))
sys.exit(1)
book_id = book_search.group(1)
print('Beginning {}...'.format(book_id), end=' ')
# get book versification info
book = next((b for b in versification_data if b.book_id == book_id), None)
if not book:
print_error('Book versification data was not found for "{}"'.format(book_id))
sys.exit(1)
# remove \s5 lines
book_text = s5_re.sub('', book_text)
# get the usfm for the book
book.set_usfm(book_text)
# do basic checks
book.verify_chapters_and_verses(True)
if book.validation_errors:
errors_found = True
# get chunks for this book
Bible.chunk_book(versification, book)
book.apply_chunks()
# produces something like '01-GEN.usfm'
book_file_name = '{0}-{1}.usfm'.format(str(book.number).zfill(2), book.book_id)
print('Writing ' + book_file_name + '...', end=' ')
write_file(usfm_file, book.usfm)
print('finished.')
if errors_found:
print_error('These USFM errors must be corrected before publishing can continue.')
sys.exit(1)
# rebuild source for tS
print()
print('Publishing to the API...')
with api_publish(api_directory) as api:
api.run()
print('Finished publishing to the API.')
# update the catalog
print()
print('Updating the catalogs...', end=' ')
update_catalog()
print('finished.')
def import_obs(lang_data, git_repo, door43_url, no_pdf):
global download_dir, root, api_dir
lang_code = lang_data['lc']
# pre-flight checklist
link_source = '/var/www/vhosts/api.unfoldingword.org/httpdocs/obs/jpg/1/en'
if not os.path.isdir(link_source):
print_error('Image source directory not found: {0}.'.format(link_source))
sys.exit(1)
if git_repo[-1:] != '/':
git_repo += '/'
if no_pdf:
tools_dir = None
else:
tools_dir = '/var/www/vhosts/door43.org/tools'
if not os.path.isdir(tools_dir):
tools_dir = os.path.expanduser('~/Projects/tools')
# prompt if tools not found
if not os.path.isdir(tools_dir):
tools_dir = None
print_notice('The tools directory was not found. The PDF cannot be generated.')
resp = prompt('Do you want to continue without generating a PDF? [Y|n]: ')
if resp != '' and resp != 'Y' and resp != 'y':
sys.exit(0)
if git_repo:
if git_repo[-1:] == '/':
git_repo = git_repo[:-1]
download_dir = '/tmp/{0}'.format(git_repo.rpartition('/')[2])
make_dir(download_dir)
# make sure OBS is initialized on Dokuwiki
test_dir = root.format(lang_code, '')
if not os.path.isdir(test_dir):
print_warning('It seems OBS has not been initialized on Door43.org for {0}'.format(lang_code))
sys.exit(1)
elif door43_url:
print_error('URL not yet implemented.')
return
else:
print_error('Source not provided.')
return
# get the source files from the git repository
if 'github' in git_repo:
# https://github.com/unfoldingWord/obs-ru
# https://raw.githubusercontent.com/unfoldingWord/obs-ru/master/obs-ru.json
raw_url = git_repo.replace('github.com', 'raw.githubusercontent.com')
elif 'git.door43.org' in git_repo:
raw_url = join_url_parts(git_repo, 'raw')
else:
# this is to keep IntelliJ happy, is should have been caught in sub main
return
# download needed files from the repository
file_suffix = '-{0}.json'.format(lang_code.lower())
files_to_download = [
join_url_parts(raw_url, 'master/obs' + file_suffix),
join_url_parts(raw_url, 'master/status' + file_suffix)
]
for file_to_download in files_to_download:
downloaded_file = os.path.join(download_dir, file_to_download.rpartition('/')[2])
try:
print('Downloading {0}...'.format(file_to_download), end=' ')
download_file(file_to_download, downloaded_file)
finally:
print('finished.')
# read the files from the git repository
file_suffix = '-{0}.json'.format(lang_code.lower())
obs_obj = None
status_obj = None
# front_matter_found = False
# back_matter_found = False
try:
print('Examining the files...', end=' ')
for root_path, dirs, files in os.walk(download_dir):
if not len(files):
continue
for git_file in files:
if git_file == 'obs' + file_suffix:
obs_obj = OBS(os.path.join(root_path, git_file))
elif git_file == 'status' + file_suffix:
status_obj = OBSStatus(os.path.join(root_path, git_file))
# elif 'front-matter' in git_file:
# front_matter_found = True
# elif 'back-matter' in git_file:
# back_matter_found = True
finally:
print('finished.')
# check data integrity
if not obs_obj.verify_all():
sys.exit(1)
if not status_obj:
print_error('The file "status{0}" was not found in the git repository.'.format(file_suffix))
sys.exit(1)
# create Dokuwiki pages
print_ok('Begin: ', 'creating Dokuwiki pages.')
for chapter in obs_obj.chapters:
chapter_title = '====== {0} ======'.format(chapter['title'])
chapter_ref = '//{0}//'.format(chapter['ref'])
chapter_body = ''
chapter_num = chapter['number'].zfill(2)
for frame in chapter['frames']:
chapter_body += '{{{{{0}?direct&}}}}\n\n{1}\n\n'.format(frame['img'], frame['text'])
file_name = root.format(lang_code, chapter_num + '.txt')
print(' Writing {0}'.format(file_name))
with codecs.open(file_name, 'w', 'utf-8-sig') as out_file:
out_file.write('{0}\n\n{1}{2}\n'.format(chapter_title, chapter_body, chapter_ref))
print_ok('Finished: ', 'creating Dokuwiki pages.')
# Create image symlinks on api.unfoldingword.org
try:
print('Creating symlink to images directory...', end=' ')
link_name = '/var/www/vhosts/api.unfoldingword.org/httpdocs/obs/jpg/1/{0}'.format(lang_code.lower())
if not os.path.isfile(link_name) and not os.path.isdir(link_name) and not os.path.islink(link_name):
os.symlink(link_source, link_name)
finally:
print('finished.')
# Create PDF via ConTeXt
if not no_pdf and tools_dir and os.path.isdir(tools_dir):
try:
print_ok('Beginning: ', 'PDF generation.')
script_file = os.path.join(tools_dir, 'obs/book/pdf_export.sh')
out_dir = api_dir.format(lang_code)
make_dir(out_dir)
process = subprocess.Popen([script_file,
'-l', lang_code,
'-c', status_obj.checking_level,
'-v', status_obj.version,
'-o', out_dir],
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# wait for the process to terminate
out, err = process.communicate()
exit_code = process.returncode
out = out.strip().decode('utf-8')
err = err.strip().decode('utf-8')
# the error message may be in stdout
if exit_code != 0:
if not err:
err = out
out = None
if err:
print_error(err, 2)
if out:
print(' ' + out)
print(' PDF subprocess finished with exit code {0}'.format(exit_code))
finally:
print_ok('Finished:', 'generating PDF.')
