def export_to_api(lang, status, today, cur_json):
global unfoldingWord_dir, lang_cat, github_org, pages
print('Getting Github credentials...', end=' ')
try:
github_org = None
if os.path.isfile('/root/.github_pass'):
# noinspection PyTypeChecker
pw = open('/root/.github_pass', 'r').read().strip()
g_user = githubLogin('dsm-git', pw)
github_org = getGithubOrg('unfoldingword', g_user)
else:
print('none found...', end=' ')
except GithubException as e:
print_error('Problem logging into Github: {0}'.format(e))
sys.exit(1)
print('finished.')
print('Loading the uw catalog...', end=' ')
uw_cat_path = os.path.join(unfoldingWord_dir, 'obs-catalog.json')
uw_catalog = load_json_object(uw_cat_path, [])
uw_cat_langs = [x['language'] for x in uw_catalog]
print('finished')
unfolding_word_lang_dir = os.path.join(unfoldingWord_dir, lang)
if 'checking_level' in status and 'publish_date' in status:
if status.checking_level in ['1', '2', '3']:
front_json = OBS.get_front_matter(pages, lang, today)
back_json = OBS.get_back_matter(pages, lang, today)
print('Exporting {0}...'.format(lang), end=' ')
export_unfolding_word(status, unfolding_word_lang_dir, cur_json,
lang, github_org, front_json, back_json)
if lang in uw_cat_langs:
uw_catalog.pop(uw_cat_langs.index(lang))
uw_cat_langs.pop(uw_cat_langs.index(lang))
uw_catalog.append(lang_cat)
uw_cat_json = json.dumps(uw_catalog,
sort_keys=True,
cls=OBSEncoder)
write_file(uw_cat_path, uw_cat_json)
# update uw_admin status page
ObsPublishedLangs.update_page(ObsPublishedLangs.cat_url,
ObsPublishedLangs.uw_stat_page)
print('finished.')
else:
print_error('The `checking_level` is invalid.')
sys.exit(1)
else:
print_error(
'The status is missing `checking_level` or `publish_date`.')
sys.exit(1)
def export_to_api(lang, status, today, cur_json):
global unfoldingWord_dir, lang_cat, github_org, pages
print('Getting Github credentials...', end=' ')
try:
github_org = None
if os.path.isfile('/root/.github_pass'):
# noinspection PyTypeChecker
pw = open('/root/.github_pass', 'r').read().strip()
g_user = githubLogin('dsm-git', pw)
github_org = getGithubOrg('unfoldingword', g_user)
else:
print('none found...', end=' ')
except GithubException as e:
print_error('Problem logging into Github: {0}'.format(e))
sys.exit(1)
print('finished.')
print('Loading the uw catalog...', end=' ')
uw_cat_path = os.path.join(unfoldingWord_dir, 'obs-catalog.json')
uw_catalog = load_json_object(uw_cat_path, [])
uw_cat_langs = [x['language'] for x in uw_catalog]
print('finished')
unfolding_word_lang_dir = os.path.join(unfoldingWord_dir, lang)
if 'checking_level' in status and 'publish_date' in status:
if status.checking_level in ['1', '2', '3']:
front_json = OBS.get_front_matter(pages, lang, today)
back_json = OBS.get_back_matter(pages, lang, today)
print('Exporting {0}...'.format(lang), end=' ')
export_unfolding_word(status, unfolding_word_lang_dir, cur_json,
lang, github_org, front_json, back_json)
if lang in uw_cat_langs:
uw_catalog.pop(uw_cat_langs.index(lang))
uw_cat_langs.pop(uw_cat_langs.index(lang))
uw_catalog.append(lang_cat)
uw_cat_json = json.dumps(uw_catalog, sort_keys=True, cls=OBSEncoder)
write_file(uw_cat_path, uw_cat_json)
# update uw_admin status page
ObsPublishedLangs.update_page(ObsPublishedLangs.cat_url, ObsPublishedLangs.uw_stat_page)
print('finished.')
else:
print_error('The `checking_level` is invalid.')
sys.exit(1)
else:
print_error('The status is missing `checking_level` or `publish_date`.')
sys.exit(1)
def clean_obs_json_file(json_file):
obs_obj = OBS(json_file)
# check data integrity
if not obs_obj.verify_all():
sys.exit(1)
# write the cleaned file
cleaned_name = json_file.replace('.json', '')
cleaned_name += '.cleaned.json'
with codecs.open(cleaned_name, 'w', 'utf-8-sig') as out_file:
json.dump(obs_obj.__dict__, out_file, sort_keys=True, indent=2)
print('Finished. {0}'.format(cleaned_name))
def clean_obs_json_file(json_file):
obs_obj = OBS(json_file)
# check data integrity
if not obs_obj.verify_all():
sys.exit(1)
# write the cleaned file
cleaned_name = json_file.replace('.json', '')
cleaned_name += '.cleaned.json'
with codecs.open(cleaned_name, 'w', 'utf-8-sig') as out_file:
json.dump(obs_obj.__dict__, out_file, sort_keys=True, indent=2)
print('Finished. {0}'.format(cleaned_name))
def export_unfolding_word(status, git_dir, json_data, lang_code,
github_organization, front_matter, back_matter):
"""
Exports JSON data for each language into its own Github repo.
"""
global github_org
write_file(os.path.join(git_dir, 'obs-{0}.json'.format(lang_code)),
json_data)
write_file(
os.path.join(git_dir, 'obs-{0}-front-matter.json'.format(lang_code)),
front_matter)
write_file(
os.path.join(git_dir, 'obs-{0}-back-matter.json'.format(lang_code)),
back_matter)
status_str = json.dumps(status, sort_keys=True, cls=OBSEncoder)
write_file(os.path.join(git_dir, 'status-{0}.json'.format(lang_code)),
status_str)
write_file(os.path.join(git_dir, 'README.md'), OBS.get_readme_text())
if not github_org:
return
gitCreate(git_dir)
name = 'obs-{0}'.format(lang_code)
desc = 'Open Bible Stories for {0}'.format(lang_code)
url = 'http://unfoldingword.org/{0}/'.format(lang_code)
githubCreate(git_dir, name, desc, url, github_organization)
commit_msg = status_str
gitCommit(git_dir, commit_msg)
gitPush(git_dir)
def export_unfolding_word(status_dict, git_dir, json_data, lang_code, github_organization, front_matter, back_matter):
"""
Exports JSON data for each language into its own Github repo.
"""
write_page(os.path.join(git_dir, 'obs-{0}.json'.format(lang_code)), json_data)
write_page(os.path.join(git_dir, 'obs-{0}-front-matter.json'.format(lang_code)), front_matter)
write_page(os.path.join(git_dir, 'obs-{0}-back-matter.json'.format(lang_code)), back_matter)
write_page(os.path.join(git_dir, 'status-{0}.json'.format(lang_code)), clean_status(status_dict))
write_page(os.path.join(git_dir, 'README.md'), OBS.get_readme_text())
if not github_org:
return
gitCreate(git_dir)
name = 'obs-{0}'.format(lang_code)
desc = 'Open Bible Stories for {0}'.format(lang_code)
url = 'http://unfoldingword.org/{0}/'.format(lang_code)
githubCreate(git_dir, name, desc, url, github_organization)
commit_msg = str(status_dict)
gitCommit(git_dir, commit_msg)
gitPush(git_dir)
def main(git_repo, tag, no_pdf):
global download_dir
# clean up the git repo url
if git_repo[-4:] == '.git':
git_repo = git_repo[:-4]
if git_repo[-1:] == '/':
git_repo = git_repo[:-1]
# initialize some variables
today = ''.join(str(datetime.date.today()).rsplit('-')[0:3]) # str(datetime.date.today())
download_dir = '/tmp/{0}'.format(git_repo.rpartition('/')[2])
make_dir(download_dir)
downloaded_file = '{0}/{1}.zip'.format(download_dir, git_repo.rpartition('/')[2])
file_to_download = join_url_parts(git_repo, 'archive/' + tag + '.zip')
manifest = None
status = None # type: OBSStatus
content_dir = None
# download the repository
try:
print('Downloading {0}...'.format(file_to_download), end=' ')
if not os.path.isfile(downloaded_file):
download_file(file_to_download, downloaded_file)
finally:
print('finished.')
try:
print('Unzipping...'.format(downloaded_file), end=' ')
unzip(downloaded_file, download_dir)
finally:
print('finished.')
# examine the repository
for root, dirs, files in os.walk(download_dir):
if 'manifest.json' in files:
# read the manifest
try:
print('Reading the manifest...', end=' ')
content_dir = root
manifest = load_json_object(os.path.join(root, 'manifest.json'))
finally:
print('finished.')
if 'status.json' in files:
# read the meta data
try:
print('Reading the status...', end=' ')
content_dir = root
status = OBSStatus(os.path.join(root, 'status.json'))
finally:
print('finished.')
# if we have everything, exit the loop
if content_dir and manifest and status:
break
# check for valid repository structure
if not manifest:
print_error('Did not find manifest.json in {}'.format(git_repo))
sys.exit(1)
if not status:
print_error('Did not find status.json in {}'.format(git_repo))
sys.exit(1)
print('Initializing OBS object...', end=' ')
lang = manifest['target_language']['id']
obs_obj = OBS()
obs_obj.date_modified = today
obs_obj.direction = manifest['target_language']['direction']
obs_obj.language = lang
print('finished')
obs_obj.chapters = load_obs_chapters(content_dir)
obs_obj.chapters.sort(key=lambda c: c['number'])
if not obs_obj.verify_all():
print_error('Quality check did not pass.')
sys.exit(1)
print('Loading languages...', end=' ')
lang_dict = OBS.load_lang_strings()
print('finished.')
print('Loading the catalog...', end=' ')
export_dir = '/var/www/vhosts/door43.org/httpdocs/exports'
# uw_cat_path = os.path.join(unfoldingWord_dir, 'obs-catalog.json')
# uw_catalog = load_json_object(uw_cat_path, [])
# uw_cat_langs = [x['language'] for x in uw_catalog]
cat_path = os.path.join(export_dir, 'obs-catalog.json')
catalog = load_json_object(cat_path, [])
print('finished')
print('Getting already published languages...', end=' ')
json_lang_file_path = os.path.join(export_dir, lang, 'obs', 'obs-{0}.json'.format(lang))
# prev_json_lang = load_json_object(json_lang_file_path, {})
if lang not in lang_dict:
print("Configuration for language {0} missing.".format(lang))
sys.exit(1)
print('finished.')
updated = update_language_catalog(lang, obs_obj.direction, status, today, lang_dict, catalog)
print('Writing the OBS file to the exports directory...', end=' ')
cur_json = json.dumps(obs_obj, sort_keys=True, cls=OBSEncoder)
if updated:
([x for x in catalog if x['language'] == lang][0]['date_modified']) = today
write_file(json_lang_file_path.replace('.txt', '.json'), cur_json)
print('finished.')
export_to_api(lang, status, today, cur_json)
cat_json = json.dumps(catalog, sort_keys=True, cls=OBSEncoder)
write_file(cat_path, cat_json)
# update the catalog
print_ok('STARTING: ', 'updating the catalogs.')
update_catalog()
print_ok('FINISHED: ', 'updating the catalogs.')
if no_pdf:
return
create_pdf(lang, status.checking_level, status.version)
def main(git_repo, tag, no_pdf):
global download_dir
# clean up the git repo url
if git_repo[-4:] == '.git':
git_repo = git_repo[:-4]
if git_repo[-1:] == '/':
git_repo = git_repo[:-1]
# initialize some variables
today = ''.join(str(datetime.date.today()).rsplit(
str('-'))[0:3]) # str(datetime.date.today())
download_dir = '/tmp/{0}'.format(git_repo.rpartition('/')[2])
make_dir(download_dir)
downloaded_file = '{0}/{1}.zip'.format(download_dir,
git_repo.rpartition('/')[2])
file_to_download = join_url_parts(git_repo, 'archive/{0}.zip'.format(tag))
manifest = None
status = None # type: OBSStatus
content_dir = None
# download the repository
try:
print('Downloading {0}...'.format(file_to_download), end=' ')
if not os.path.isfile(downloaded_file):
download_file(file_to_download, downloaded_file)
finally:
print('finished.')
try:
print('Unzipping...'.format(downloaded_file), end=' ')
unzip(downloaded_file, download_dir)
finally:
print('finished.')
# examine the repository
for root, dirs, files in os.walk(download_dir):
if 'manifest.json' in files:
# read the manifest
try:
print('Reading the manifest...', end=' ')
content_dir = root
manifest = load_json_object(os.path.join(
root, 'manifest.json'))
status = OBSStatus.from_manifest(manifest)
finally:
print('finished.')
if 'content' in dirs:
content_dir = os.path.join(root, 'content')
# if we have everything, exit the loop
if content_dir and manifest and status:
break
# check for valid repository structure
if not manifest:
print_error('Did not find manifest.json in {}'.format(git_repo))
sys.exit(1)
print('Initializing OBS object...', end=' ')
lang = manifest['language']['slug']
obs_obj = OBS()
obs_obj.date_modified = today
obs_obj.direction = manifest['language']['dir']
obs_obj.language = lang
print('finished')
obs_obj.chapters = load_obs_chapters(content_dir)
obs_obj.chapters.sort(key=lambda c: int(c['number']))
if not obs_obj.verify_all():
print_error('Quality check did not pass.')
sys.exit(1)
print('Loading languages...', end=' ')
lang_dict = OBS.load_lang_strings()
print('finished.')
print('Loading the catalog...', end=' ')
export_dir = '/var/www/vhosts/door43.org/httpdocs/exports'
cat_path = os.path.join(export_dir, 'obs-catalog.json')
catalog = load_json_object(cat_path, [])
print('finished')
print('Getting already published languages...', end=' ')
json_lang_file_path = os.path.join(export_dir, lang, 'obs',
'obs-{0}.json'.format(lang))
if lang not in lang_dict:
print("Configuration for language {0} missing.".format(lang))
sys.exit(1)
print('finished.')
updated = update_language_catalog(lang, obs_obj.direction, status, today,
lang_dict, catalog)
print('Writing the OBS file to the exports directory...', end=' ')
cur_json = json.dumps(obs_obj, sort_keys=True, cls=OBSEncoder)
if updated:
([x for x in catalog
if x['language'] == lang][0]['date_modified']) = today
# noinspection PyTypeChecker
write_file(json_lang_file_path.replace('.txt', '.json'), cur_json)
print('finished.')
export_to_api(lang, status, today, cur_json)
cat_json = json.dumps(catalog, sort_keys=True, cls=OBSEncoder)
write_file(cat_path, cat_json)
# update the catalog
print_ok('STARTING: ', 'updating the catalogs.')
update_catalog()
print_ok('FINISHED: ', 'updating the catalogs.')
if no_pdf:
return
create_pdf(lang, status.checking_level, status.version)
def import_obs(lang_data, git_repo, door43_url, no_pdf):
global download_dir, root, api_dir
lang_code = lang_data['lc']
# pre-flight checklist
link_source = '/var/www/vhosts/api.unfoldingword.org/httpdocs/obs/jpg/1/en'
if not os.path.isdir(link_source):
print_error(
'Image source directory not found: {0}.'.format(link_source))
sys.exit(1)
if git_repo[-1:] != '/':
git_repo += '/'
if no_pdf:
tools_dir = None
else:
tools_dir = '/var/www/vhosts/door43.org/tools'
if not os.path.isdir(tools_dir):
tools_dir = os.path.expanduser('~/Projects/tools')
# prompt if tools not found
if not os.path.isdir(tools_dir):
tools_dir = None
print_notice(
'The tools directory was not found. The PDF cannot be generated.'
)
resp = prompt(
'Do you want to continue without generating a PDF? [Y|n]: ')
if resp != '' and resp != 'Y' and resp != 'y':
sys.exit(0)
if git_repo:
if git_repo[-1:] == '/':
git_repo = git_repo[:-1]
download_dir = '/tmp/{0}'.format(git_repo.rpartition('/')[2])
make_dir(download_dir)
# make sure OBS is initialized on Dokuwiki
test_dir = root.format(lang_code, '')
if not os.path.isdir(test_dir):
print_warning(
'It seems OBS has not been initialized on Door43.org for {0}'.
format(lang_code))
sys.exit(1)
elif door43_url:
print_error('URL not yet implemented.')
return
else:
print_error('Source not provided.')
return
# get the source files from the git repository
if 'github' in git_repo:
# https://github.com/unfoldingWord/obs-ru
# https://raw.githubusercontent.com/unfoldingWord/obs-ru/master/obs-ru.json
raw_url = git_repo.replace('github.com', 'raw.githubusercontent.com')
elif 'git.door43.org' in git_repo:
raw_url = join_url_parts(git_repo, 'raw')
else:
# this is to keep IntelliJ happy, is should have been caught in sub main
return
# download needed files from the repository
file_suffix = '-{0}.json'.format(lang_code.lower())
files_to_download = [
join_url_parts(raw_url, 'master/obs' + file_suffix),
join_url_parts(raw_url, 'master/status' + file_suffix)
]
for file_to_download in files_to_download:
downloaded_file = os.path.join(download_dir,
file_to_download.rpartition('/')[2])
try:
print('Downloading {0}...'.format(file_to_download), end=' ')
download_file(file_to_download, downloaded_file)
finally:
print('finished.')
# read the files from the git repository
file_suffix = '-{0}.json'.format(lang_code.lower())
obs_obj = None
status_obj = None
# front_matter_found = False
# back_matter_found = False
try:
print('Examining the files...', end=' ')
for root_path, dirs, files in os.walk(download_dir):
if not len(files):
continue
for git_file in files:
if git_file == 'obs' + file_suffix:
obs_obj = OBS(os.path.join(root_path, git_file))
elif git_file == 'status' + file_suffix:
status_obj = OBSStatus(os.path.join(root_path, git_file))
# elif 'front-matter' in git_file:
# front_matter_found = True
# elif 'back-matter' in git_file:
# back_matter_found = True
finally:
print('finished.')
# check data integrity
if not obs_obj.verify_all():
sys.exit(1)
if not status_obj:
print_error(
'The file "status{0}" was not found in the git repository.'.format(
file_suffix))
sys.exit(1)
# create Dokuwiki pages
print_ok('Begin: ', 'creating Dokuwiki pages.')
for chapter in obs_obj.chapters:
chapter_title = '====== {0} ======'.format(chapter['title'])
chapter_ref = '//{0}//'.format(chapter['ref'])
chapter_body = ''
chapter_num = chapter['number'].zfill(2)
for frame in chapter['frames']:
chapter_body += '{{{{{0}?direct&}}}}\n\n{1}\n\n'.format(
frame['img'], frame['text'])
file_name = root.format(lang_code, chapter_num + '.txt')
print(' Writing {0}'.format(file_name))
with codecs.open(file_name, 'w', 'utf-8-sig') as out_file:
out_file.write('{0}\n\n{1}{2}\n'.format(chapter_title,
chapter_body, chapter_ref))
print_ok('Finished: ', 'creating Dokuwiki pages.')
# Create image symlinks on api.unfoldingword.org
try:
print('Creating symlink to images directory...', end=' ')
link_name = '/var/www/vhosts/api.unfoldingword.org/httpdocs/obs/jpg/1/{0}'.format(
lang_code.lower())
if not os.path.isfile(link_name) and not os.path.isdir(
link_name) and not os.path.islink(link_name):
os.symlink(link_source, link_name)
finally:
print('finished.')
# Create PDF via ConTeXt
if not no_pdf and tools_dir and os.path.isdir(tools_dir):
try:
print_ok('Beginning: ', 'PDF generation.')
script_file = os.path.join(tools_dir, 'obs/book/pdf_export.sh')
out_dir = api_dir.format(lang_code)
make_dir(out_dir)
process = subprocess.Popen([
script_file, '-l', lang_code, '-c', status_obj.checking_level,
'-v', status_obj.version, '-o', out_dir
],
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# wait for the process to terminate
out, err = process.communicate()
exit_code = process.returncode
out = out.strip().decode('utf-8')
err = err.strip().decode('utf-8')
# the error message may be in stdout
if exit_code != 0:
if not err:
err = out
out = None
if err:
print_error(err, 2)
if out:
print(' ' + out)
print(' PDF subprocess finished with exit code {0}'.format(
exit_code))
finally:
print_ok('Finished:', 'generating PDF.')
tools_dir = '/var/www/vhosts/door43.org/tools'
if not os.path.isdir(tools_dir):
tools_dir = os.path.expanduser('~/Projects/tools')
# prompt if tools not found
if not os.path.isdir(tools_dir):
tools_dir = None
print_notice('The tools directory was not found. The PDF cannot be generated.')
resp = prompt('Do you want to continue without generating a PDF? [Y|n]: ')
if resp != '' and resp != 'Y' and resp != 'y':
sys.exit(0)
today = ''.join(str(datetime.date.today()).rsplit('-')[0:3])
print('Loading languages...', end=' ')
lang_dict = OBS.load_lang_strings()
print('finished.')
print('Loading the catalog...', end=' ')
uw_cat_path = os.path.join(unfoldingWord_dir, 'obs-catalog.json')
uw_catalog = load_json_object(uw_cat_path, [])
uw_cat_langs = [x['language'] for x in uw_catalog]
cat_path = os.path.join(export_dir, 'obs-catalog.json')
catalog = load_json_object(cat_path, [])
print('finished')
if 'obs' not in os.listdir(os.path.join(pages, lang)):
print('OBS not configured in Door43 for {0}'.format(lang))
sys.exit(1)
print('Getting metadata...', end=' ')
# prompt if tools not found
if not os.path.isdir(tools_dir):
tools_dir = None
print_notice(
'The tools directory was not found. The PDF cannot be generated.'
)
resp = prompt(
'Do you want to continue without generating a PDF? [Y|n]: ')
if resp != '' and resp != 'Y' and resp != 'y':
sys.exit(0)
today = ''.join(str(datetime.date.today()).rsplit('-')[0:3])
print('Loading languages...', end=' ')
lang_dict = OBS.load_lang_strings()
print('finished.')
print('Loading the catalog...', end=' ')
uw_cat_path = os.path.join(unfoldingWord_dir, 'obs-catalog.json')
uw_catalog = load_json_object(uw_cat_path, [])
uw_cat_langs = [x['language'] for x in uw_catalog]
cat_path = os.path.join(export_dir, 'obs-catalog.json')
catalog = load_json_object(cat_path, [])
print('finished')
if 'obs' not in os.listdir(os.path.join(pages, lang)):
print('OBS not configured in Door43 for {0}'.format(lang))
sys.exit(1)
print('Getting metadata...', end=' ')
def import_obs(lang_data, git_repo, door43_url, no_pdf):
global download_dir, root, api_dir
lang_code = lang_data['lc']
# pre-flight checklist
link_source = '/var/www/vhosts/api.unfoldingword.org/httpdocs/obs/jpg/1/en'
if not os.path.isdir(link_source):
print_error('Image source directory not found: {0}.'.format(link_source))
sys.exit(1)
if git_repo[-1:] != '/':
git_repo += '/'
if no_pdf:
tools_dir = None
else:
tools_dir = '/var/www/vhosts/door43.org/tools'
if not os.path.isdir(tools_dir):
tools_dir = os.path.expanduser('~/Projects/tools')
# prompt if tools not found
if not os.path.isdir(tools_dir):
tools_dir = None
print_notice('The tools directory was not found. The PDF cannot be generated.')
resp = prompt('Do you want to continue without generating a PDF? [Y|n]: ')
if resp != '' and resp != 'Y' and resp != 'y':
sys.exit(0)
if git_repo:
if git_repo[-1:] == '/':
git_repo = git_repo[:-1]
download_dir = '/tmp/{0}'.format(git_repo.rpartition('/')[2])
make_dir(download_dir)
# make sure OBS is initialized on Dokuwiki
test_dir = root.format(lang_code, '')
if not os.path.isdir(test_dir):
print_warning('It seems OBS has not been initialized on Door43.org for {0}'.format(lang_code))
sys.exit(1)
elif door43_url:
print_error('URL not yet implemented.')
return
else:
print_error('Source not provided.')
return
# get the source files from the git repository
if 'github' in git_repo:
# https://github.com/unfoldingWord/obs-ru
# https://raw.githubusercontent.com/unfoldingWord/obs-ru/master/obs-ru.json
raw_url = git_repo.replace('github.com', 'raw.githubusercontent.com')
elif 'git.door43.org' in git_repo:
raw_url = join_url_parts(git_repo, 'raw')
else:
# this is to keep IntelliJ happy, is should have been caught in sub main
return
# download needed files from the repository
file_suffix = '-{0}.json'.format(lang_code.lower())
files_to_download = [
join_url_parts(raw_url, 'master/obs' + file_suffix),
join_url_parts(raw_url, 'master/status' + file_suffix)
]
for file_to_download in files_to_download:
downloaded_file = os.path.join(download_dir, file_to_download.rpartition('/')[2])
try:
print('Downloading {0}...'.format(file_to_download), end=' ')
download_file(file_to_download, downloaded_file)
finally:
print('finished.')
# read the files from the git repository
file_suffix = '-{0}.json'.format(lang_code.lower())
obs_obj = None
status_obj = None
# front_matter_found = False
# back_matter_found = False
try:
print('Examining the files...', end=' ')
for root_path, dirs, files in os.walk(download_dir):
if not len(files):
continue
for git_file in files:
if git_file == 'obs' + file_suffix:
obs_obj = OBS(os.path.join(root_path, git_file))
elif git_file == 'status' + file_suffix:
status_obj = OBSStatus(os.path.join(root_path, git_file))
# elif 'front-matter' in git_file:
# front_matter_found = True
# elif 'back-matter' in git_file:
# back_matter_found = True
finally:
print('finished.')
# check data integrity
if not obs_obj.verify_all():
sys.exit(1)
if not status_obj:
print_error('The file "status{0}" was not found in the git repository.'.format(file_suffix))
sys.exit(1)
# create Dokuwiki pages
print_ok('Begin: ', 'creating Dokuwiki pages.')
for chapter in obs_obj.chapters:
chapter_title = '====== {0} ======'.format(chapter['title'])
chapter_ref = '//{0}//'.format(chapter['ref'])
chapter_body = ''
chapter_num = chapter['number'].zfill(2)
for frame in chapter['frames']:
chapter_body += '{{{{{0}?direct&}}}}\n\n{1}\n\n'.format(frame['img'], frame['text'])
file_name = root.format(lang_code, chapter_num + '.txt')
print(' Writing {0}'.format(file_name))
with codecs.open(file_name, 'w', 'utf-8-sig') as out_file:
out_file.write('{0}\n\n{1}{2}\n'.format(chapter_title, chapter_body, chapter_ref))
print_ok('Finished: ', 'creating Dokuwiki pages.')
# Create image symlinks on api.unfoldingword.org
try:
print('Creating symlink to images directory...', end=' ')
link_name = '/var/www/vhosts/api.unfoldingword.org/httpdocs/obs/jpg/1/{0}'.format(lang_code.lower())
if not os.path.isfile(link_name) and not os.path.isdir(link_name) and not os.path.islink(link_name):
os.symlink(link_source, link_name)
finally:
print('finished.')
# Create PDF via ConTeXt
if not no_pdf and tools_dir and os.path.isdir(tools_dir):
try:
print_ok('Beginning: ', 'PDF generation.')
script_file = os.path.join(tools_dir, 'obs/book/pdf_export.sh')
out_dir = api_dir.format(lang_code)
make_dir(out_dir)
process = subprocess.Popen([script_file,
'-l', lang_code,
'-c', status_obj.checking_level,
'-v', status_obj.version,
'-o', out_dir],
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# wait for the process to terminate
out, err = process.communicate()
exit_code = process.returncode
out = out.strip().decode('utf-8')
err = err.strip().decode('utf-8')
# the error message may be in stdout
if exit_code != 0:
if not err:
err = out
out = None
if err:
print_error(err, 2)
if out:
print(' ' + out)
print(' PDF subprocess finished with exit code {0}'.format(exit_code))
finally:
print_ok('Finished:', 'generating PDF.')