def main(source):
global temp_dir
today = "".join(str(datetime.date.today()).rsplit("-")[0:3])
dirs = []
if source:
dirs.append(source)
else:
udb_dir = [os.path.join(UDBSource, x) for x in os.listdir(UDBSource)]
dirs += udb_dir
ulb_dir = [os.path.join(ULBSource, x) for x in os.listdir(ULBSource)]
dirs += ulb_dir
for d in dirs:
ver, lang = d.rsplit("/", 1)[1].split("-", 1)
temp_dir = "/tmp/{0}-{1}".format(ver, lang)
if os.path.isdir(temp_dir):
shutil.rmtree(temp_dir)
transform.buildUSX(d, temp_dir, "", True)
print "#### Chunking..."
for f in os.listdir(temp_dir):
# use utf-8-sig to remove the byte order mark
usx = codecs.open(os.path.join(temp_dir, f), "r", encoding="utf-8-sig").readlines()
slug = f.split(".")[0].lower()
print " ({0})".format(slug.upper())
book = parse(usx)
payload = {"chapters": book, "date_modified": today}
write_json(os.path.join(api_v2, slug, lang, ver, "source.json"), payload)
chunks = get_chunks(book)
write_json(os.path.join(api_v2, slug, lang, ver, "chunks.json"), chunks)
def main(source):
today = ''.join(str(datetime.date.today()).rsplit('-')[0:3])
dirs = []
if source:
source_dirs = [os.path.join(source, x) for x in os.listdir(source)]
dirs += source_dirs
else:
udbd = [os.path.join(UDBSource, x) for x in os.listdir(UDBSource)]
dirs += udbd
ulbd = [os.path.join(ULBSource, x) for x in os.listdir(ULBSource)]
dirs += ulbd
for d in dirs:
ver, lang = d.rsplit('/', 1)[1].split('-')
tmpdir = '/tmp/{0}-{1}'.format(ver, lang)
if os.path.isdir(tmpdir):
shutil.rmtree(tmpdir)
transform.buildUSX(d, tmpdir, '', True)
print "#### Chunking..."
for f in os.listdir(tmpdir):
usx = codecs.open(os.path.join(tmpdir, f), 'r',
encoding='utf-8').readlines()
slug = f.split('.')[0].lower()
print ' ({0})'.format(slug.upper())
book = parse(usx)
payload = {'chapters': book, 'date_modified': today}
writeJSON(os.path.join(api_v2, slug, lang, ver, 'source.json'),
payload)
chunks = getChunks(book)
writeJSON(os.path.join(api_v2, slug, lang, ver, 'chunks.json'),
chunks)
def main(source):
today = ''.join(str(datetime.date.today()).rsplit('-')[0:3])
dirs = []
if source:
source_dirs = [os.path.join(source, x) for x in os.listdir(source)]
dirs += source_dirs
udbd = [os.path.join(UDBSource, x) for x in os.listdir(UDBSource)]
dirs += udbd
ulbd = [os.path.join(ULBSource, x) for x in os.listdir(ULBSource)]
dirs += ulbd
for d in dirs:
ver, lang = d.rsplit('/', 1)[1].split('-')
tmpdir = '/tmp/{0}-{1}'.format(ver, lang)
if os.path.isdir(tmpdir):
shutil.rmtree(tmpdir)
transform.buildUSX(d, tmpdir, '', True)
print "#### Chunking..."
for f in os.listdir(tmpdir):
usx = codecs.open(os.path.join(tmpdir, f), 'r', encoding='utf-8'
).readlines()
slug = f.split('.')[0].lower()
print ' ({0})'.format(slug.upper())
book = parse(usx)
payload = { 'chapters': book,
'date_modified': today
}
writeJSON(os.path.join(api_v2, slug, lang, ver, 'source.json'),
payload)
chunks = getChunks(book)
writeJSON(os.path.join(api_v2, slug, lang, ver, 'chunks.json'),
chunks)
def main(source):
global temp_dir
today = ''.join(str(datetime.date.today()).rsplit('-')[0:3])
dirs = []
if source:
dirs.append(source)
else:
udb_dir = [os.path.join(UDBSource, x) for x in os.listdir(UDBSource)]
dirs += udb_dir
ulb_dir = [os.path.join(ULBSource, x) for x in os.listdir(ULBSource)]
dirs += ulb_dir
for d in dirs:
ver, lang = d.rsplit('/', 1)[1].split('-', 1)
temp_dir = '/tmp/{0}-{1}'.format(ver, lang)
if os.path.isdir(temp_dir):
shutil.rmtree(temp_dir)
transform.buildUSX(d, temp_dir, '', True)
print "#### Chunking..."
for f in os.listdir(temp_dir):
# use utf-8-sig to remove the byte order mark
usx = codecs.open(os.path.join(temp_dir, f), 'r', encoding='utf-8-sig'
).readlines()
slug = f.split('.')[0].lower()
print ' ({0})'.format(slug.upper())
book = parse(usx)
payload = {'chapters': book,
'date_modified': today
}
write_json(os.path.join(api_v2, slug, lang, ver, 'source.json'),
payload)
chunks = get_chunks(book)
write_json(os.path.join(api_v2, slug, lang, ver, 'chunks.json'),
chunks)
