def main(source): global temp_dir today = "".join(str(datetime.date.today()).rsplit("-")[0:3]) dirs = [] if source: dirs.append(source) else: udb_dir = [os.path.join(UDBSource, x) for x in os.listdir(UDBSource)] dirs += udb_dir ulb_dir = [os.path.join(ULBSource, x) for x in os.listdir(ULBSource)] dirs += ulb_dir for d in dirs: ver, lang = d.rsplit("/", 1)[1].split("-", 1) temp_dir = "/tmp/{0}-{1}".format(ver, lang) if os.path.isdir(temp_dir): shutil.rmtree(temp_dir) transform.buildUSX(d, temp_dir, "", True) print "#### Chunking..." for f in os.listdir(temp_dir): # use utf-8-sig to remove the byte order mark usx = codecs.open(os.path.join(temp_dir, f), "r", encoding="utf-8-sig").readlines() slug = f.split(".")[0].lower() print " ({0})".format(slug.upper()) book = parse(usx) payload = {"chapters": book, "date_modified": today} write_json(os.path.join(api_v2, slug, lang, ver, "source.json"), payload) chunks = get_chunks(book) write_json(os.path.join(api_v2, slug, lang, ver, "chunks.json"), chunks)
def main(source): today = ''.join(str(datetime.date.today()).rsplit('-')[0:3]) dirs = [] if source: source_dirs = [os.path.join(source, x) for x in os.listdir(source)] dirs += source_dirs else: udbd = [os.path.join(UDBSource, x) for x in os.listdir(UDBSource)] dirs += udbd ulbd = [os.path.join(ULBSource, x) for x in os.listdir(ULBSource)] dirs += ulbd for d in dirs: ver, lang = d.rsplit('/', 1)[1].split('-') tmpdir = '/tmp/{0}-{1}'.format(ver, lang) if os.path.isdir(tmpdir): shutil.rmtree(tmpdir) transform.buildUSX(d, tmpdir, '', True) print "#### Chunking..." for f in os.listdir(tmpdir): usx = codecs.open(os.path.join(tmpdir, f), 'r', encoding='utf-8').readlines() slug = f.split('.')[0].lower() print ' ({0})'.format(slug.upper()) book = parse(usx) payload = {'chapters': book, 'date_modified': today} writeJSON(os.path.join(api_v2, slug, lang, ver, 'source.json'), payload) chunks = getChunks(book) writeJSON(os.path.join(api_v2, slug, lang, ver, 'chunks.json'), chunks)
def main(source): today = ''.join(str(datetime.date.today()).rsplit('-')[0:3]) dirs = [] if source: source_dirs = [os.path.join(source, x) for x in os.listdir(source)] dirs += source_dirs udbd = [os.path.join(UDBSource, x) for x in os.listdir(UDBSource)] dirs += udbd ulbd = [os.path.join(ULBSource, x) for x in os.listdir(ULBSource)] dirs += ulbd for d in dirs: ver, lang = d.rsplit('/', 1)[1].split('-') tmpdir = '/tmp/{0}-{1}'.format(ver, lang) if os.path.isdir(tmpdir): shutil.rmtree(tmpdir) transform.buildUSX(d, tmpdir, '', True) print "#### Chunking..." for f in os.listdir(tmpdir): usx = codecs.open(os.path.join(tmpdir, f), 'r', encoding='utf-8' ).readlines() slug = f.split('.')[0].lower() print ' ({0})'.format(slug.upper()) book = parse(usx) payload = { 'chapters': book, 'date_modified': today } writeJSON(os.path.join(api_v2, slug, lang, ver, 'source.json'), payload) chunks = getChunks(book) writeJSON(os.path.join(api_v2, slug, lang, ver, 'chunks.json'), chunks)
def main(source): global temp_dir today = ''.join(str(datetime.date.today()).rsplit('-')[0:3]) dirs = [] if source: dirs.append(source) else: udb_dir = [os.path.join(UDBSource, x) for x in os.listdir(UDBSource)] dirs += udb_dir ulb_dir = [os.path.join(ULBSource, x) for x in os.listdir(ULBSource)] dirs += ulb_dir for d in dirs: ver, lang = d.rsplit('/', 1)[1].split('-', 1) temp_dir = '/tmp/{0}-{1}'.format(ver, lang) if os.path.isdir(temp_dir): shutil.rmtree(temp_dir) transform.buildUSX(d, temp_dir, '', True) print "#### Chunking..." for f in os.listdir(temp_dir): # use utf-8-sig to remove the byte order mark usx = codecs.open(os.path.join(temp_dir, f), 'r', encoding='utf-8-sig' ).readlines() slug = f.split('.')[0].lower() print ' ({0})'.format(slug.upper()) book = parse(usx) payload = {'chapters': book, 'date_modified': today } write_json(os.path.join(api_v2, slug, lang, ver, 'source.json'), payload) chunks = get_chunks(book) write_json(os.path.join(api_v2, slug, lang, ver, 'chunks.json'), chunks)