# Create datetime using updated_on:
modified_since = convert_date_string(updated_on)
# Map feed entries to list of import objects:
print(
f'Importing all entries that have been updated since {modified_since}.'
)
modified_entries = filter_modified_since(d.entries, modified_since)
print(f'{len(modified_entries)} import objects created.')
if not dry_run:
create_batch(modified_entries)
print(
f'{len(modified_entries)} entries added to the batch import job.')
else:
for record in modified_entries:
print(json.dumps(record))
# Store timestamp for header
if not dry_run:
with open(LAST_UPDATED_TIME, 'w+') as f:
f.write(last_modified)
print(f'Last updated timestamp written to: {LAST_UPDATED_TIME}')
if __name__ == '__main__':
print("Start: Standard Ebooks import job")
FnToCLI(import_job).run()
print("End: Standard Ebooks import job")
offset = 0
while True:
if done:
# Done! Wait for any previous workers that are still going
await asyncio.gather(*active_workers)
break
elif len(active_workers) >= instances:
# Too many running; wait for one to finish
finished, pending = await asyncio.wait(
active_workers,
return_when=asyncio.FIRST_COMPLETED)
active_workers = pending
done = any(task.result() < chunk_size for task in finished)
else:
# Can start another worker
task = asyncio.create_task(index_subjects(
subject_type,
offset=offset,
limit=chunk_size,
solr_base_url=solr_base_url,
skip_id_check=skip_id_check,
))
active_workers.add(task)
offset += chunk_size
if __name__ == '__main__':
cli = FnToCLI(index_all_subjects)
print(cli.parse_args())
cli.run()
if isinstance(dest_ol, OpenLibrary):
section = "[%s]" % web.lstrips(dest, "http://").strip("/")
if section in read_lines(os.path.expanduser("~/.olrc")):
dest_ol.autologin()
else:
dest_ol.login("admin", "admin123")
for list_key in (lists or []):
copy_list(src_ol, dest_ol, list_key, comment=comment)
if search:
assert isinstance(src_ol, OpenLibrary), "Search only works with OL src"
keys += [
doc['key'] for doc in src_ol.search(
search, limit=search_limit, fields=['key'])['docs']
]
keys = list(expand(src_ol, ('/' + k.lstrip('/') for k in keys)))
copy(src_ol,
dest_ol,
keys,
comment=comment,
recursive=recursive,
editions=editions)
if __name__ == '__main__':
FnToCLI(main).run()
while True:
records = logfile.read_records()
keys = parse_log(records, load_ia_scans)
count = await update_keys(keys)
if logfile.tell() != offset:
offset = logfile.tell()
logger.info("saving offset %s", offset)
with open(state_file, "w") as f:
f.write(offset)
if commit:
solr.commit(ndocs=count)
else:
logger.info("not doing solr commit as commit is off")
# don't sleep after committing some records.
# While the commit was on, some more edits might have happened.
if count == 0:
logger.debug("No more log records available, sleeping...")
time.sleep(5)
if __name__ == "__main__":
from scripts.solr_builder.solr_builder.fn_to_cli import FnToCLI
cli = FnToCLI(main)
args = cli.args_dict()
cli.run()
