def upload_pdfs_to_internet_archive(options, do_non_free=False):
"""Upload items to the Internet Archive."""
q = options['queue']
rds = RECAPDocument.objects.filter(
Q(ia_upload_failure_count__lt=3) | Q(ia_upload_failure_count=None),
is_available=True,
filepath_ia='',
).exclude(
filepath_local='',
).values_list(
'pk',
flat=True,
).order_by()
if do_non_free:
rds = rds.filter(Q(is_free_on_pacer=False) | Q(is_free_on_pacer=None))
else:
rds = rds.filter(is_free_on_pacer=True)
count = rds.count()
logger.info("Sending %s items to Internet Archive.", count)
throttle = CeleryThrottle(queue_name=q)
for i, rd in enumerate(rds):
throttle.maybe_wait()
if i > 0 and i % 1000 == 0:
logger.info("Sent %s/%s tasks to celery so far.", i, count)
upload_pdf_to_ia.si(rd).set(queue=q).apply_async()
def upload_to_internet_archive(options, do_non_free=False):
"""Upload items to the Internet Archive."""
q = options['queue']
rds = RECAPDocument.objects.filter(
Q(ia_upload_failure_count__lt=3) | Q(ia_upload_failure_count=None),
is_available=True,
filepath_ia='',
).exclude(
filepath_local='',
).values_list(
'pk',
flat=True,
).order_by()
if do_non_free:
rds = rds.filter(Q(is_free_on_pacer=False) | Q(is_free_on_pacer=None))
else:
rds = rds.filter(is_free_on_pacer=True)
count = rds.count()
logger.info("Sending %s items to Internet Archive." % count)
throttle = CeleryThrottle(queue_name=q)
for i, rd in enumerate(rds):
throttle.maybe_wait()
if i > 0 and i % 1000 == 0:
logger.info("Sent %s/%s tasks to celery so far." % (i, count))
upload_pdf_to_ia.si(rd).set(queue=q).apply_async()