def update_index():
for item in ItemInfo.poll_status_index(TaskStage.Fetching,
TaskStatus.Queued,
limit=config.limit.fetch):
service = get_service(item.service)
ItemInfo.set_status(item.service, item.item_id, TaskStage.Fetching,
TaskStatus.Pending)
try:
full_item = service.pull_item(IndexItem(item.service,
item.item_id))
except Exception as err:
traceback.print_exc()
ItemInfo.set_status(item.service, item.item_id, TaskStage.Fetching,
TaskStatus.Queued)
else:
print(item, full_item)
if full_item is None:
ItemInfo.set_status(item.service, item.item_id,
TaskStage.Fetching, TaskStatus.Failed)
else:
ItemInfo.add_item(full_item, [])
ItemInfo.set_status(item.service, item.item_id,
TaskStage.Downloading, TaskStatus.Queued)
time.sleep(1)
ItemInfo.abandon_tasks(TaskStage.Fetching, TaskStatus.Queued, 20,
TaskStage.Fetching, TaskStatus.Failed)
def download_images():
for item in ItemInfo.poll_status(TaskStage.Downloading,
TaskStatus.Queued,
limit=config.limit.download):
service = get_service(item.service)
ItemInfo.set_status(item.service, item.item_id, TaskStage.Downloading,
TaskStatus.Pending)
try:
for url in item.image_urls:
print(url)
raw = service.download_item_image(item, url)
img = Image.open(raw)
with BytesIO() as buf:
img.save(buf, format="PNG")
buf.seek(0)
ItemInfo.save_image(item, url, buf)
time.sleep(1)
if item.attachment_urls:
ctr = itertools.count()
for att_url in item.attachment_urls:
for zf in service.extract_attachments(item, att_url):
img = Image.open(zf)
with BytesIO() as buf:
img.save(buf, format="PNG")
buf.seek(0)
ItemInfo.save_attachment_image(
item, next(ctr), buf)
except Exception as err:
traceback.print_exc()
ItemInfo.set_status(item.service, item.item_id,
TaskStage.Downloading, TaskStatus.Queued)
else:
ItemInfo.set_status(item.service, item.item_id, TaskStage.Posting,
TaskStatus.Queued)
ItemInfo.abandon_tasks(TaskStage.Downloading, TaskStatus.Queued, 20,
TaskStage.Downloading, TaskStatus.Failed)