def post_images():
for item in ItemInfo.poll_status(TaskStage.Posting, TaskStatus.Queued):
channels = ItemInfo.get_channels(item)
for ch in channels:
for pipe in config.pipeline[ch].push:
SecondaryTask.add_task(item.service, item.item_id, pipe.service, pipe.config, ch)
ItemInfo.set_status(item.service, item.item_id, TaskStage.Posting, TaskStatus.Pending)
for stype, item_id, ptype, conf, ch, poll_counter in SecondaryTask.poll_tasks(20):
SecondaryTask.acquire_task(stype, item_id, ptype, conf, ch)
print((stype.value, item_id), '=>', (ptype.value, conf))
item = ItemInfo.get_item(stype, item_id)
if not service_exists(ptype, conf):
continue
client = get_service(ptype, conf)
if poll_counter >= client.push_limit():
print("Failed to push item.")
SecondaryTask.close_task(stype, item_id, ptype, conf, ch)
else:
images = [BytesIO(i.read()) for i in ItemInfo.get_images(item)]
if item.attachment_urls:
attachment_images = [
BytesIO(i.read()) for i in ItemInfo.get_attachment_images(item)
]
images.extend(attachment_images)
converted_username = pull_services[item.service].convert_username(item.source_id)
try:
client.push_item(item, images, ch, converted_username)
except Exception as err:
traceback.print_exc()
SecondaryTask.release_task(stype, item_id, ptype, conf, ch)
else:
SecondaryTask.close_task(stype, item_id, ptype, conf, ch)
if SecondaryTask.task_done(stype, item_id):
print("Post Done", (item.service, item.item_id))
ItemInfo.set_status(item.service, item.item_id, TaskStage.Cleaning, TaskStatus.Queued)
def download_images():
for item in ItemInfo.poll_status(TaskStage.Downloading,
TaskStatus.Queued,
limit=config.limit.download):
service = get_service(item.service)
ItemInfo.set_status(item.service, item.item_id, TaskStage.Downloading,
TaskStatus.Pending)
try:
for url in item.image_urls:
print(url)
raw = service.download_item_image(item, url)
img = Image.open(raw)
with BytesIO() as buf:
img.save(buf, format="PNG")
buf.seek(0)
ItemInfo.save_image(item, url, buf)
time.sleep(1)
if item.attachment_urls:
ctr = itertools.count()
for att_url in item.attachment_urls:
for zf in service.extract_attachments(item, att_url):
img = Image.open(zf)
with BytesIO() as buf:
img.save(buf, format="PNG")
buf.seek(0)
ItemInfo.save_attachment_image(
item, next(ctr), buf)
except Exception as err:
traceback.print_exc()
ItemInfo.set_status(item.service, item.item_id,
TaskStage.Downloading, TaskStatus.Queued)
else:
ItemInfo.set_status(item.service, item.item_id, TaskStage.Posting,
TaskStatus.Queued)
ItemInfo.abandon_tasks(TaskStage.Downloading, TaskStatus.Queued, 20,
TaskStage.Downloading, TaskStatus.Failed)
def clean_cache():
for item in ItemInfo.poll_status(TaskStage.Cleaning, TaskStatus.Queued):
ItemInfo.clean_cache(item)
ItemInfo.set_status(item.service, item.item_id, TaskStage.Done,
TaskStatus.Queued)