def complete_queue_item(self, result: FetcherResult, context: KarnakSqsFetcherThreadContext):
"""Put fetched results in queue (in case of success or failure with no retry)"""
item = result.queue_item
kl.trace(f'completing item: {item.keys}')
message_str = result.to_string()
worker_queue_name = self.sqs_fetcher.worker_queue_name(extractor=item.extractor, priority=item.priority)
try:
if len(message_str) > 262144:
kl.warn(f'message too long to put in queue, key: {item.keys}, {len(item.to_string())} bytes')
else:
ksqs.send_messages(self.sqs_fetcher.results_queue_name(), [message_str])
ksqs.remove_message(worker_queue_name, item.handle)
except Exception as e:
kl.exception(f'exception putting message in results queue: {item.keys}', e)
def refresh_queue_item(self, item: FetcherQueueItem,
context: KarnakSqsFetcherThreadContext,
new_extractor: str = None,
new_priority: int = None):
_extractor = item.extractor if new_extractor is None else new_extractor
_priority = item.priority if new_priority is None else new_priority
old_queue_name = self.sqs_fetcher.worker_queue_name(extractor=item.extractor, priority=item.priority)
new_queue_name = self.sqs_fetcher.worker_queue_name(extractor=_extractor, priority=_priority)
new_item = item
if new_extractor is not None:
new_item = item.set_extractor(extractor=new_extractor)
if new_priority is not None:
new_item = item.set_priority(priority=new_priority)
ksqs.send_messages(new_queue_name, [new_item.to_string()])
ksqs.remove_message(old_queue_name, item.handle)
def complete_item(self, item: FetcherItem):
"""Put fetcher item in results queue (in case of success or failure with no retry)"""
kl.trace(f'completing item: {item.key}')
message_str = item.to_string()
try:
if len(message_str) > 262144:
kl.warn(
f'message too long to put in queue, key: {item.key}, {len(item.to_string())} bytes'
)
ksqs.remove_message(self.worker_queue_name, item.handle)
else:
ksqs.send_messages(self.fetcher.results_queue,
[item.to_string()])
ksqs.remove_message(self.worker_queue_name, item.handle)
except Exception as e:
kl.exception(f'exception putting message in queue: {item.key}', e)
def resend_item(self, item: FetcherItem):
strategy = item.strategy
queue = self.fetcher.worker_queue(strategy)
ksqs.send_messages(queue, [item.to_string()])
ksqs.remove_message(self.worker_queue_name, item.handle)
def send_item(self, item: FetcherItem):
strategy = item.strategy
queue = self.fetcher.worker_queue(strategy)
ksqs.send_messages(queue, [item.to_string()])
def populate_worker_queue(self, items: List[FetcherItem], strategy: str):
contents = [i.to_string() for i in items]
ksqs.send_messages(self.worker_queue(strategy=strategy), contents)
def push_queue_item(self, item: FetcherQueueItem, context: KarnakSqsFetcherThreadContext):
queue_name = self.sqs_fetcher.worker_queue_name(extractor=item.extractor, priority=item.priority)
ksqs.send_messages(queue_name, [item.to_string()])
def populate_worker_queue(self, items: List[FetcherQueueItem], extractor: str, priority: Optional[int]):
worker_queue_name = self.worker_queue_name(extractor=extractor, priority=priority)
kl.trace(f'putting {len(items)} messages in queue {worker_queue_name}')
contents = [i.to_string() for i in items]
ksqs.send_messages(worker_queue_name, contents)