def ack_docket_item(scrape_key: ScrapeKey, ack_id: str):
"""Ack a specific docket item
Acknowledges a specific docket item once it's been completed. This indicates
to pubsub that it should not be delivered again.
Args:
ack_id: (string) Id used to ack the message
Returns:
N/A
"""
def inner():
pubsub_helper.get_subscriber().acknowledge(
subscription=pubsub_helper.get_subscription_path(
scrape_key, pubsub_type=PUBSUB_TYPE),
ack_ids=[ack_id])
pubsub_helper.retry_with_create(scrape_key, inner, pubsub_type=PUBSUB_TYPE)
def get_new_docket_item(
scrape_key: ScrapeKey,
return_immediately=False) -> Optional[pubsub.types.ReceivedMessage]:
"""Retrieves an item from the docket for the specified region / scrape type
Retrieves an arbitrary item still in the docket (whichever docket
type is specified). If the docket is currently empty this will wait for
a bounded period of time for a message to be published, ensuring that newly
created tasks are received. This behavior can be overriden using the
return_immediately param.
Args:
scrape_key: (ScrapeKey) The scraper to lease an item for
return_immediately: (bool) Whether to return immediately or to wait for
a bounded period of time for a message to enter the docket.
Returns:
Task entity from queue
None if query returns None
"""
docket_message = None
subscription_path = pubsub_helper.get_subscription_path(
scrape_key, pubsub_type=PUBSUB_TYPE)
def inner() -> pubsub.types.PullResponse:
return pubsub_helper.get_subscriber().pull(
subscription=subscription_path,
max_messages=1,
return_immediately=return_immediately,
)
response = pubsub_helper.retry_with_create(scrape_key,
inner,
pubsub_type=PUBSUB_TYPE)
if response.received_messages:
docket_message = response.received_messages[0]
logging.info("Leased docket item from subscription: [%s]",
subscription_path)
else:
logging.info(
"No matching docket item found in the docket queue for "
"scraper: %s",
scrape_key,
)
return docket_message