def worker_queue(self, strategy: str = None) -> Optional[str]:
if strategy is None:
return self.worker_queues[0]
elif strategy in self.strategies:
return self.worker_queues[self.strategies.index(strategy)]
else:
kl.error(f'invalid worker strategy: {strategy}')
return None
def from_string(cls, string, handle=None):
d = json.loads(string)
try:
f = FetcherItem(key=d['key'],
start_ts=datetime.datetime.fromisoformat(
d['start_ts']),
strategy=d['strategy'],
current_retries=d['current_retries'],
is_success=d['is_success'],
is_timeout=d['is_timeout'],
content=d['content'],
fetch_errors=d['fetch_errors'],
handle=handle)
except KeyError:
msg = string.replace("\n", "")
kl.error(f'invalid message: {msg}')
return None
return f
def rebalance(self, from_strategy: str, to_strategy: str, items_cnt: int):
rebalance_cnt = min(items_cnt, 100_000)
kl.debug(
f'rebalancing {items_cnt} items from {from_strategy} to {to_strategy}'
)
from_queue = self.worker_queue(from_strategy)
to_queue = self.worker_queue(to_strategy)
if from_queue is None or to_queue is None:
kl.error('rebalance not possible: invalid strategy')
else:
items = self.fetch_items(from_queue, max_items=rebalance_cnt)
rebalanced_items = [
i.reset_strategy(to_strategy, reset_errors=True) for i in items
]
self.populate_worker_queue(rebalanced_items, to_strategy)
handles = [i.handle for i in items]
ksqs.remove_messages(queue_name=from_queue,
receipt_handles=handles)
kl.info('rebalance: finished')
def from_string(cls, s: str, handle=None):
d = orjson.loads(s)
try:
f = FetcherQueueItem(keys=d.get('keys'),
key_property=d.get('key_property'),
table=d.get('table'),
extractor=d.get('extractor'),
creation_ts=datetime.datetime.fromisoformat(d.get('creation_ts')),
update_dt=datetime.datetime.fromisoformat(d.get('update_dt')),
cohort=d.get('cohort'),
priority=d.get('priority'),
current_retries=d.get('current_retries'),
# fetch_errors=d.get('fetch_errors'),
handle=handle)
except KeyError:
msg = s.replace("\n", "")
kl.error(f'invalid message: {msg}')
return None
return f
def from_string(cls, s: str, handle=None):
try:
d = orjson.loads(s)
elapsed_raw = d.get('elapsed')
elapsed = None if elapsed_raw is None else datetime.timedelta(seconds=elapsed_raw)
queue_item = FetcherQueueItem.from_string(d['queue_item'])
fr = FetcherResult(queue_item=queue_item,
data=d.get('data'),
capture_ts=d.get('capture_ts'),
elapsed=elapsed,
rows=d.get('rows'),
is_success=d.get('is_success'),
can_retry=d.get('can_retry'),
compression=d.get('compression'),
error_type=d.get('error_type'),
error_message=d.get('error_message'),
response_edition=d.get('response_edition'),
handle=handle)
return fr
except KeyError:
msg = s.replace("\n", " ")
kl.error(f'invalid message: {msg}')
return None