def enqueue(self,
observable: Observable,
group: str = 'default-group',
retries: int = 0,
description: str = None) -> Observable:
def log_status(status):
logging.debug(
str({
'WorkQueue': str(self),
'group': group,
status: description
}))
log_status('ENQUEUED')
output = Subject()
errors = Subject()
output_finalized = Subject()
def handle_error(e, _):
log_status('FAILED')
errors.on_next(e)
return empty()
def set_output_finalized():
output_finalized.on_next(True)
work = of(True).pipe(
do_action(lambda _: log_status('STARTED')),
take_until(output_finalized),
flat_map(lambda _: observable.pipe(
map(lambda value: of({
'value': value,
'output': output
})),
retry_with_backoff(
retries=retries,
description='{}.enqueue(group={}, description={})'.format(
self, group, description)),
catch(handler=handle_error), take_until(output_finalized),
take_until_disposed())),
concat(of(of({
'completed': True,
'output': output
}))), finally_action(lambda: log_status('COMPLETED')))
self._queue.on_next({'work': work, 'group': group})
return output.pipe(observe_on(self.request_scheduler),
throw_when(errors),
take_while(lambda r: not r.get('completed')),
map(lambda r: r.get('value')),
finally_action(set_output_finalized))
def get_products(shop: Dict[str, str], search_term: str,
options) -> Observable:
domain = re.findall("\.(.+)\.com", shop['url'])[0]
print(f"Lauching {domain}")
browser = launch_browser(f"{shop['url']}{search_term}", options, shop)
base_obs = rx.of(browser).pipe(
ops.do_action(
lambda el: print(f"Getting products prices from {domain}")),
ops.flat_map(lambda browser: rx.from_(
browser.find_elements_by_xpath(shop["xpath"]["parent"]))),
ops.filter(lambda el: el.is_displayed()),
ops.map(lambda el: (
get_property(el, shop["xpath"]["product_name"]),
get_property(el, shop["xpath"]["price"]),
)),
ops.filter(lambda el: el[0] and el[1]),
ops.map(lambda el: {
"name": el[0],
"price": el[1]
}),
ops.map(lambda product: transform_price(product, shop["priceRegexp"])),
ops.finally_action(lambda: browser.close()),
)
return base_obs
def test_finally_only_called_once_empty(self):
invasserte_count = [0]
def action():
invasserte_count[0] += 1
return invasserte_count
some_observable = rx.empty().pipe(ops.finally_action(action))
d = some_observable.subscribe()
d.dispose()
d.dispose()
self.assertEqual(1, invasserte_count[0])
def merge_finalize(
handler: Callable[[],
Observable]) -> Callable[[Observable], Observable]:
def invoke_handler():
try:
return handler().pipe(flat_map(lambda _: empty()))
except:
logging.exception('Error while finalizing')
return empty()
return rx.pipe(
finally_action(lambda: subscribe_and_wait(invoke_handler())))
def test_finally_only_called_once_empty(self):
invasserte_count = [0]
def action():
invasserte_count[0] += 1
return invasserte_count
some_observable = rx.empty().pipe(ops.finally_action(action))
d = some_observable.subscribe()
d.dispose()
d.dispose()
self.assertEqual(1, invasserte_count[0])
def info_generator(self, fullnames):
with ThreadPoolExecutor() as executor:
queue = Queue(FETCHES)
executor.submit(lambda: from_iterable(fullnames).pipe(
buffer_with_count(100),
map(lambda chunk: {"id": ",".join(chunk)}),
map(lambda params: lambda: self.get(API_PATH["info"],
params=params)),
map(lambda task: executor.submit(task)),
map(lambda future: queue.put(future)),
finally_action(lambda: queue.put(None))).subscribe())
while True:
response = queue.get()
if not response:
break
yield from response.result()
def execute_task(credentials, task):
def start():
task.start()
def load_status():
return task.status()
def extract_state(status):
state = status['state']
if state == 'FAILED':
return throw(ee.EEException(status.get('error_message')))
else:
return of(state)
def monitor():
def is_running(state):
return state in [Task.State.UNSUBMITTED, Task.State.READY, Task.State.RUNNING]
return interval(credentials, _MONITORING_FREQUENCY).pipe(
flat_map(
lambda _: execute(
credentials,
action=load_status,
description='monitor task ' + str(task),
retries=3
)
),
flat_map(extract_state),
distinct_until_changed(),
take_while(is_running, inclusive=True)
)
return execute(
credentials,
start,
description='start task ' + str(task),
retries=3
).pipe(
flat_map(lambda _: monitor()),
finally_action(lambda: task.cancel())
)
def create():
def action():
invasserted[0] = True
return invasserted[0]
return xs.pipe(ops.finally_action(action))
def enqueue(self,
observable: Observable,
group: str = 'default-group',
retries: int = 0,
description: str = None):
description = description or str(Observable)
key = '{}({})'.format(description, random.random())
def log_status(status):
logging.debug(
str({
'WorkQueue': str(self),
'group': group,
'key': key,
status: description
}))
log_status('ENQUEUED')
error: Optional[Exception] = None
def handle_error(e):
log_status('FAILED')
nonlocal error
error = e
return of({'key': key, 'error': e})
def throw_if_error(r):
if error:
return throw(error)
else:
return of(r)
request_disposed = Subject()
def dispose_request():
request_disposed.on_next(True)
request = of(True).pipe(
do_action(lambda _: log_status('STARTED')),
flat_map(
lambda _: observable.pipe(
map(lambda value: {
'key': key,
'value': value
}),
retry_with_backoff(retries=retries,
description=
'{}.enqueue(group={}, description={})'.
format(self, group, description)),
catch(handler=lambda e, o: handle_error(e)),
take_until(request_disposed),
take_until_disposed(),
), ),
concat(
of({
'key': key,
'complete': True
}).pipe(do_action(lambda _: log_status('COMPLETED')))),
)
result_stream = self._output.pipe(
observe_on(self.request_scheduler),
filter(lambda r: r['key'] == key),
flat_map(lambda r: throw_if_error(r)),
take_while(lambda r: not r.get('complete')),
flat_map(lambda r: of(r.get('value'))),
finally_action(dispose_request))
self._requests.on_next({
'request': request,
'concurrency_group': group
})
return result_stream
def notifying_source():
return source.pipe(finally_action(lambda: completed.on_next(True)))
def execute_task(credentials, task):
def start():
def action():
task.start()
return execute(credentials,
action,
description='start task ' + str(task),
retries=3)
def cancel():
if task.state in [
Task.State.COMPLETED, Task.State.CANCEL_REQUESTED,
Task.State.CANCELLED, Task.State.FAILED
]:
return
ee.InitializeThread(credentials)
try:
task.cancel()
except Exception:
logging.exception('{}: cancelling failed'.format(task))
return empty()
def load_status():
return task.status()
def extract_state(status):
state = status['state']
task.state = state
if state == 'FAILED':
return throw(ee.EEException(status.get('error_message')))
else:
return of(state)
def to_progress(state):
if state == 'PENDING':
return progress(default_message=
'Submitting export task to Google Earth Engine...',
message_key='tasks.ee.export.pending')
elif state == 'READY':
return progress(
default_message=
'Waiting for Google Earth Engine to start export...',
message_key='tasks.ee.export.ready')
elif state == 'RUNNING':
return progress(
default_message='Google Earth Engine is exporting...',
message_key='tasks.ee.export.running')
else:
return empty()
def monitor():
def is_running(state):
return state in [
Task.State.UNSUBMITTED, Task.State.READY, Task.State.RUNNING
]
return interval(credentials, _MONITORING_FREQUENCY).pipe(
flat_map(lambda _: execute(credentials,
action=load_status,
description='monitor task ' + str(task),
retries=3)), flat_map(extract_state),
distinct_until_changed(), take_while(is_running, inclusive=True),
flat_map(to_progress))
return start().pipe(
flat_map(lambda _: monitor()),
finally_action(cancel),
)
def handler(args):
return interval(0.01).pipe(
take(10), finally_action(lambda: subject.on_completed()))
def create():
def action():
invasserted[0] = True
return invasserted[0]
return xs.pipe(ops.finally_action(action))