async def test_merge(assert_run, event_loop):
with event_loop.assert_cleanup():
xs = stream.range(1, 5, 2, interval=2) | pipe.delay(1)
ys = stream.range(0, 5, 2, interval=2) | pipe.merge(xs)
await assert_run(ys, [0, 1, 2, 3, 4])
assert event_loop.steps == [1, 1, 1, 1]
with event_loop.assert_cleanup():
xs = stream.range(1, 5, 2, interval=2) | pipe.delay(1)
ys = stream.range(0, 5, 2, interval=2) | pipe.merge(xs)
await assert_run(ys[:3], [0, 1, 2])
assert event_loop.steps == [1, 1]
with event_loop.assert_cleanup():
xs = stream.just(1) + stream.never()
ys = xs | pipe.merge(xs) | pipe.timeout(1)
await assert_run(ys, [1, 1], asyncio.TimeoutError())
assert event_loop.steps == [1]
# Reproduce issue #65
with event_loop.assert_cleanup():
xs = stream.iterate([1, 2])
ys = stream.iterate([3, 4])
zs = stream.merge(xs, ys) | pipe.take(3)
await assert_run(zs, [1, 2, 3])
with event_loop.assert_cleanup():
xs = stream.iterate([1, 2, 3])
ys = stream.throw(ZeroDivisionError)
zs = stream.merge(xs, ys) | pipe.delay(1) | pipe.take(3)
await assert_run(zs, [1, 2, 3])
# Silencing of a CancelledError
async def agen1():
if False:
yield
try:
await asyncio.sleep(2)
except asyncio.CancelledError:
return
async def agen2():
yield 1
with event_loop.assert_cleanup():
xs = stream.merge(agen1(), agen2()) | pipe.delay(1) | pipe.take(1)
await assert_run(xs, [1])
async def main(args):
app_id = get_app_id(args)
if app_id is None:
logger.critical('No APP_ID found')
return False
# Loading modules
logger.info('Loading modules')
modules = ModuleCache(args.modules)
assert modules, 'No modules loaded'
# Loading token
with closing(appstorage.create('vk-dump', 'the-island.ru')) as db:
token = await get_or_create_token(args, app_id, db, modules.scope)
async with TokenSession(token) as session:
api = API(session)
rate = TokenLimiter(args.parallel)
async with aiohttp.ClientSession() as session:
if not os.path.isdir(args.root):
logger.info('Creating root dump directory %s', args.root)
os.makedirs(args.root)
downloader = Downloader(args.root, session, rate)
xs = (
stream.merge(*[m.get_items(api, rate) for m in modules])
| pipe.map(downloader.download) # pylint: disable=E1101
)
await xs
logger.info('Done')
async def run(self):
'''run the engine'''
# setup future queue
self._queued_events = deque()
# await all connections
await asyncio.gather(*(asyncio.create_task(exch.connect())
for exch in self.exchanges))
await asyncio.gather(*(asyncio.create_task(exch.instruments())
for exch in self.exchanges))
# send start event to all callbacks
await self.tick(Event(type=EventType.START, target=None))
async with merge(*(
exch.tick() for exch in self.exchanges
if inspect.isasyncgenfunction(exch.tick))).stream() as stream:
# stream through all events
async for event in stream:
# tick exchange event to handlers
await self.tick(event)
# TODO move out of critical path
self._latest = event.target.timestamp if hasattr(
event, 'target') and hasattr(event.target,
'timestamp') else self._latest
# process any secondary events
while self._queued_events:
event = self._queued_events.popleft()
await self.tick(event)
await self.tick(Event(type=EventType.EXIT, target=None))
async def main() -> None:
new_e3 = NewE3()
old_e3 = OldE3()
with open("config.json", "r") as f:
config = json.loads(f.read())
username = config.get("studentId", "")
old_e3_pwd = config.get("oldE3Password", "")
new_e3_pwd = config.get("newE3Password", "")
download_path = config.get("downloadPath", "e3")
gdrive_enable = config.get("gdrive_enable", True)
download_path = os.path.expanduser(download_path)
if gdrive_enable:
store = oauth_file.Storage("token.json")
creds = store.get()
if not creds or creds.invalid:
flow = client.flow_from_clientsecrets("credentials.json", SCOPES)
creds = tools.run_flow(flow, store)
while True:
if username == "":
username = input("StudentID: ")
if old_e3_pwd == "":
old_e3_pwd = getpass("Old E3 Password: "******"", ""
print("ID or Old E3 Password Error")
while True:
if new_e3_pwd == "":
new_e3_pwd = getpass("New E3 Password: "******""
print("New E3 Password Error")
downloader = Downloader(download_path)
async with stream.merge(new_e3.all_files(), old_e3.all_files()).stream() as files:
async for file in files:
downloader.add_file(file)
modified_files = await downloader.done()
if gdrive_enable:
gdirve_client = GDrive(download_path)
await gdirve_client.upload()
print("")
if modified_files:
print("The below files are added or modified")
modified_files.sort(key=lambda x: x.course_name)
for modified_file in modified_files:
print(f"{modified_file.course_name} - {modified_file.name}")
else:
print("No files are added or modified")
async def _streamSSEAsync(url, exit=None):
"""internal"""
from asyncio import Event
from aiohttp_sse_client import client as sse_client
from aiostream.stream import merge
async with sse_client.EventSource(url) as event_source:
if isinstance(exit, Event):
async def _waitExit():
yield await exit.wait()
waits = (_waitExit(), event_source)
else:
waits = (event_source,)
try:
async with merge(*waits).stream() as stream:
try:
async for event in stream:
if event == True: # noqa: E712
return
yield json.loads(event.data)
except ConnectionError:
raise PyEXception("Could not connect to SSE Stream")
except PyEXStopSSE:
return
except BaseException:
raise
except (json.JSONDecodeError, KeyboardInterrupt):
raise
def _read_sensor( # pylint: disable=too-many-arguments
self, source_uuid: UUID, sid: int, unit: str, topic: str,
callback_config: AdvancedCallbackConfiguration
) -> AsyncGenerator[DataEvent, None]:
monitor_stream = (
stream.repeat(self.device, interval=1)
| pipe.map(
async_(lambda sensor: sensor.get_callback_configuration(sid)))
| pipe.map(lambda current_config: None
if current_config == callback_config else self.device)
| pipe.filter(lambda sensor: sensor is not None)
| pipe.action(lambda sensor: logging.getLogger(__name__).info(
"Resetting callback config for %s", sensor))
| pipe.action(
async_(lambda sensor: sensor.set_callback_configuration(
sid, *callback_config)))
| pipe.filter(lambda x: False))
return stream.merge(
stream.just(monitor_stream),
stream.iterate(self.device.read_events(sids=(sid, )))
| pipe.map(lambda item: DataEvent(sender=source_uuid,
topic=topic,
value=item.payload,
sid=item.sid,
unit=str(unit))),
)
async def get_tweets(self):
queries = [Query(q_str, limit=self.limit, sem=self.sem).get_tweets()
for q_str in self.q_strs]
qs = stream.merge(*queries)
async with qs.stream() as streamer:
async for q in streamer:
yield q
async def r_invoice_gen(user: User, *_):
# create new new pub sub client for streaming locally paid invoices
local_stream = PUBSUB.add_client(user.username)
# create stream for remotely paid invoices
remote_stream = await LND.stub.SubscribeInvoices(ln.InvoiceSubscription())
global_stream = stream.merge(local_stream, remote_stream)
async with global_stream.stream() as streamer:
async for response in streamer:
try:
# check if response if from lnd
# external payment or pubsub - local payment
if isinstance(response, Invoice):
# invoice model received from pubsub client
# yield and default resolver will retrieve requested fields
yield response
else:
# payment comes from lnd,
# check if its associated with this user
invoice = None
if response.state == 1:
invoice = await Invoice.get(response.r_hash)
if invoice and invoice.payee == user.username:
# received a paid invoice with this user as payee
updated = await invoice.update(
paid=True, paid_at=invoice.settle_date
).apply()
yield updated
except GeneratorExit:
# user closed stream, del pubsub queue
del local_stream
if len(PUBSUB[user.username]) == 0:
del PUBSUB[user.username]
async def fetch(self, *args, **kwargs):
gens = [fetch(url, **kwargs) for url in self._urls]
async with stream.merge(*gens).stream() as chunks:
async for chunk in chunks:
if isinstance(chunk, bytes):
yield chunk
elif isinstance(chunk, Exception):
self.errors.append(chunk)
async def run(self):
'''run the engine'''
# setup future queue
self._queued_events = deque()
self._queued_targeted_events = deque()
# await all connections
await asyncio.gather(*(asyncio.create_task(exch.connect())
for exch in self.exchanges))
await asyncio.gather(*(asyncio.create_task(exch.instruments())
for exch in self.exchanges))
# send start event to all callbacks
await self.processEvent(Event(type=EventType.START, target=None))
# **************** #
# Main event loop
# **************** #
async with merge(
*(exch.tick() for exch in self.exchanges + [self]
if inspect.isasyncgenfunction(exch.tick))).stream() as stream:
# stream through all events
async for event in stream:
# tick exchange event to handlers
await self.processEvent(event)
# TODO move out of critical path
if self._offline():
# use time of last event
self._latest = event.target.timestamp if hasattr(
event, 'target') and hasattr(
event.target, 'timestamp') else self._latest
else:
# use now
self._latest = datetime.now()
# process any secondary events
while self._queued_events:
event = self._queued_events.popleft()
await self.processEvent(event)
# process any secondary callback-targeted events (e.g. order fills)
# these need to route to a specific callback,
# rather than all callbacks
while self._queued_targeted_events:
strat, event = self._queued_targeted_events.popleft()
# send to the generating strategy
await self.processEvent(event, strat)
# process any periodics
await asyncio.gather(
*(asyncio.create_task(p.execute(self._latest))
for p in self.manager._periodics))
# Before engine shutdown, send an exit event
await self.processEvent(Event(type=EventType.EXIT, target=None))
async def fetch(self, *args, **kwargs):
gens = [fetch(url, **kwargs) for url in self._urls]
async with stream.merge(*gens).stream() as chunks:
async for chunk in chunks:
if isinstance(chunk, (bytes, str)):
yield chunk
elif isinstance(chunk, Exception):
self.errors.append(chunk)
else:
logger.warning("Could not yield chunk")
async def aiter_opened_prs(seed_prs, session):
for pr in seed_prs:
event = await pr_number_as_pull_event(pr, session)
if event_is_pull_request_opened(event):
yield event
async with stream.merge(aiter_nixpkgs_events(session),
aiter_server_events(session)).stream() as streamer:
async for event in streamer:
if event_is_pull_request_opened(event):
yield event
async def all_files(self) -> AsyncIterable[E3File]:
if self._logged_in_session is None:
raise RuntimeError("Please Login First")
courses = list(await self._get_course_list(self._logged_in_session))
gens = [self._get_course_all_files(course) for course in courses[:-1]]
gens.append(
self._get_course_all_files(courses[-1], self._logged_in_session))
async with stream.merge(*gens).stream() as files:
async for file in files:
yield file
self._logged_in_session = None
async def run(self):
'''run the engine'''
# await all connections
await asyncio.gather(*(asyncio.create_task(exch.connect()) for exch in self.exchanges))
# send start event to all callbacks
await self.tick(Event(type=EventType.START, target=None))
async with merge(*(exch.tick() for exch in self.exchanges)).stream() as stream:
async for event in stream:
await self.tick(event)
async def receive(self) -> None:
'''gemini has its own receive method because it uses 1 connection per symbol instead of multiplexing'''
async def get_data_sub_pair(ws, sub=None):
async for ret in ws:
yield ret, sub
# add one for private stream
async for val in stream.merge(*[
get_data_sub_pair(self.ws[i], sub)
for i, sub in enumerate(self.subscription() + [None])
]):
jsn = json.loads(val[0].data)
if isinstance(jsn, dict) and 'events' in jsn:
events = jsn.get('events', [])
elif not isinstance(jsn, list):
events = [jsn]
else:
events = jsn
if val[1]:
# data stream
pair = json.loads(val[1]).get('product_id')
else:
# private events
pair = None
for item in events:
if item.get('type', 'subscription_ack') in ('subscription_ack',
'heartbeat'):
# can skip these
continue
if item.get('type') == 'accepted':
# can ignore these as well, will have a fill and/or booked
# https://docs.gemini.com/websocket-api/#workflow
continue
if item.get('type') == 'closed':
# can ignore these as well, will have a fill or cancelled
# https://docs.gemini.com/websocket-api/#workflow
continue
if pair is None:
# private events
pair = item['symbol']
item['symbol'] = pair
res = self.tickToData(item)
if not self._running:
pass
if res.type != TickType.HEARTBEAT:
self.callback(res.type, res)
async def websocket_handler(self, request):
"""
Push new active visitor counts to websocket.
Accepts '{"type": "setFilter", "filter": {"account": <id>}}' payload to
filter out account messages sent on connection.
"""
# Capture account to send events for.
account_id = None
# Prepare request for websocket
ws = aiohttp.web.WebSocketResponse()
await ws.prepare(request)
# combine websocket stream, and active visitors channel
combined = stream.takewhile(stream.merge(active_visitors_channel, ws),
lambda _: not ws.closed)
async with combined.stream() as s:
async for event in s:
if isinstance(event, Event):
av = event.value
if (not account_id) or (account_id == av.account_id):
await ws.send_json({
"account_id": av.account_id,
"store_id": av.store_id,
"ts_from": av.window[0],
"ts_to": av.window[1],
"count": av.count,
})
elif event.type == aiohttp.WSMsgType.TEXT:
if event.data == "close":
await ws.close()
else:
try:
data = json.loads(event.data)
if data.get("type") == "setFilter":
account_id = data["filter"]["account"]
logger.info(
f"Sending messages for account {account_id}")
except Exception as e:
logger.exception(
f"Error handling message '{event.data}': {str(e)}")
elif event.type == aiohttp.WSMsgType.ERROR:
logger.warning(
f"ws connection closed with error: {ws.exception()}")
async def connect(self, *args, **kwargs): # NOQA
self.user = self.scope['user']
if not self.user.is_authenticated:
await self.close()
return
try:
qs = parse_qs(self.scope['query_string'].decode())
except Exception:
await self.close()
return
if 'path' not in qs:
await self.close()
return
self.path = pathlib.Path(qs['path'][0])
self.preview = preview_re.match(str(self.path))
self.dummy_val = dict()
if self.preview:
await self.accept()
return
try:
self.md = await misc.get_markdown_factory().get_markdown(
self.path, self.user)
except KeyError:
await self.accept()
return
streams = list()
for idx, field in enumerate(self.md.display_fields):
streams.append(my_stream.display(self, idx))
for name, field in self.md.input_fields.items():
streams.append(my_stream.input(self, name))
self.stream = stream.merge(*streams)
# python 3.6 does not have create_task() yet
# self.run_task = asyncio.create_task(self.run())
self.run_task = asyncio.ensure_future(self.run())
await self.accept()
async def execute_shell_cmd(cmd: str, context: Context, log_outputs=True):
async with context.start(action="execute shell command",
with_labels=["BASH"]) as ctx:
await ctx.info(text=cmd)
p = await asyncio.create_subprocess_shell(
cmd=cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
shell=True)
outputs = []
async with stream.merge(p.stdout,
p.stderr).stream() as messages_stream:
async for message in messages_stream:
outputs.append(message.decode('utf-8'))
log_outputs and await context.info(text=outputs[-1])
await p.communicate()
return p.returncode, outputs
async def main():
async with Bergen(
host="p-tnagerl-lab1",
port=8000,
client_id="DSNwVKbSmvKuIUln36FmpWNVE2KrbS2oRX0ke8PJ",
client_secret=
"Gp3VldiWUmHgKkIxZjL2aEjVmNwnSyIGHWbQJo6bWMDoIUlBqvUyoGWUWAe6jI3KRXDOsD13gkYVCZR0po1BLFO9QT4lktKODHDs0GyyJEzmIjkpEOItfdCC4zIa3Qzu",
name=
"frankomanko", # if we want to specifically only use pods on this innstance we would use that it in the selector
):
sleep = await Node.asyncs.get(package="karl", interface="karl")
twentysleeps = aios.merge(
*[sleep.stream({"interval": 1}) for i in range(0, 1)])
async with twentysleeps.stream() as stream:
async for item in stream:
print(item)
async def find_congruences_lect11_paralel(g, h, p, B: int, primes: list,
max_equations: int):
congruences, bases = [], []
unique = lambda l: list(set(l))
inv_h = inverse(h, p)
# while True:
new_seeds = sample(range(1, 5000), 4)
#[worker(g, h, p, B, primesB, s) for s in new_seeds]
combined_stream = stream.merge(
worker(g, inv_h, p, B, primes, new_seeds[0]),
worker(g, inv_h, p, B, primes, new_seeds[1]),
worker(g, inv_h, p, B, primes, new_seeds[2]),
worker(g, inv_h, p, B, primes, new_seeds[3]))
async with combined_stream.stream() as streamer:
async for item in streamer:
congruences.append((item[0], item[1]))
bases = unique([
base for c in [c[0].keys() for c in congruences] for base in c
])
if len(congruences) >= max_equations: break
return bases, congruences
async def scanner(network, ports=None, timeout=0.5, csv=False):
"""
main task coroutine which manages all the other functions
if scanning over the internet, you might want to set the timeout
to around 1 second, depending on internet speed.
"""
scan_completed = asyncio.Event()
scan_completed.clear() # progress the main loop
if ports is None: # list of common-ass ports
ports = (
"9,20-23,25,37,41,42,53,67-70,79-82,88,101,102,107,109-111,"
"113,115,117-119,123,135,137-139,143,152,153,156,158,161,162,170,179,"
"194,201,209,213,218,220,259,264,311,318,323,383,366,369,371,384,387,"
"389,401,411,427,443-445,464,465,500,512,512,513,513-515,517,518,520,"
"513,524,525,530,531,532,533,540,542,543,544,546,547,548,550,554,556,"
"560,561,563,587,591,593,604,631,636,639,646,647,648,652,654,665,666,"
"674,691,692,695,698,699,700,701,702,706,711,712,720,749,750,782,829,"
"860,873,901,902,911,981,989,990,991,992,993,995,8080,2222,4444,1234,"
"12345,54321,2020,2121,2525,65535,666,1337,31337,8181,6969")
ports = parse_ports(ports)
# initialize task generator
task_gen = task_generator(network, ports, timeout)
open_ports = list()
eprint('scanning . . .')
workers = [task_worker(task_gen) for _ in range(MAX_CONCURRENCY)]
merged = merge(*workers)
async with merged.stream() as streamer:
async for task in streamer:
open_ports.append(task)
eprint('gathering output . . .')
open_ports.sort(key=ip_sort)
fancy_print(open_ports, csv=csv)
eprint('shutting down . . .')
async def run(self):
'''run the engine'''
# setup future queue
self._queued_events = deque()
# await all connections
await asyncio.gather(*(asyncio.create_task(exch.connect()) for exch in self.exchanges))
await asyncio.gather(*(asyncio.create_task(exch.instruments()) for exch in self.exchanges))
# send start event to all callbacks
await self.tick(Event(type=EventType.START, target=None))
async with merge(*(exch.tick() for exch in self.exchanges)).stream() as stream:
# stream through all events
async for event in stream:
# tick exchange event to handlers
await self.tick(event)
# process any secondary events
while self._queued_events:
event = self._queued_events.popleft()
await self.tick(event)
await self.tick(Event(type=EventType.EXIT, target=None))
async def receive(self) -> None:
'''gemini has its own receive method because it uses 1 connection per symbol instead of multiplexing'''
async def get_data_sub_pair(ws, sub=None):
async for ret in ws:
yield ret, sub
async for val in stream.merge(*[
get_data_sub_pair(self.ws[i], sub)
for i, sub in enumerate(self.subscription())
]):
pair = json.loads(val[1]).get('product_id')
jsn = json.loads(val[0].data)
if jsn.get('type') == 'heartbeat':
pass
else:
for item in jsn.get('events'):
item['symbol'] = pair
res = self.tickToData(item)
if not self._running:
pass
if res.type != TickType.HEARTBEAT:
self.callback(res.type, res)
async def docker(ic, args):
if len(args) < 1:
yield None
return
if not isinstance(args[0], pathlib.Path):
yield ('err', "⚠ wrong first argument format ⚠")
return
docker_path = pathlib.Path(os.path.normpath(ic.path / args[0]))
dapi = None
try:
dapi = aiodocker.Docker()
con = await get_container(dapi, docker_path, ic.user)
if con is None:
return
logger.info(f"{con['id'][:12]}: created")
ain = dict()
aout = dict()
for n, arg in enumerate(args[1:]):
ain[n + 1] = stream_enum(
n + 1, await my_stream.arg_stream(ic, ic.user, arg))
ws = await con.websocket(stdin=True,
stdout=True,
stderr=True,
stream=True)
await con.start()
logger.debug(f"{con['id'][:12]}: started")
msgs_n = 0
msgs_ts = time.time()
try:
restart = True
while restart:
restart = False
if ain:
s = stream.merge(*list(ain.values()), websocket_reader(ws))
else:
s = stream.merge(websocket_reader(ws))
# if input is empty, send empty message to container
logger.debug(f"{con['id'][:12]}: < []")
await ws.send_str(json.dumps(list()) + "\n")
out = list()
async with core.streamcontext(s) as streamer:
async for item in streamer:
if item[0] == 'ws':
logger.debug(
f"{con['id'][:12]}: > {item[1][:120]}")
msgs_n += 1
tdiff = time.time() - msgs_ts
if tdiff > 20 and msgs_n > 1000 and msgs_n / tdiff > 20:
logger.info(
f"{con['id'][:12]}: receiving too much messages {msgs_n} in {tdiff}s"
)
yield {
'type':
'error',
'val':
f"receiving too much messages {msgs_n} in {tdiff}s"
}
break
m = wi_native_re.match(item[1])
if m:
if m.group(1) == 'clear':
out = list()
yield {'type': None, 'val': ""}
elif m.group(1).startswith('progress'):
yield {
'type':
'html',
'val':
'<div class="progress"><div class="progress-bar progress-bar-striped progress-bar-animated" role="progressbar" aria-valuenow="50" aria-valuemin="0" aria-valuemax="100" style="width: 50%"></div></div>'
}
else:
try:
msg = json.loads(m.group(1))
if msg.get('type') in ['getval']:
mid = msg.get('id')
mval = msg.get('val')
muser = msg.get('user')
if muser is None:
muser = ic.user.pk
if mid is None or mval is None:
logger.warning(
f"{con['id'][:12]}: getval: broken msg: > {item[1][:120]}"
)
continue
if mid in ain:
continue
try:
u = User.objects.get(pk=muser)
except User.DoesNotExist:
logger.warning(
f"{con['id'][:12]}: getval: unknown user {muser}"
)
continue
arg = pathlib.Path(mval)
ain[mid] = stream_enum(
mid, await
my_stream.arg_stream(
ic, u, arg))
restart = True
break
elif msg['type'] in ['error']:
yield msg
break
else:
yield msg
except json.JSONDecodeError as e:
logger.warning(
f"{con['id'][:12]}: broken msg: {e!s}"
)
continue
else:
out.append(item[1])
yield {'type': 'stdout', 'val': "\n".join(out)}
elif item[0] == 'err':
logger.debug(f"{con['id'][:12]}: !! " +
' '.join(str(item[1]).split())[:120])
yield {'type': 'error', 'val': item[1]}
break
else:
await send_item(ws, con, aout, item)
except GeneratorExit:
pass
finally:
logger.debug(f"{con['id'][:12]}: delete")
try:
await con.kill()
await con.delete()
pass
except Exception:
pass
ws.close()
except MyException as e:
yield {'type': 'error', 'val': f"⚠ {e!s} ⚠"}
except aiodocker.exceptions.DockerError as e:
yield {'type': 'error', 'val': f"⚠ {e!s} ⚠"}
except Exception as e:
logger.exception(e)
yield {'type': 'error', 'val': f"⚠ {e!s} ⚠"}
finally:
if dapi:
await dapi.close()
async def amerge(*agens) -> AsyncGenerator[Any, None]:
"""Thin wrapper around aiostream.stream.merge."""
xs = stream.merge(*agens)
async with xs.stream() as streamer:
async for x in streamer:
yield x
async def rules_from_kubernetes(ctx) -> AsyncIterable[List[Rule]]:
"""This generator continuously runs, watching Kubernetes for
certain resources, consuming changes, and determining which
snapshot rules have been defined.
Every value it returns is a list of `Rule` objects, a complete
set of snapshot rules defined at this point in time. Every set
of rule objects replaces the previous one.
"""
# These are rules that we are ready to "run".
rules = {}
# These are resources that we know we have to recheck, because
# they will become rules pending a resource creation. For example:
# A `SnapshotRule` resource points to volume claim. However, this
# volume claim is not yet bound. Once Kubernetes creates the volume,
# we will notify us about creating a `PersistentVolume` and updating
# a `PersistentVolumeClaim`. It will not, however, send us an
# update for the `SnapshotRule` - where the rule is actually
# defined. We thus have to link the rule to the volume.
pending_rules: Dict[Tuple, pykube.objects.APIObject] = {}
_logger.debug('volume-events.watch')
merged_stream = stream.merge(
watch_resources(ctx, pykube.objects.PersistentVolume, delay=0),
watch_resources(ctx, pykube.objects.PersistentVolumeClaim, delay=2),
watch_resources(ctx, SnapshotRule, delay=3, allow_missing=True)
)
iterable: AsyncIterable[_WatchEvent] = merged_stream.stream()
async with iterable as merged_events:
async for event in merged_events:
_log = _logger.bind(
event_type=event.type,
event_object=event.object.obj,
)
_log.info(
events.VolumeEvent.RECEIVED,
key_hints=[
'event_type',
'event_object.metadata.name',
],
)
# This is how we uniquely identify the rule. This is important
# such that when an object is deleted, we delete the correct
# rule.
key_by = (
event.object.kind,
event.object.namespace,
event.object.name
)
events_to_process = [
(event.type, key_by, event.object)
]
# Is there some other object that was depending on *this*
# object?
if key_by in pending_rules:
depending_object_key, depending_object = pending_rules.pop(key_by)
if event.type != 'DELETED':
events_to_process.append(('MODIFIED', depending_object_key, depending_object))
for (event_type, rule_key, resource) in events_to_process:
# TODO: there is probably a bug here, where for rule deletion
# we should not have to first successfully build the rule; the key
# is enough to delete it. Same with a modification that causes
# the rule to break; we should remove it until fixed.
try:
if isinstance(resource, SnapshotRule):
rule = await rule_from_snapshotrule(ctx, resource)
elif isinstance(resource, pykube.objects.PersistentVolumeClaim):
rule = await rule_from_persistent_volume_claim(ctx, resource)
elif isinstance(resource, pykube.objects.PersistentVolume):
rule = await rule_from_persistent_volume(ctx, resource)
else:
raise RuntimeError(f'{resource} is not supported.')
except RuleDependsOn as exc:
# We have to remember this so that when we get an
# update for the dependency that we lack here, we
# can process this resource once more.
pending_rules[(
exc.data['kind'],
exc.data['namespace'],
exc.data['name'],
)] = (rule_key, resource)
continue
if not rule:
continue
_log = _log.bind(
rule=rule
)
if event_type == 'ADDED' or event_type == 'MODIFIED':
if rule:
if event_type == 'ADDED' or rule_key not in rules:
_log.info(
events.Rule.ADDED,
key_hints=['rule.name']
)
else:
_log.info(
events.Rule.UPDATED,
key_hints=['rule.name']
)
rules[rule_key] = rule
else:
if rule_key in rules:
_log.info(
events.Rule.REMOVED,
key_hints=['volume_name']
)
rules.pop(rule_key)
elif event_type == 'DELETED':
if rule_key in rules:
_log.info(
events.Rule.REMOVED,
key_hints=['volume_name']
)
rules.pop(rule_key)
else:
_log.warning('Unhandled event')
# We usually have duplicate disks within in `rules`,
# which is indexed by resource kind. One reason is we
# watching both PVCs and PVs, and a PVC/PV pair resolve
# to the same disk. It is also possible that custom rules
# the user defined contain duplicates. Let's make sure
# we only have one rule for every disk. Note that which
# one we pick is undefined.
#
# In the (internal) case of PV/PVC pairs it does't matter,
# since our code is written thus: The rule always references
# the volume, and we always check the volume, then the claim
# for deltas. The behaviour for this case is well-defined.
unique_rules = {rule.disk: rule for rule in rules.values()}.values()
# TODO: Log in a different place, in a debounced way
#_logger.info('sync-get-rules.yield', rule_count=len(unique_rules))
yield list(unique_rules)
_logger.debug('sync-get-rules.done')
async def rules_from_volumes(ctx):
rules = {}
_logger.debug('volume-events.watch')
merged_stream = stream.merge(
watch_resources(ctx, pykube.objects.PersistentVolume),
watch_resources(ctx, pykube.objects.PersistentVolumeClaim)
)
async with merged_stream.stream() as merged_events:
async for event in merged_events:
_log_event = _logger.bind(
event_type=event.type,
event_object=event.object.obj,
)
_log_event.info(
events.VolumeEvent.RECEIVED,
key_hints=[
'event_type',
'event_object.metadata.name',
],
)
try:
volume = await volume_from_resource(ctx, event.object)
except VolumeNotFound:
_log_event.exception(
events.Volume.NOT_FOUND,
key_hints=[
'event_type',
'event_object.metadata.name',
],
)
continue
volume_name = volume.name
_log = _logger.new(
volume_name=volume_name,
volume_event_type=event.type,
volume=volume.obj,
)
if event.type == 'ADDED' or event.type == 'MODIFIED':
rule = None
try:
rule = await rule_from_pv(
ctx,
volume,
ctx.config.get('deltas_annotation_key'),
use_claim_name=ctx.config.get('use_claim_name'))
except AnnotationNotFound as exc:
_log.info(
events.Annotation.NOT_FOUND,
key_hints=['volume.metadata.name'],
exc_info=exc,
)
except AnnotationError:
_log.exception(
events.Annotation.ERROR,
key_hints=['volume.metadata.name'],
)
except UnsupportedVolume as exc:
_log.info(
events.Volume.UNSUPPORTED,
key_hints=['volume.metadata.name'],
exc_info=exc,
)
_log = _log.bind(
rule=rule
)
if rule:
if event.type == 'ADDED' or volume_name not in rules:
_log.info(
events.Rule.ADDED,
key_hints=['rule.name']
)
else:
_log.info(
events.Rule.UPDATED,
key_hints=['rule.name']
)
rules[volume_name] = rule
else:
if volume_name in rules:
_log.info(
events.Rule.REMOVED,
key_hints=['volume_name']
)
rules.pop(volume_name)
elif event.type == 'DELETED':
if volume_name in rules:
_log.info(
events.Rule.REMOVED,
key_hints=['volume_name']
)
rules.pop(volume_name)
else:
_log.warning('Unhandled event')
yield list(rules.values())
_logger.debug('sync-get-rules.done')
async def data_stream_async(url,
files,
columns=None,
map_func=None,
reduce_func=None,
initializer=None,
producer_num=2,
data_handler_num=2,
executor_type='process'):
data_type = files[0].split('.')[-1]
columns_mapping = get_parquet_mapping() if data_type == 'parquet' else None
if columns:
c = [k for k, v in columns_mapping.items() if v in columns]
c = c + list(
set(columns).difference(set(list(columns_mapping.values()))))
else:
c = columns
global pbar_handler
pbar_handler = PbarHandler(len(files))
global executor
if executor_type == 'process':
executor = ProcessPoolExecutor(max_workers=data_handler_num)
elif executor_type == 'thread':
executor = ThreadPoolExecutor(max_workers=data_handler_num)
if map_func:
map_task = partial(data_load, func=map_func)
else:
map_task = data_load
pbar_handler.register(mapper, len(files))
if reduce_func:
reduce_task = partial(reducer, func=reduce_func)
else:
reduce_task = partial(reducer, func=concat)
initializer = pd.DataFrame()
file_streams = [
stream.preserve(file_stream(files[i::producer_num], url))
for i in range(producer_num)
]
file_list = []
aws = (stream.merge(*file_streams)
| pipe.map(
async_(lambda x: mapper(x, map_task, c, columns_mapping)),
task_limit=data_handler_num)
| pipe.map(async_(lambda x: file_list.append(x[0]) or x[1]),
task_limit=data_handler_num))
if reduce_func:
pbar_handler.register(reducer, len(files) - 1)
rs = stream.reduce(aws, async_(reduce_task), initializer)
reduced = await stream.takelast(rs, 1)
return reduced
else:
data_list = await asyncio.gather(stream.list(aws))
data_list = data_list[0]
tmp_list = list(zip(file_list, data_list))
tmp_list = sorted(tmp_list, key=lambda pair: files.index(pair[0]))
if map_func:
return tmp_list
else:
return pd.concat(list(map(lambda pair: pair[1], tmp_list)), axis=0)
async def run(self):
"""run the engine"""
# setup future queue
self._queued_events = deque()
self._queued_targeted_events = deque()
# await all connections
await asyncio.gather(*(asyncio.create_task(exch.connect())
for exch in self.exchanges))
await asyncio.gather(*(asyncio.create_task(exch.instruments())
for exch in self.exchanges))
# send start event to all callbacks
await self.processEvent(Event(type=EventType.START, target=None))
# **************** #
# Main event loop
# **************** #
async with merge(
*(exch.tick() for exch in self.exchanges + [self]
if inspect.isasyncgenfunction(exch.tick))).stream() as stream:
# stream through all events
async for event in stream:
# TODO move out of critical path
if self._offline():
# inject periodics
# TODO optimize
# Manager should keep track of the intervals for its periodics,
# then we don't need to go through seconds (which is what the
# live engine's `tick` method does below). Instead we can just
# calculate exactly the intervals
if (self._latest != datetime.fromtimestamp(0)
and hasattr(event, "target")
and hasattr(event.target, "timestamp")):
# TODO in progress optimization
intervals = self.manager.periodicIntervals()
# not the first tick
for _ in range(
int((event.target.timestamp -
self._latest).total_seconds() /
intervals)):
self._latest = self._latest + timedelta(seconds=1 *
intervals)
if any(
p.expires(self._latest)
for p in self.manager.periodics()):
await asyncio.gather(
*(asyncio.create_task(
p.execute(self._latest))
for p in self.manager.periodics()
if p.expires(self._latest)))
# tick exchange event to handlers
await self.processEvent(event)
# TODO move out of critical path
if self._offline():
# use time of last event
self._latest = (event.target.timestamp
if hasattr(event, "target")
and hasattr(event.target, "timestamp") else
self._latest)
else:
# use now
self._latest = datetime.now()
# process any secondary events
while self._queued_events:
event = self._queued_events.popleft()
await self.processEvent(event)
# process any secondary callback-targeted events (e.g. order fills)
# these need to route to a specific callback,
# rather than all callbacks
while self._queued_targeted_events:
strat, event = self._queued_targeted_events.popleft()
# send to the generating strategy
await self.processEvent(event, strat)
# process any periodics
await asyncio.gather(
*(asyncio.create_task(p.execute(self._latest))
for p in self.manager.periodics()
if p.expires(self._latest)))
# Before engine shutdown, send an exit event
await self.processEvent(Event(type=EventType.EXIT, target=None))
async def from_streams(base_reader):
'''
Asyncio coroutine task to read from all streams &
detect and then execute the command based on the confirmation tap
'''
try:
# Wait for half a second before processing the events
await asyncio.sleep(0.5)
# Grab the touchpad to draw gesture until this coroutine is cancelled
base_reader.grab()
# Init all the readers
x_movement_reader = Reader(base_reader)
y_movement_reader = Reader(base_reader)
tap_detector_reader = Reader(base_reader)
# Reload gesture command map
reload_config()
# Store the received coordinates
coordinates_set = []
start_time = end_time = 0
# Zip the X and Y axis events for clarity.
# It is processed separtely though when sanitizing the input
zip_xy = ziplatest(y_movement(x_movement_reader),
x_movement(y_movement_reader))
# Read the tap events as well to indicate
# the start and end of gesture drawing
merge_tap_xy = merge(zip_xy,
tap_detector(tap_detector_reader))
async with merge_tap_xy.stream() as merged:
async for event in merged:
# The zip_xy events are in the form of tuples
# while the tap events are evdev event objects
if not isinstance(event, tuple):
if event.value == 1:
start_time = event.timestamp()
elif event.value == 0:
end_time = event.timestamp()
# If the draw is too short, ignore and reset
# cause it's not meaningful
if (end_time - start_time) < 0.3:
coordinates_set = []
continue
detected_gesture = sanitize_and_notify(coordinates_set)
# print(f'Detected gesture :- {detected_gesture}')
coordinates_set = []
if detected_gesture is None:
continue
# If gesture detected then wait for a confirmation tap
tapped, event = await confirmation_tap(base_reader)
if tapped:
notify(f"Confirmed. Running- {detected_gesture}")
execute_command(detected_gesture)
else:
notify("Clearing gestures")
else:
coordinates_set.append(event)
except asyncio.CancelledError:
# Exit all readers from the base_reader once they are done
x_movement_reader.exit()
y_movement_reader.exit()
tap_detector_reader.exit()
# Ungrab and yield control of touchpad to the user
base_reader.ungrab()
