class Waiter():
def __init__(self, event_loop, timeout_msec=-1):
self._timeout_msec = timeout_msec
self._event = Subject()
self._wait_exit = False
self.result_ob = None
self._event_loop = event_loop
self._timer = None
self.scheduler = AsyncIOScheduler(loop=event_loop)
self.done = asyncio.Future()
print('done init')
def set_timeout(self, timeout_msec):
self._timeout_msec = timeout_msec
def _timeout_callback(self):
print('called _timeout_callback')
self._event.on_next('timeout')
async def wait(self):
if self._timeout_msec > 0:
self._timer = threading.Timer(self._timeout_msec / 1000,
self._timeout_callback)
self._timer.start()
self._wait_exit = False
def cb_next(data):
print('received data : {}'.format(data))
if self._timer:
self._timer.cancel()
self._wait_exit = True
self.done.set_result(data)
def cb_completed():
print('received completed')
self.done.set_result('done')
#self._event.subscribe(on_next=cb_next, on_completed=cb_completed, scheduler=self.scheduler)
self._event.subscribe(on_next=cb_next, scheduler=self.scheduler)
result = await self.done
print('done wait')
return result
def end(self, msg):
self._event.on_next(msg or 'success')
self._event.on_completed()
#self.result_ob.dispose()
def set_next(self, msg):
self._event.on_next(msg)
def test_base_on_completed(self):
source = Subject()
source.pipe(
trace_observable(prefix='foo',
date=datetime.datetime(year=2018, month=8,
day=3))).subscribe()
source.on_completed()
self.assertEqual(
'2018-08-03 00:00:00:foo - on_subscribe\n'
'2018-08-03 00:00:00:foo - on_completed\n'
'2018-08-03 00:00:00:foo - dispose',
self.out.getvalue().strip())
def action(scheduler, state=None):
s = None
if is_shift:
s = Subject()
q.append(s)
observer.on_next(add_ref(s, ref_count_disposable))
if is_span:
s = q.pop(0)
s.on_completed()
create_timer()
class PlotSaveHandler(rx.core.Observer):
def __init__(self, cfg: CfgNode, cfg_dir="."):
super(PlotSaveHandler, self).__init__()
self.base_path = utils.abs_or_offset_from(cfg.output.base_path, cfg_dir)
os.makedirs(self.base_path, exist_ok=True)
os.makedirs(os.path.join(self.base_path, "weights"), exist_ok=True)
self.e = []
self.train_loss = []
self.val_loss = []
self.plot_sub = Subject()
def on_next(self, value) -> None:
e = value["epoch"]
train_time = value["train_time"]
val_time = value["valid_time"]
train_loss = value["train_loss"]
val_loss = value["valid_loss"]
net: RamanAINetwork = value["net"]
print(f"Epoch:{e}, train_loss={train_loss}, val_loss={val_loss}, train_time={train_time}, val_time={val_time}")
self.e.append(e)
self.train_loss.append(train_loss)
self.val_loss.append(val_loss)
# save
torch.save(net, os.path.join(self.base_path, "weights", f"weights_{e}.pkl"))
self.plot_sub.on_next({
"epoch": self.e,
"train": self.train_loss,
"valid": self.val_loss,
"test_output": value["test_output"],
})
def on_error(self, error: Exception) -> None:
raise error
def on_completed(self) -> None:
print("Completed")
with open(os.path.join(self.base_path, f"loss.csv"), "w", newline='') as f:
writer = csv.writer(f)
writer.writerow(["Epoch", "Train", "Valid"])
for e, t, v, in zip(self.e, self.train_loss, self.val_loss):
writer.writerow([e, t, v])
self.plot_sub.on_completed()
def main():
key="z"
subject_test = Subject()
initKeyMonitoring(subject_test)
subject_test.on_next(key)
print("here")
time.sleep(2)
subject_test.on_next(key)
print("here")
time.sleep(2)
subject_test.on_next(key)
print("here")
time.sleep(2)
subject_test.on_completed()
def main():
key = "space"
keyTwo = "a"
subject_test = Subject()
initKeyMonitoring(subject_test)
subject_test.on_next(keyTwo)
print("here")
time.sleep(1)
subject_test.on_next(key)
print("here")
time.sleep(1)
subject_test.on_next(keyTwo)
print("here")
time.sleep(1)
subject_test.on_completed()
class MidiReceiver(threading.Thread):
def __init__(self, port):
super().__init__(daemon=True)
self.stop_flag = False
self.message = Subject()
self.clock_found = Subject()
self.clock = Subject()
self.clock_lost = Subject()
self.clock_set = Subject()
self.clocks_received = None
self.last_clock_time = None
self.port_name = port
self.port = mido.open_input(port)
def run(self):
for message in self.port:
if message.type == 'songpos':
self.clock_set.on_next(message.pos * 24)
print('{}: MIDI clock set: {}'.format(self.port_name, message.pos))
elif message.type == 'clock':
if not self.last_clock_time:
self.clock_found.on_next(None)
self.clocks_received = 0
print('{}: MIDI clock found'.format(self.port_name))
self.clocks_received += 1
self.clock.on_next(None)
self.last_clock_time = time.time()
else:
self.message.on_next(message)
print('{} -> {}'.format(self.port_name, message))
if self.stop_flag:
break
if self.last_clock_time and time.time() - self.last_clock_time > 1:
self.last_clock_time = None
self.clock_lost.on_next(None)
self.clocks_received = None
print('{}: MIDI clock lost'.format(self.port_name))
def stop(self):
self.stop_flag = True
self.message.on_completed()
if self.last_clock_time:
self.clock_lost.on_next(None)
class BackPressurePublisher(DefaultPublisherSubscription):
def __init__(self, wrapped_observable: Observable):
self._wrapped_observable = wrapped_observable
self._feedback = None
def subscribe(self, subscriber: Subscriber):
super().subscribe(subscriber)
self._feedback = Subject()
async_iterator = observable_to_async_event_generator(
self._wrapped_observable).__aiter__()
from_aiter(async_iterator,
self._feedback).subscribe(SubscriberAdapter(subscriber))
def request(self, n: int):
for i in range(n):
self._feedback.on_next(True)
def cancel(self):
self._feedback.on_completed()
class DeviceSniffer:
def __init__(self, user_querier: UserQuerier, interface="mon0"):
self.async_sniffer = AsyncSniffer(prn=self.handle_packet,
store=False,
iface=interface,
monitor=True)
self.device_dectect_stream = Subject()
self.user_querier = user_querier
def __del__(self):
self.device_dectect_stream.on_completed()
@staticmethod
def is_probe_request(packet: Packet):
return packet.type == 0 and packet.subtype == 4
def handle_packet(self, packet: Packet):
if not DeviceSniffer.is_probe_request(packet):
return
try:
target_ssid = packet.getlayer(Dot11Elt).getfieldval("info").decode(
"utf-8")
if len(target_ssid) == 0:
return
source_mac_addr = packet.addr2.upper()
userid = self.user_querier.get_userid(target_ssid, source_mac_addr)
if userid is not None:
self.device_dectect_stream.on_next(userid)
except Exception as err:
self.device_dectect_stream.on_error(err)
def get_observable(self) -> Subject:
return self.device_dectect_stream
def start(self):
self.async_sniffer.start()
def stop(self):
self.async_sniffer.stop()
class Lines:
def __init__(self, chat_config):
self.lines = []
self.added = Subject()
self.removed = Subject()
self._config = chat_config
@property
def _max_lines(self):
return self._config["max_messages_in_channel"]
@property
def _remove_batch(self):
return self._config["message_trim_batch_size"]
def __getitem__(self, n):
return self.lines[n]
def __iter__(self):
return iter(self.lines)
def __len__(self):
return len(self.lines)
def add(self, line):
self.lines.append(line)
self.added.on_next(line)
self._check_trim()
def clear(self):
ll = len(self)
self.lines.clear()
self.removed.on_next(ll)
def _check_trim(self):
if len(self) > self._max_lines:
self.lines = self.lines[self._remove_batch:]
self.removed.on_next(self._remove_batch)
def complete(self):
self.added.on_completed()
self.removed.on_completed()
class ModelSet(Mapping):
def __init__(self):
Mapping.__init__(self)
self._items = {}
self.added = Subject()
self.removed = Subject()
self.cleared = Subject()
def __getitem__(self, item):
return self._items[item]
def __len__(self):
return len(self._items)
def __iter__(self):
return iter(self._items)
def add(self, item):
self._items[item.id_key] = item
def remove(self, item):
del self._items[item.id_key]
def clear(self):
self._items.clear()
def complete(self):
self.added.on_completed()
self.removed.on_completed()
self.cleared.on_completed()
class Inquiry:
def __init__(self, target: InstanceReference, timeout = 10):
self.id = uuid.uuid4().bytes
self.target = target
self.time = time.time()
self.complete = Subject()
self.timer = Timer(timeout, self.__timeout)
self.timer.start()
def response_received(self) -> float:
self.timer.cancel()
delay = time.time() - self.time
self.complete.on_next(delay)
self.complete.on_completed()
return delay
def __timeout(self):
Log.warn("Inquiry timed out")
self.complete.on_error(TimeoutError("The inquiry timed out."))
class Player(ModelItem):
def __init__(self, current_player_game, login):
ModelItem.__init__(self)
self._current_player_game = current_player_game
self.login = login
self._add_obs("id", 0)
self._add_obs("global_rating", (1500, 500))
self._add_obs("ladder_rating", (1500, 500))
self._add_obs("number_of_games", 0)
self._add_obs("avatar")
self._add_obs("country")
self._add_obs("clan")
self._add_obs("league")
self.obs_game = Subject()
# The game this player is currently playing.
@property
def game(self):
return self._current_player_game.get(self.id_key)
def complete(self):
ModelItem.complete(self)
self.obs_game.on_completed()
@property
def rating_estimate(self):
"Conservative estimate of the players global trueskill rating."
return max(0, (self.global_rating[0] - 3 * self.global_rating[1]))
# Unfortunately, games refer to players by login.
# I have no idea how this plays with renaming, hopefully the server doesn't
# change player names mid-session
@property
def id_key(self):
return self.login
# 操作数据流
print('求所有偶数')
some_data = rx.of(1, 2, 3, 4, 5, 6, 7, 8)
some_data2 = rx.from_iterable(range(10, 20))
some_data.pipe(
op.merge(some_data2),
op.filter(lambda i: i % 2 == 0),
# op.map(lambda i: i * 2)
).subscribe(lambda i: print(i))
# debounce操作符，仅在时间间隔之外的可以发射
print('防止重复发送')
ob = Subject()
ob.pipe(
op.throttle_first(3)
# op.debounce(3)
).subscribe(
on_next=lambda i: print(i),
on_completed=lambda: print('Completed')
)
print('press enter to print, press other key to exit')
while True:
s = input()
if s == '':
ob.on_next(datetime.datetime.now().time())
else:
ob.on_completed()
break
import rx
import rx.operators as ops
from rx.subject import Subject
first = Subject()
second = Subject()
first.pipe(ops.amb(second)).subscribe(
on_next=lambda i: print("on_next {}".format(i)),
on_error=lambda e: print("on_error: {}".format(e)),
on_completed=lambda: print("on_completed")
)
first.on_next(1)
second.on_next(2)
first.on_completed()
class WriteApi(AbstractClient):
def __init__(
self, influxdb_client,
write_options: WriteOptions = WriteOptions()) -> None:
self._influxdb_client = influxdb_client
self._write_service = WriteService(influxdb_client.api_client)
self._write_options = write_options
if self._write_options.write_type is WriteType.batching:
# Define Subject that listen incoming data and produces writes into InfluxDB
self._subject = Subject()
# Define a scheduler that is used for processing incoming data - default singleton
observable = self._subject.pipe(
ops.observe_on(self._write_options.write_scheduler))
self._disposable = observable \
.pipe( # Split incoming data to windows by batch_size or flush_interval
ops.window_with_time_or_count(count=write_options.batch_size,
timespan=timedelta(milliseconds=write_options.flush_interval)),
# Map incoming batch window in groups defined by 'organization', 'bucket' and 'precision'
ops.flat_map(lambda v: _window_to_group(v)),
# Write data into InfluxDB (possibility to retry if its fail)
ops.map(mapper=lambda batch: self._retryable(data=batch, delay=self._jitter_delay())), #
ops.merge_all()) \
.subscribe(self._on_next, self._on_error, self._on_complete)
else:
self._subject = None
self._disposable = None
def write(
self,
bucket: str,
org: str,
record: Union[str, List['str'], Point, List['Point'], dict,
List['dict'], bytes, List['bytes'], Observable],
write_precision: WritePrecision = DEFAULT_WRITE_PRECISION) -> None:
"""
Writes time-series data into influxdb.
:param str org: specifies the destination organization for writes; take either the ID or Name interchangeably; if both orgID and org are specified, org takes precedence. (required)
:param str bucket: specifies the destination bucket for writes (required)
:param WritePrecision write_precision: specifies the precision for the unix timestamps within the body line-protocol
:param record: Points, line protocol, RxPY Observable to write
"""
if self._write_options.write_type is WriteType.batching:
return self._write_batching(bucket, org, record, write_precision)
final_string = self._serialize(record, write_precision)
_async_req = True if self._write_options.write_type == WriteType.asynchronous else False
return self._post_write(_async_req, bucket, org, final_string,
write_precision)
def flush(self):
# TODO
pass
def __del__(self):
if self._subject:
self._subject.on_completed()
self._subject.dispose()
self._subject = None
# Wait for finish writing
while not self._disposable.is_disposed:
sleep(0.1)
if self._disposable:
self._disposable = None
pass
def _serialize(self, record, write_precision) -> bytes:
_result = b''
if isinstance(record, bytes):
_result = record
elif isinstance(record, str):
_result = record.encode("utf-8")
elif isinstance(record, Point):
_result = self._serialize(record.to_line_protocol(),
write_precision=write_precision)
elif isinstance(record, dict):
_result = self._serialize(Point.from_dict(
record, write_precision=write_precision),
write_precision=write_precision)
elif isinstance(record, list):
_result = b'\n'.join([
self._serialize(item, write_precision=write_precision)
for item in record
])
return _result
def _write_batching(self,
bucket,
org,
data,
precision=DEFAULT_WRITE_PRECISION):
_key = _BatchItemKey(bucket, org, precision)
if isinstance(data, bytes):
self._subject.on_next(_BatchItem(key=_key, data=data))
elif isinstance(data, str):
self._write_batching(bucket, org, data.encode("utf-8"), precision)
elif isinstance(data, Point):
self._write_batching(bucket, org, data.to_line_protocol(),
precision)
elif isinstance(data, dict):
self._write_batching(
bucket, org, Point.from_dict(data, write_precision=precision),
precision)
elif isinstance(data, list):
for item in data:
self._write_batching(bucket, org, item, precision)
elif isinstance(data, Observable):
data.subscribe(
lambda it: self._write_batching(bucket, org, it, precision))
pass
return None
def _http(self, batch_item: _BatchItem):
logger.debug("http post to: %s", batch_item)
self._post_write(False, batch_item.key.bucket, batch_item.key.org,
batch_item.data, batch_item.key.precision)
return _BatchResponse(data=batch_item)
def _post_write(self, _async_req, bucket, org, body, precision):
return self._write_service.post_write(
org=org,
bucket=bucket,
body=body,
precision=precision,
async_req=_async_req,
content_encoding="identity",
content_type="text/plain; charset=utf-8")
def _retryable(self, data: str, delay: timedelta):
return rx.of(data).pipe(
# use delay if its specified
ops.delay(duetime=delay,
scheduler=self._write_options.write_scheduler),
# invoke http call
ops.map(lambda x: self._http(x)),
# if there is an error than retry
ops.catch(handler=lambda exception, source: self._retry_handler(
exception, source, data)),
)
def _retry_handler(self, exception, source, data):
if isinstance(exception, ApiException):
if exception.status == 429 or exception.status == 503:
_delay = self._jitter_delay() + timedelta(
milliseconds=self._write_options.retry_interval)
return self._retryable(data, delay=_delay)
return rx.just(_BatchResponse(exception=exception, data=data))
def _jitter_delay(self):
return timedelta(milliseconds=random() *
self._write_options.jitter_interval)
@staticmethod
def _on_next(response: _BatchResponse):
if response.exception:
logger.error(
"The batch item wasn't processed successfully because: %s",
response.exception)
else:
logger.debug("The batch item: %s was processed successfully.",
response)
@staticmethod
def _on_error(ex):
logger.error("unexpected error during batching: %s", ex)
def _on_complete(self):
self._disposable.dispose()
logger.info("the batching processor was disposed")
class Context:
def __init__(self, config_path: str):
deepkit.globals.last_context = self
self.log_lock = Lock()
self.defined_metrics = {}
self.log_subject = Subject()
self.metric_subject = Subject()
self.speed_report_subject = Subject()
self.client = deepkit.client.Client(config_path)
self.wait_for_connect()
atexit.register(self.shutdown)
self.last_iteration_time = 0
self.last_batch_time = 0
self.job_iteration = 0
self.job_iterations = 0
self.seconds_per_iteration = 0
self.seconds_per_iterations = []
self.debugger_controller = None
if deepkit.utils.in_self_execution():
self.job_controller = JobController()
self.debugger_controller = JobDebuggerController()
def on_connect(connected):
if connected:
if deepkit.utils.in_self_execution():
asyncio.run_coroutine_threadsafe(
self.client.register_controller(
'job/' + self.client.job_id, self.job_controller),
self.client.loop)
asyncio.run_coroutine_threadsafe(
self.client.register_controller(
'job/' + self.client.job_id + '/debugger',
self.debugger_controller), self.client.loop)
self.client.connected.subscribe(on_connect)
def on_log(data: List):
if len(data) == 0: return
packed = ''
for d in data:
packed += d
self.client.job_action('log', ['main_0', packed])
self.log_subject.pipe(buffer(interval(1))).subscribe(on_log)
if len(deepkit.globals.last_logs.getvalue()) > 0:
self.log_subject.on_next(deepkit.globals.last_logs.getvalue())
def on_metric(data: List):
if len(data) == 0: return
packed = {}
for d in data:
if d['id'] not in packed:
packed[d['id']] = []
packed[d['id']].append(d['row'])
for i, v in packed.items():
self.client.job_action('channelData', [i, v])
self.metric_subject.pipe(buffer(interval(1))).subscribe(on_metric)
def on_speed_report(rows):
# only save latest value, each second
if len(rows) == 0: return
self.client.job_action('streamJsonFile',
['.deepkit/speed.csv', [rows[-1]]])
self.speed_report_subject.pipe(buffer(
interval(1))).subscribe(on_speed_report)
p = psutil.Process()
self.client.job_action('streamJsonFile', [
'.deepkit/hardware/main_0.csv',
[[
'time', 'cpu', 'memory', 'network_rx', 'network_tx',
'block_write', 'block_read'
]]
])
def on_hardware_metrics(dummy):
net = psutil.net_io_counters()
disk = psutil.disk_io_counters()
data = [
time.time(),
(p.cpu_percent(interval=None) / 100) / psutil.cpu_count(),
p.memory_percent() / 100,
net.bytes_recv,
net.bytes_sent,
disk.write_bytes,
disk.read_bytes,
]
self.client.job_action('streamJsonFile',
['.deepkit/hardware/main_0.csv', [data]])
interval(1).subscribe(on_hardware_metrics)
def wait_for_connect(self):
async def wait():
await self.client.connecting
asyncio.run_coroutine_threadsafe(wait(), self.client.loop).result()
def shutdown(self):
self.metric_subject.on_completed()
self.log_subject.on_completed()
self.client.shutdown()
def epoch(self, current: int, total: Optional[int]):
self.iteration(current, total)
def iteration(self, current: int, total: Optional[int]):
self.job_iteration = current
if total:
self.job_iterations = total
now = time.time()
if self.last_iteration_time:
self.seconds_per_iterations.append({
'diff': now - self.last_iteration_time,
'when': now,
})
self.last_iteration_time = now
self.last_batch_time = now
# remove all older than twenty seconds
self.seconds_per_iterations = [
x for x in self.seconds_per_iterations if (now - x['when']) < 20
]
self.seconds_per_iterations = self.seconds_per_iterations[-30:]
if len(self.seconds_per_iterations) > 0:
diffs = [x['diff'] for x in self.seconds_per_iterations]
self.seconds_per_iteration = sum(diffs) / len(diffs)
if self.seconds_per_iteration:
self.client.patch('secondsPerIteration',
self.seconds_per_iteration)
self.client.patch('iteration', self.job_iteration)
if total:
self.client.patch('iterations', self.job_iterations)
iterations_left = self.job_iterations - self.job_iteration
if iterations_left > 0:
self.client.patch('eta',
self.seconds_per_iteration * iterations_left)
else:
self.client.patch('eta', 0)
def step(self, current: int, total: int = None, size: int = None):
self.client.patch('step', current)
now = time.time()
x = self.job_iteration + (current / total)
speed_per_second = size / (
now - self.last_batch_time) if self.last_batch_time else size
if self.last_batch_time:
self.seconds_per_iterations.append({
'diff': (now - self.last_batch_time) * total,
'when':
now
})
# remove all older than twenty seconds
self.seconds_per_iterations = [
x for x in self.seconds_per_iterations if (now - x['when']) < 20
]
self.seconds_per_iterations = self.seconds_per_iterations[-30:]
if len(self.seconds_per_iterations) > 0:
diffs = [x['diff'] for x in self.seconds_per_iterations]
self.seconds_per_iteration = sum(diffs) / len(diffs)
iterations_left = self.job_iterations - self.job_iteration
self.client.patch('eta',
self.seconds_per_iteration * iterations_left)
self.last_batch_time = now
if self.seconds_per_iteration:
self.client.patch('secondsPerIteration',
self.seconds_per_iteration)
self.client.patch('speed', speed_per_second)
self.speed_report_subject.on_next([x, now, speed_per_second])
if total:
self.client.patch('steps', total)
def set_title(self, s: str):
self.client.patch('title', s)
def set_info(self, name: str, value: any):
self.client.patch('infos.' + name, value)
def set_description(self, description: any):
self.client.patch('description', description)
def add_tag(self, tag: str):
self.client.job_action('addTag', [tag])
def rm_tag(self, tag: str):
self.client.job_action('rmTag', [tag])
def set_parameter(self, name: str, value: any):
self.client.patch('config.parameters.' + name, value)
def define_metric(self, name: str, options: dict):
self.defined_metrics[name] = {}
self.client.job_action('defineMetric', [name, options])
def debug_snapshot(self, graph: dict):
self.client.job_action('debugSnapshot', [graph])
def add_file(self, path: str):
self.client.job_action(
'uploadFile',
[path,
base64.b64encode(open(path, 'rb').read()).decode('utf8')])
def add_file_content(self, path: str, content: bytes):
self.client.job_action(
'uploadFile',
[path, base64.b64encode(content).decode('utf8')])
def set_model_graph(self, graph: dict):
self.client.job_action('setModelGraph', [graph])
def metric(self, name: str, x, y):
if name not in self.defined_metrics:
self.define_metric(name, {})
if not isinstance(y, list):
y = [y]
self.metric_subject.on_next({'id': name, 'row': [x, time.time()] + y})
self.client.patch('channels.' + name + '.lastValue', y)
def log(self, s: str):
self.log_subject.on_next(s)
import rx
from rx.subject import Subject, AsyncSubject, BehaviorSubject, ReplaySubject
# Subject同时是Observer和Observable
print('--------Subject---------')
subject = Subject()
subject.on_next(1)
subject.subscribe(lambda i: print(i))
subject.on_next(2)
subject.on_next(3)
subject.on_next(4)
subject.on_completed()
# ReplaySubject会缓存所有值，如果指定参数的话只会缓存最近的几个值
print('--------ReplaySubject---------')
subject = ReplaySubject()
subject.on_next(1)
subject.subscribe(lambda i: print(i))
subject.on_next(2)
subject.on_next(3)
subject.on_next(4)
subject.on_completed()
# BehaviorSubject会缓存上次发射的值，除非Observable已经关闭
print('--------BehaviorSubject---------')
subject = BehaviorSubject(0)
subject.on_next(1)
subject.on_next(2)
subject.subscribe(lambda i: print(i))
subject.on_next(3)
import rx
import rx.operators as ops
from rx.subject import Subject
numbers = Subject()
trigger = Subject()
numbers.pipe(ops.skip_until(trigger)).subscribe(
on_next=lambda i: print("on_next {}".format(i)),
on_error=lambda e: print("on_error: {}".format(e)),
on_completed=lambda: print("on_completed")
)
numbers.on_next(1)
numbers.on_next(2)
trigger.on_next(True)
numbers.on_next(3)
numbers.on_next(4)
numbers.on_completed()
class _RxWriter(object):
success_count = 0
failed_count = 0
raise_retry_exception = 0
def __init__(self) -> None:
self._subject = Subject()
self._scheduler = ThreadPoolScheduler(max_workers=1)
obs = self._subject.pipe(ops.observe_on(self._scheduler))
self._disposable = obs \
.pipe(ops.window_with_time_or_count(count=5, timespan=datetime.timedelta(milliseconds=10_000)),
ops.flat_map(lambda x: self._window_to_group(x)),
ops.map(mapper=lambda x: self._retryable(data=x, delay=self._jitter_delay(jitter_interval=1000))),
ops.merge_all()) \
.subscribe(self._result, self._error, self._on_complete)
pass
def __del__(self):
if self._subject:
self._subject.on_completed()
self._subject.dispose()
self._subject = None
while not self._disposable.is_disposed:
time.sleep(0.1)
if self._disposable:
self._disposable = None
pass
def _window_to_group(self, value):
return value.pipe(
ops.to_iterable(),
ops.map(lambda x: rx.from_iterable(x).pipe(
ops.group_by(_group_by), ops.map(_group_to_batch), ops.merge_all())),
ops.merge_all())
def _retryable(self, data: str, delay: datetime.timedelta):
return rx.of(data).pipe(
ops.delay(duetime=delay, scheduler=self._scheduler),
ops.map(lambda x: self._http(x)),
ops.catch(handler=lambda exception, source: self._retry_handler(exception, source, data)),
)
def _http(self, data: str):
if "gamma" in data:
print('bad request[{}]: {}'.format(current_thread().name, data))
raise Exception('unexpected token: {}'.format(data))
pass
if "alpha" in data:
if self.raise_retry_exception < 2:
self.raise_retry_exception += 1
print('server is temporarily unavailable to accept writes[{}]: {}'.format(current_thread().name, data))
raise Exception('server is temporarily unavailable to accept writes: {}'.format(data))
else:
print("server is OK: {}".format(datetime.datetime.now()))
pass
print("http[" + current_thread().name + "]: " + data)
return _Notification(data=data)
def write(self, data: str):
print("write[" + current_thread().name + "]")
self._subject.on_next(data)
pass
def _result(self, data: _Notification):
print("result[" + current_thread().name + "]: " + str(data))
if data.exception:
self.failed_count += 1
else:
self.success_count += 1
pass
def _error(self, error):
print(error)
def _on_complete(self):
self._disposable.dispose()
print("on complete")
def _jitter_delay(self, jitter_interval=0):
_jitter = datetime.timedelta(milliseconds=random() * jitter_interval)
print('jitter: {}'.format(_jitter))
return _jitter
def _retry_handler(self, exception, source, data):
print('retry_handler: {}, source: {}'.format(exception, source))
if "server is temporarily" in str(exception):
print("RETRY!!!: {}".format(datetime.datetime.now()))
return self._retryable(data, delay=datetime.timedelta(seconds=2))
notification = _Notification(exception=exception, data=data)
return rx.just(notification)
class TimeCertificateBuilder:
def __init__(self,
trust_set: TrustSet,
trust_set_key: VerifyKey,
digest: bytes,
timeout: int = -1):
# Save the paramaters
self.trust_set = trust_set
self.trust_set_key = trust_set_key
self.timeout = timeout
self.signatures: Set[TimeSignature] = set()
self.result = Subject()
self.digest = digest
if (self.timeout == -1):
self.timeout = trust_set.maximum_deviation
self.__timer = None
self.__complete = False
def add_signature(self, signature: TimeSignature):
# Do we have a timer?
if (self.__timer == None):
# No, create it
self.__timer = Timer(self.timeout, self.__finalise)
self.__timer.start()
# Have we timed out?
if (self.__complete):
# Don't do anything
return
# Is the signature valid for this trust set?
if (signature.public_key.encode()
not in self.trust_set.valid_key_data):
# Drop it
return
# Add to the signature set
self.signatures.add(signature)
# Have we got all the signatures?
if (len(self.signatures) == len(self.trust_set.valid_keys)):
# Yes, cancel timer and finalise
self.__timer.cancel()
self.__finalise()
def __finalise(self):
# We are complete
self.__complete = True
# Do we have enough signatures?
if (len(self.signatures) < self.trust_set.required_signatures):
# No, return error
self.result.on_error(
Exception(
"Could not get required number of signatures in time"))
return
# We have enough signatures for a certificate, build it
certificate = TimeCertificate(self.trust_set, self.trust_set_key,
self.signatures)
# Verify the certificate
try:
certificate.validate(self.digest)
except Exception as e:
self.result.on_error(e)
return
# Send to the observer
self.result.on_next(certificate)
self.result.on_completed()
class Context:
def __init__(self, options: ContextOptions = None):
if options is None:
options = ContextOptions()
self.client = deepkit.client.Client(options)
deepkit.globals.last_context = self
self.log_lock = Lock()
self.defined_metrics = {}
self.log_subject = Subject()
self.metric_subject = Subject()
self.speed_report_subject = Subject()
self.shutting_down = False
atexit.register(self.shutdown)
self.wait_for_connect()
self.last_iteration_time = 0
self.last_batch_time = 0
self.job_iteration = 0
self.job_iterations = 0
self.seconds_per_iteration = 0
self.seconds_per_iterations = []
self.debugger_controller = None
if deepkit.utils.in_self_execution():
self.job_controller = JobController()
self.debugger_controller = JobDebuggerController()
def on_connect(connected):
if connected:
if deepkit.utils.in_self_execution():
asyncio.run_coroutine_threadsafe(
self.client.register_controller(
'job/' + self.client.job_id, self.job_controller),
self.client.loop)
asyncio.run_coroutine_threadsafe(
self.client.register_controller(
'job/' + self.client.job_id + '/debugger',
self.debugger_controller), self.client.loop)
self.client.connected.subscribe(on_connect)
def on_metric(data: List):
if len(data) == 0: return
packed = {}
for d in data:
if d['id'] not in packed:
packed[d['id']] = b''
packed[d['id']] += d['row']
for i, v in packed.items():
self.client.job_action(
'channelData', [i, base64.b64encode(v).decode('utf8')])
self.metric_subject.pipe(buffer(interval(1))).subscribe(on_metric)
def on_speed_report(rows):
# only save latest value, each second
if len(rows) == 0: return
self.client.job_action('streamFile', [
'.deepkit/speed.metric',
base64.b64encode(rows[-1]).decode('utf8')
])
self.speed_report_subject.pipe(buffer(
interval(1))).subscribe(on_speed_report)
if deepkit.utils.in_self_execution:
# the CLI handled output logging otherwise
def on_log(data: List):
if len(data) == 0: return
packed = ''
for d in data:
packed += d
self.client.job_action('log', ['main_0', packed])
self.log_subject.pipe(buffer(interval(1))).subscribe(on_log)
if len(deepkit.globals.last_logs.getvalue()) > 0:
self.log_subject.on_next(deepkit.globals.last_logs.getvalue())
if deepkit.utils.in_self_execution:
# the CLI handled output logging otherwise
p = psutil.Process()
def on_hardware_metrics(dummy):
net = psutil.net_io_counters()
disk = psutil.disk_io_counters()
data = struct.pack(
'<BHdHHffff',
1,
0,
time.time(),
int(((p.cpu_percent(interval=None) / 100) /
psutil.cpu_count()) *
65535), # stretch to max precision of uint16
int((p.memory_percent() / 100) *
65535), # stretch to max precision of uint16
float(net.bytes_recv),
float(net.bytes_sent),
float(disk.write_bytes),
float(disk.read_bytes),
)
self.client.job_action('streamFile', [
'.deepkit/hardware/main_0.hardware',
base64.b64encode(data).decode('utf8')
])
interval(1).subscribe(on_hardware_metrics)
def wait_for_connect(self):
async def wait():
await self.client.connecting
asyncio.run_coroutine_threadsafe(wait(), self.client.loop).result()
def shutdown(self):
if self.shutting_down: return
self.shutting_down = True
self.metric_subject.on_completed()
self.log_subject.on_completed()
self.client.shutdown()
def epoch(self, current: int, total: Optional[int]):
self.iteration(current, total)
def iteration(self, current: int, total: Optional[int]):
self.job_iteration = current
if total:
self.job_iterations = total
now = time.time()
if self.last_iteration_time:
self.seconds_per_iterations.append({
'diff': now - self.last_iteration_time,
'when': now,
})
self.last_iteration_time = now
self.last_batch_time = now
# remove all older than twenty seconds
self.seconds_per_iterations = [
x for x in self.seconds_per_iterations if (now - x['when']) < 20
]
self.seconds_per_iterations = self.seconds_per_iterations[-30:]
if len(self.seconds_per_iterations) > 0:
diffs = [x['diff'] for x in self.seconds_per_iterations]
self.seconds_per_iteration = sum(diffs) / len(diffs)
if self.seconds_per_iteration:
self.client.patch('secondsPerIteration',
self.seconds_per_iteration)
self.client.patch('iteration', self.job_iteration)
if total:
self.client.patch('iterations', self.job_iterations)
iterations_left = self.job_iterations - self.job_iteration
if iterations_left > 0:
self.client.patch('eta',
self.seconds_per_iteration * iterations_left)
else:
self.client.patch('eta', 0)
def step(self, current: int, total: int = None, size: int = None):
self.client.patch('step', current)
now = time.time()
x = self.job_iteration + (current / total)
speed_per_second = size / (
now - self.last_batch_time) if self.last_batch_time else size
if self.last_batch_time:
self.seconds_per_iterations.append({
'diff': (now - self.last_batch_time) * total,
'when':
now
})
# remove all older than twenty seconds
self.seconds_per_iterations = [
x for x in self.seconds_per_iterations if (now - x['when']) < 20
]
self.seconds_per_iterations = self.seconds_per_iterations[-30:]
if len(self.seconds_per_iterations) > 0:
diffs = [x['diff'] for x in self.seconds_per_iterations]
self.seconds_per_iteration = sum(diffs) / len(diffs)
iterations_left = self.job_iterations - self.job_iteration
self.client.patch('eta',
self.seconds_per_iteration * iterations_left)
self.last_batch_time = now
if self.seconds_per_iteration:
self.client.patch('secondsPerIteration',
self.seconds_per_iteration)
self.client.patch('speed', speed_per_second)
speed = struct.pack('<Bddd', 1, float(x), now, float(speed_per_second))
self.speed_report_subject.on_next(speed)
if total:
self.client.patch('steps', total)
def set_title(self, s: str):
self.client.patch('title', s)
def set_info(self, name: str, value: any):
self.client.patch('infos.' + str(name), value)
def set_description(self, description: any):
self.client.patch('description', description)
def add_tag(self, tag: str):
self.client.job_action('addTag', [tag])
def rm_tag(self, tag: str):
self.client.job_action('rmTag', [tag])
def set_parameter(self, name: str, value: any):
self.client.patch('config.parameters.' + name, value)
def define_metric(self, name: str, options: dict):
self.defined_metrics[name] = {}
self.client.job_action('defineMetric', [name, options])
def debug_snapshot(self, graph: dict):
self.client.job_action('debugSnapshot', [graph])
def add_file(self, path: str):
self.client.job_action(
'uploadFile',
[path,
base64.b64encode(open(path, 'rb').read()).decode('utf8')])
def add_file_content(self, path: str, content: bytes):
self.client.job_action(
'uploadFile',
[path, base64.b64encode(content).decode('utf8')])
def set_model_graph(self, graph: dict):
self.client.job_action('setModelGraph', [graph])
def metric(self, name: str, x, y):
if name not in self.defined_metrics:
self.define_metric(name, {})
if not isinstance(y, list):
y = [y]
row_binary = struct.pack('<BHdd', 1, len(y), float(x), time.time())
for y1 in y:
row_binary += struct.pack('<d',
float(y1) if y1 is not None else 0.0)
self.metric_subject.on_next({'id': name, 'row': row_binary})
self.client.patch('channels.' + name + '.lastValue', y)
def log(self, s: str):
self.log_subject.on_next(s)
class RPP:
def __init__(self, muxer: MX2, discoverer: AIP, is_repeater: bool = False):
self.__muxer = muxer
self.__discoverer = discoverer
self.is_repeater = is_repeater
self.instance_info: Dict[InstanceReference, PeerInfo] = {}
self.instance_network: Dict[InstanceReference, Network] = {}
self.__announced_instances: Set[InstanceReference] = set()
# Create instance
self.__instance = self.__muxer.create_instance(RPP_NAMESPACE)
# Create transport
self.__transport = STP(self.__muxer, self.__instance)
# Create application information
self.__info = ApplicationInformation.from_instance(self.__instance)
# Subscribe to muxer and discoverer events
# self.__discoverer.ready.subscribe(self.__aip_ready)
self.__instance.incoming_greeting.subscribe(self.__received_greeting)
self.__transport.incoming_stream.subscribe(self.__new_stream)
self.__ready = False
self.ready = Subject()
# Are we a repeater?
if(self.is_repeater):
# Yes, tag ourself with that resource
self.__info.resources.add(REPEATER_RESOURCE)
# Add the application to the discoverer
self.__discoverer.add_application(self.__info).subscribe(self.__new_aip_app_peer)
# Keep a set of reachable RPP repeater peers
self.__repeaters: Set[InstanceReference] = set()
self._ready = False
self.ready = Subject()
@property
def instance_reference(self):
return self.__instance.reference
def add_instance(self, instance: InstanceReference):
# Add to our set
self.__announced_instances.add(instance)
# Announce to connected repeaters
self.__send_command(COMMAND_ANNOUNCE, Announcement([instance]).serialise().read())
def find_path(self, instance: InstanceReference):
# Construct a query
query = PathQuery(instance, 15, [])
# Create a Path Info from a stream
def read_response(stream: IngressStream):
response = PathResponse.deserialise(stream)
stream.close()
# Prepend hop to respondant repeater
# TODO these flags may need to be looked at some more or the
# PathNode object redesigned
# HACK we should probably just get some sort of full representation
# from the RPP repeater we ask the path from, but can't at the moment as we require peer info
response.nodes.insert(0, PathNode.PathNode(stream.origin, PathNode.FLAGS_NONE, self.__muxer.get_peer_info(stream.origin)))
# TODO generate many possible strategies
return [PathStrategy(PathInfo([x.instance for x in response.nodes]), response.nodes[0].peer_info),]
# Send the query and map reply to PathResponse
return self.__send_command(COMMAND_FIND_PATH, query.serialise().read(), True).pipe(operators.take(1), operators.map(read_response))
def __new_aip_app_peer(self, instance):
# Query for RPP repeater instances
self.__discoverer.find_application_resource(self.__info, REPEATER_RESOURCE).answer.subscribe(self.__found_instance)
def __found_instance(self, instance_info: InstanceInformation):
# Is this peer already reachable?
if(instance_info.instance_reference in self.__repeaters):
# Don't harras it
return
# Inquire about the peer
self.__muxer.inquire(self.__instance, instance_info.instance_reference, instance_info.connection_methods)
def __received_greeting(self, instance: InstanceReference):
# Do we already know about this peer?
if(instance in self.__repeaters):
# Nothing to do
return
# No, announce our instances
self.__send_command_to(COMMAND_ANNOUNCE, Announcement(self.__announced_instances).serialise().read(), instance).subscribe(on_completed=self.__set_ready)
# Save as repeater
self.__repeaters.add(instance)
# Are we a repeater?
if(self.is_repeater):
# Send join (empty)
self.__send_command_to(COMMAND_JOIN, b"", instance)
# Save instance info for flag lookups
self.instance_info[instance] = self.__muxer.get_peer_info(instance)
self.instance_network[instance] = self.__muxer.get_peer_network(instance)
def __set_ready(self):
if(self._ready):
return
self.ready.on_completed()
def __send_command(self, command_type: bytes, data: bytes, expect_reply: bool = False, blacklist: Set[InstanceReference] = set()) -> Subject:
# If we expect replies to this command, create a subject
reply_subject = None
if(expect_reply):
reply_subject = Subject()
count = 0
# Loop over each repeater
for repeater in self.__repeaters:
# Is this a blacklisted repeater?
if(repeater in blacklist):
# Yes, skip
continue
# Send command
subject = self.__send_command_to(command_type, data, repeater, expect_reply)
count += 1
# Do we expect a reply?
if(expect_reply):
# Yes, connect to subject
subject.subscribe(reply_subject.on_next)
Log.debug("Broadcasted a command to {} repeater/s".format(count))
# Return reply subject
return reply_subject
def __send_command_to(self, command_type: bytes, data: bytes, instance: InstanceReference, expect_reply: bool = False) -> Subject:
# Returns when the command has been sent, or with the reply if expctant
subject = Subject()
# Handler for eventual opening of stream
def on_connected(stream: EgressStream):
# Do we expect a reply?
if(expect_reply):
# Subscribe to reply
stream.reply.subscribe(subject.on_next)
# Send command type and command
stream.write(command_type + data)
# Close the stream
stream.close()
# Do we expect a reply?
if(not expect_reply):
# No, let caller know we are done
subject.on_completed()
# Open stream with the peer
self.__transport.initialise_stream(instance).subscribe(on_connected)
# Return the subject
return subject
def __new_stream(self, stream: IngressStream):
# New command, what is it?
command_type = stream.read(1)
# Announce
if(command_type == COMMAND_ANNOUNCE):
# Read announcement
announcement = Announcement.deserialise(stream)
# Get peer info and network
info = self.__muxer.get_peer_info(stream.origin)
network = self.__muxer.get_peer_network(stream.origin)
Log.debug("Peer announced itself with {} attached instance/s".format(len(announcement.instances)))
# Update records
for instance in announcement.instances:
self.instance_info[instance] = info
self.instance_network[instance] = network
# Also add info for the RPP peer
self.instance_info[stream.origin] = info
self.instance_network[stream.origin] = network
# Join
elif(command_type == COMMAND_JOIN):
Log.debug("Repeater peer joined")
# Add as repeater peer
self.__repeaters.add(stream.origin)
# Save instance info for flag lookups
self.instance_info[stream.origin] = self.__muxer.get_peer_info(stream.origin)
self.instance_network[stream.origin] = self.__muxer.get_peer_network(stream.origin)
# Find path
elif(command_type == COMMAND_FIND_PATH):
# Read the query
query = PathQuery.deserialise(stream)
# Is it an instance directly connected to us?
if(query.target in self.instance_info):
# Yes, get the flags
flags = self.__get_instance_flags(stream.origin, query.target)
Log.debug("Servicing query for path to an instance that is currently connected")
self.__query_respond(stream.origin, stream.id, PathResponse([]))
elif(query.ttl > 0):
Log.debug("Forwarding query for path to instance")
# Instance is not directly connected, forward query if still alive
self.__forward_query(stream.origin, stream.id, query)
def __forward_query(self, instance: InstanceReference, reply_to, query: PathQuery):
# Add out own instance reference to the blacklist so it can't route through us again
query.blacklist.append(self.__instance.reference)
# Handle responses
def on_responded(stream: IngressStream):
# Read response
response = PathResponse.deserialise(stream)
# TODO: Save PeerInfo of other repeaters we may be able to communicate with
# as the client asking for this path will try combinations
# Get flags for respondant repeater
flags = self.__get_instance_flags(instance, stream.origin)
# Prepend hop to respondant repeater
response.nodes.insert(0, PathNode.PathNode(stream.origin, flags, self.__muxer.get_peer_info(stream.origin)))
# Forward the query to all my repeater friends
self.__send_command(COMMAND_FIND_PATH, query.serialise().read(), True, set(query.blacklist)).pipe(operators.take(1)).subscribe(on_responded)
def __get_instance_flags(self, origin: InstanceReference, target: InstanceReference):
#set up flags
flags = PathNode.FLAGS_NONE
# Is it on the same network as the caller?
# TODO this may need to use self.instance_network instead of the muxer lookup
if(self.instance_network[origin] != self.instance_network[target]):
# No, add bridge flag
flags = flags | PathNode.FLAGS_BRIDGE
return flags
def __query_respond(self, instance: InstanceReference, reply_to, response: PathResponse):
# Handler for stream setup
def on_stream(stream: EgressStream):
# Send the response
stream.write(response.serialise().read())
# Close
stream.close()
# Set up connection with instance
self.__transport.initialise_stream(instance, in_reply_to=reply_to).subscribe(on_stream)