async def _handle_connection_cr(self, reader, writer): try: line = await reader.readline() if line != _init_string: return obj = { "targets": sorted(self.targets.keys()), "description": self.description } line = pyon.encode(obj) + "\n" writer.write(line.encode()) line = await reader.readline() if not line: return target_name = line.decode()[:-1] try: target = self.targets[target_name] except KeyError: return if callable(target): target = target() while True: line = await reader.readline() if not line: break reply = await self._process_action(target, pyon.decode(line.decode())) writer.write((pyon.encode(reply) + "\n").encode()) except (ConnectionResetError, ConnectionAbortedError, BrokenPipeError): # May happens on Windows when client disconnects pass finally: writer.close()
def test_rpc_encode_function(self): """Test that `pc_rpc` can encode a function properly. Used in `get_rpc_method_list` part of :meth:`artiq.protocols.pc_rpc.Server._process_action` """ def _annotated_function( arg1: str, arg2: np.ndarray = np.array([1,]) ) -> np.ndarray: """Sample docstring.""" return arg1 argspec_documented, docstring = pc_rpc.Server._document_function( _annotated_function ) self.assertEqual(docstring, "Sample docstring.") # purposefully ignore how argspec["annotations"] is treated. # allows option to change PYON later to encode annotations. argspec_master = dict(inspect.getfullargspec(_annotated_function)._asdict()) argspec_without_annotation = argspec_master.copy() del argspec_without_annotation["annotations"] # check if all items (excluding annotations) are same in both dictionaries self.assertLessEqual( argspec_without_annotation.items(), argspec_documented.items() ) self.assertDictEqual( argspec_documented, pyon.decode(pyon.encode(argspec_documented)) )
def test_encdec(self): for enc in pyon.encode, lambda x: pyon.encode(x, True): with self.subTest(enc=enc): self.assertEqual(pyon.decode(enc(_pyon_test_object)), _pyon_test_object) # NaNs don't compare equal, so test separately. assert np.isnan(pyon.decode(enc(np.nan)))
async def _handle_connection_cr(self, reader, writer): try: line = await reader.readline() if line != _protocol_banner: return line = await reader.readline() if not line: return notifier_name = line.decode()[:-1] try: notifier = self.notifiers[notifier_name] except KeyError: return obj = {"action": ModAction.init.value, "struct": notifier.raw_view} line = pyon.encode(obj) + "\n" writer.write(line.encode()) queue = asyncio.Queue() self._recipients[notifier_name].add(queue) try: while True: line = await queue.get() writer.write(line) # raise exception on connection error await writer.drain() finally: self._recipients[notifier_name].remove(queue) except (ConnectionError, TimeoutError): # subscribers disconnecting are a normal occurrence pass finally: writer.close()
def _show_devices(devices): clear_screen() table = PrettyTable(["Name", "Description"]) table.align["Description"] = "l" for k, v in sorted(devices.items(), key=itemgetter(0)): table.add_row([k, pyon.encode(v, True)]) print(table)
def _handle_connection_cr(self, reader, writer): try: line = yield from reader.readline() if line != _init_string: return line = yield from reader.readline() if not line: return notifier_name = line.decode()[:-1] try: notifier = self.notifiers[notifier_name] except KeyError: return obj = {"action": "init", "struct": notifier.read} line = pyon.encode(obj) + "\n" writer.write(line.encode()) queue = asyncio.Queue() self._recipients[notifier_name].add(queue) try: while True: line = yield from queue.get() writer.write(line) # raise exception on connection error yield from writer.drain() finally: self._recipients[notifier_name].remove(queue) except ConnectionResetError: # subscribers disconnecting are a normal occurence pass finally: writer.close()
async def _handle_connection_cr(self, reader, writer): try: line = await reader.readline() if line != _init_string: return line = await reader.readline() if not line: return notifier_name = line.decode()[:-1] try: notifier = self.notifiers[notifier_name] except KeyError: return obj = {"action": "init", "struct": notifier.read} line = pyon.encode(obj) + "\n" writer.write(line.encode()) queue = asyncio.Queue() self._recipients[notifier_name].add(queue) try: while True: line = await queue.get() writer.write(line) # raise exception on connection error await writer.drain() finally: self._recipients[notifier_name].remove(queue) except (ConnectionResetError, ConnectionAbortedError, BrokenPipeError): # subscribers disconnecting are a normal occurence pass finally: writer.close()
def test_encdec_array(self): orig = {k: (np.array(v), np.array([v])) for k, v in _pyon_test_object.items() if np.isscalar(v)} for enc in pyon.encode, lambda x: pyon.encode(x, True): result = pyon.decode(enc(orig)) for k in orig: with self.subTest(enc=enc, k=k, v=orig[k]): np.testing.assert_equal(result[k], orig[k])
def format_influxdb(v): if np.issubdtype(type(v), np.bool_): return "bool={}".format(v) if np.issubdtype(type(v), np.integer): return "int={}i".format(v) if np.issubdtype(type(v), np.floating): return "float={}".format(v) if np.issubdtype(type(v), np.str_): return "str=\"{}\"".format(v.replace('"', '\\"')) return "pyon=\"{}\"".format(pyon.encode(v).replace('"', '\\"'))
def test_encdec_array(self): orig = { k: (np.array(v), np.array([v])) for k, v in _pyon_test_object.items() if np.isscalar(v) } for enc in pyon.encode, lambda x: pyon.encode(x, True): result = pyon.decode(enc(orig)) for k in orig: with self.subTest(enc=enc, k=k, v=orig[k]): np.testing.assert_equal(result[k], orig[k])
async def _handle_connection_cr(self, reader, writer): try: line = await reader.readline() if line != _init_string: return obj = { "targets": sorted(self.targets.keys()), "description": self.description } line = pyon.encode(obj) + "\n" writer.write(line.encode()) line = await reader.readline() if not line: return target_name = line.decode()[:-1] try: target = self.targets[target_name] except KeyError: return if callable(target): target = target() valid_methods = inspect.getmembers(target, inspect.ismethod) valid_methods = {m[0] for m in valid_methods} if self.builtin_terminate: valid_methods.add("terminate") writer.write((pyon.encode(valid_methods) + "\n").encode()) while True: line = await reader.readline() if not line: break reply = await self._process_action(target, pyon.decode(line.decode())) writer.write((pyon.encode(reply) + "\n").encode()) except (ConnectionResetError, ConnectionAbortedError, BrokenPipeError): # May happens on Windows when client disconnects pass finally: writer.close()
def broadcast(self, name, obj): if name in self._recipients: line = pyon.encode(obj) + "\n" line = line.encode() for recipient in self._recipients[name]: try: recipient.put_nowait(line) except asyncio.QueueFull: # do not log: log messages may be sent back to us # as broadcasts, and cause infinite recursion. pass
def _send(self, obj, timeout): line = pyon.encode(obj) self.process.stdin.write(line.encode()) self.process.stdin.write("\n".encode()) try: fut = self.process.stdin.drain() if fut is not (): # FIXME: why does Python return this? yield from asyncio.wait_for(fut, timeout=timeout) except asyncio.TimeoutError: raise WorkerFailed("Timeout sending data from worker") except: raise WorkerFailed("Failed to send data to worker")
def format_influxdb(v): if isinstance(v, bool): if v: return "bool", "t" else: return "bool", "f" elif np.issubdtype(type(v), int): return "int", "{}i".format(v) elif np.issubdtype(type(v), float): return "float", "{}".format(v) elif isinstance(v, str): return "str", influxdb_str(v) else: return "pyon", influxdb_str(pyon.encode(v))
def _handle_connection_cr(self, reader, writer): try: line = yield from reader.readline() if line != _init_string: return obj = { "targets": sorted(self.targets.keys()), "parameters": self.id_parameters } line = pyon.encode(obj) + "\n" writer.write(line.encode()) line = yield from reader.readline() if not line: return target_name = line.decode()[:-1] try: target = self.targets[target_name] except KeyError: return while True: line = yield from reader.readline() if not line: break obj = pyon.decode(line.decode()) try: method = getattr(target, obj["name"]) ret = method(*obj["args"], **obj["kwargs"]) obj = {"status": "ok", "ret": ret} except Exception: obj = {"status": "failed", "message": traceback.format_exc()} line = pyon.encode(obj) + "\n" writer.write(line.encode()) finally: writer.close()
def format_influxdb(v, tag=True): if np.issubdtype(type(v), np.bool_): return "{}".format(v) if np.issubdtype(type(v), np.integer): return "{}i".format(v) if np.issubdtype(type(v), np.floating): return "{}".format(v) if not np.issubdtype(type(v), np.str_): v = pyon.encode(v) if tag: for i in ",= ": v = v.replace(i, "\\" + i) return v else: return "\"{}\"".format(v.replace('"', '\\"'))
async def _send(self, obj, cancellable=True): assert self.io_lock.locked() line = pyon.encode(obj) self.ipc.write((line + "\n").encode()) ifs = [self.ipc.drain()] if cancellable: ifs.append(self.closed.wait()) fs = await asyncio_wait_or_cancel( ifs, timeout=self.send_timeout, return_when=asyncio.FIRST_COMPLETED) if all(f.cancelled() for f in fs): raise WorkerTimeout("Timeout sending data to worker") for f in fs: if not f.cancelled() and f.done(): f.result() # raise any exceptions if cancellable and self.closed.is_set(): raise WorkerError("Data transmission to worker cancelled")
def put_object(obj): ds = pyon.encode(obj) ipc.write((ds + "\n").encode())
def test_encdec(self): for enc in pyon.encode, lambda x: pyon.encode(x, True): self.assertEqual(pyon.decode(enc(_pyon_test_object)), _pyon_test_object)
def main(): global ipc multiline_log_config(level=int(sys.argv[2])) ipc = pipe_ipc.ChildComm(sys.argv[1]) start_time = None rid = None expid = None exp = None exp_inst = None repository_path = None device_mgr = DeviceManager(ParentDeviceDB) device_mgr.virtual_devices["scheduler"] = Scheduler(device_mgr) dataset_mgr = DatasetManager(ParentDatasetDB) try: while True: obj = get_object() action = obj["action"] if action == "build": start_time = time.localtime() rid = obj["rid"] expid = obj["expid"] if obj["wd"] is not None: # Using repository experiment_file = os.path.join(obj["wd"], expid["file"]) repository_path = obj["wd"] else: experiment_file = expid["file"] repository_path = None setup_diagnostics(experiment_file, repository_path) exp = get_exp(experiment_file, expid["class_name"]) device_mgr.virtual_devices["scheduler"].set_run_info( rid, obj["pipeline_name"], expid, obj["priority"]) dirname = os.path.join("results", time.strftime("%Y-%m-%d", start_time), time.strftime("%H", start_time)) os.makedirs(dirname, exist_ok=True) os.chdir(dirname) argument_mgr = ProcessArgumentManager(expid["arguments"]) exp_inst = exp((device_mgr, dataset_mgr, argument_mgr)) put_object({"action": "completed"}) elif action == "prepare": exp_inst.prepare() put_object({"action": "completed"}) elif action == "run": exp_inst.run() put_object({"action": "completed"}) elif action == "analyze": exp_inst.analyze() put_object({"action": "completed"}) elif action == "write_results": filename = "{:09}-{}.h5".format(rid, exp.__name__) with h5py.File(filename, "w") as f: dataset_mgr.write_hdf5(f.create_group("datasets")) f["artiq_version"] = artiq_version f["rid"] = rid f["start_time"] = int(time.mktime(start_time)) f["expid"] = pyon.encode(expid) put_object({"action": "completed"}) elif action == "examine": examine(ExamineDeviceMgr, ParentDatasetDB, obj["file"]) put_object({"action": "completed"}) elif action == "terminate": break except Exception as exc: # When we get CompileError, a more suitable diagnostic has already # been printed. if not isinstance(exc, CompileError): short_exc_info = type(exc).__name__ exc_str = str(exc) if exc_str: short_exc_info += ": " + exc_str.splitlines()[0] lines = ["Terminating with exception (" + short_exc_info + ")\n"] if hasattr(exc, "artiq_core_exception"): lines.append(str(exc.artiq_core_exception)) if hasattr(exc, "parent_traceback"): lines += exc.parent_traceback lines += traceback.format_exception_only(type(exc), exc) logging.error("".join(lines).rstrip(), exc_info=not hasattr(exc, "parent_traceback")) put_object({"action": "exception"}) finally: device_mgr.close_devices() ipc.close()
def put_object(obj): ds = pyon.encode(obj) sys.__stdout__.write(ds) sys.__stdout__.write("\n") sys.__stdout__.flush()
def test_encdec(self): for enc in pyon.encode, lambda x: pyon.encode(x, True), json.dumps: for dec in pyon.decode, json.loads: self.assertEqual(dec(enc(_json_test_object)), _json_test_object)
async def _handle_connection_cr(self, reader, writer): try: line = await reader.readline() if line != _init_string: return obj = { "targets": sorted(self.targets.keys()), "description": self.description } line = pyon.encode(obj) + "\n" writer.write(line.encode()) line = await reader.readline() if not line: return target_name = line.decode()[:-1] try: target = self.targets[target_name] except KeyError: return while True: line = await reader.readline() if not line: break obj = pyon.decode(line.decode()) try: if obj["action"] == "get_rpc_method_list": members = inspect.getmembers(target, inspect.ismethod) doc = { "docstring": inspect.getdoc(target), "methods": {} } for name, method in members: if name.startswith("_"): continue method = getattr(target, name) argspec = inspect.getfullargspec(method) doc["methods"][name] = (dict(argspec.__dict__), inspect.getdoc(method)) if self.builtin_terminate: doc["methods"]["terminate"] = ({ "args": ["self"], "defaults": None, "varargs": None, "varkw": None, "kwonlyargs": [], "kwonlydefaults": [], }, "Terminate the server.") obj = {"status": "ok", "ret": doc} elif obj["action"] == "call": logger.debug("calling %s", _PrettyPrintCall(obj)) if self.builtin_terminate and obj[ "name"] == "terminate": self._terminate_request.set() obj = {"status": "ok", "ret": None} else: method = getattr(target, obj["name"]) ret = method(*obj["args"], **obj["kwargs"]) obj = {"status": "ok", "ret": ret} else: raise ValueError("Unknown action: {}".format( obj["action"])) except Exception: obj = { "status": "failed", "message": traceback.format_exc() } line = pyon.encode(obj) + "\n" writer.write(line.encode()) finally: writer.close()
def _update_ndscan_params(arguments, params): arguments[PARAMS_ARG_KEY]["state"] = pyon.encode(params)
def __send(self, obj): line = pyon.encode(obj) + "\n" self.__socket.sendall(line.encode())
def main(): global ipc multiline_log_config(level=int(sys.argv[2])) ipc = pipe_ipc.ChildComm(sys.argv[1]) start_time = None rid = None expid = None exp = None exp_inst = None repository_path = None device_mgr = DeviceManager(ParentDeviceDB) device_mgr.virtual_devices["scheduler"] = Scheduler(device_mgr) dataset_mgr = DatasetManager(ParentDatasetDB) try: while True: obj = get_object() action = obj["action"] if action == "build": start_time = time.localtime() rid = obj["rid"] expid = obj["expid"] if obj["wd"] is not None: # Using repository experiment_file = os.path.join(obj["wd"], expid["file"]) repository_path = obj["wd"] else: experiment_file = expid["file"] repository_path = None setup_diagnostics(experiment_file, repository_path) exp = get_exp(experiment_file, expid["class_name"]) device_mgr.virtual_devices["scheduler"].set_run_info( rid, obj["pipeline_name"], expid, obj["priority"]) dirname = os.path.join("results", time.strftime("%Y-%m-%d", start_time), time.strftime("%H", start_time)) os.makedirs(dirname, exist_ok=True) os.chdir(dirname) argument_mgr = ProcessArgumentManager(expid["arguments"]) exp_inst = exp((device_mgr, dataset_mgr, argument_mgr)) put_object({"action": "completed"}) elif action == "prepare": exp_inst.prepare() put_object({"action": "completed"}) elif action == "run": exp_inst.run() put_object({"action": "completed"}) elif action == "analyze": exp_inst.analyze() put_object({"action": "completed"}) elif action == "write_results": filename = "{:09}-{}.h5".format(rid, exp.__name__) with h5py.File(filename, "w") as f: dataset_mgr.write_hdf5(f.create_group("datasets")) f["artiq_version"] = artiq_version f["rid"] = rid f["start_time"] = int(time.mktime(start_time)) f["expid"] = pyon.encode(expid) put_object({"action": "completed"}) elif action == "examine": examine(ExamineDeviceMgr, ParentDatasetDB, obj["file"]) put_object({"action": "completed"}) elif action == "terminate": break except Exception as exc: # When we get CompileError, a more suitable diagnostic has already # been printed. if not isinstance(exc, CompileError): short_exc_info = type(exc).__name__ exc_str = str(exc) if exc_str: short_exc_info += ": " + exc_str.splitlines()[0] lines = ["Terminating with exception ("+short_exc_info+")\n"] if hasattr(exc, "artiq_core_exception"): lines.append(str(exc.artiq_core_exception)) if hasattr(exc, "parent_traceback"): lines += exc.parent_traceback lines += traceback.format_exception_only(type(exc), exc) logging.error("".join(lines).rstrip(), exc_info=not hasattr(exc, "parent_traceback")) put_object({"action": "exception"}) finally: device_mgr.close_devices() ipc.close()
def describe(self): d = {"ty": self.__class__.__name__} if hasattr(self, "default_value"): d["default"] = pyon.encode(self.default_value) return d
def main(): global ipc multiline_log_config(level=int(sys.argv[2])) ipc = pipe_ipc.ChildComm(sys.argv[1]) start_time = None run_time = None rid = None expid = None exp = None exp_inst = None repository_path = None device_mgr = DeviceManager(ParentDeviceDB, virtual_devices={"scheduler": Scheduler(), "ccb": CCB()}) dataset_mgr = DatasetManager(ParentDatasetDB) import_cache.install_hook() try: while True: obj = get_object() action = obj["action"] if action == "build": start_time = time.time() rid = obj["rid"] expid = obj["expid"] if obj["wd"] is not None: # Using repository experiment_file = os.path.join(obj["wd"], expid["file"]) repository_path = obj["wd"] else: experiment_file = expid["file"] repository_path = None setup_diagnostics(experiment_file, repository_path) exp = get_exp(experiment_file, expid["class_name"]) device_mgr.virtual_devices["scheduler"].set_run_info( rid, obj["pipeline_name"], expid, obj["priority"]) start_local_time = time.localtime(start_time) dirname = os.path.join("results", time.strftime("%Y-%m-%d", start_local_time), time.strftime("%H", start_local_time)) os.makedirs(dirname, exist_ok=True) os.chdir(dirname) argument_mgr = ProcessArgumentManager(expid["arguments"]) exp_inst = exp((device_mgr, dataset_mgr, argument_mgr)) put_object({"action": "completed"}) elif action == "prepare": exp_inst.prepare() put_object({"action": "completed"}) elif action == "run": run_time = time.time() exp_inst.run() put_object({"action": "completed"}) elif action == "analyze": try: exp_inst.analyze() except: # make analyze failure non-fatal, as we may still want to # write results afterwards put_exception_report() else: put_object({"action": "completed"}) elif action == "write_results": filename = "{:09}-{}.h5".format(rid, exp.__name__) with h5py.File(filename, "w") as f: dataset_mgr.write_hdf5(f) f["artiq_version"] = artiq_version f["rid"] = rid f["start_time"] = start_time f["run_time"] = run_time f["expid"] = pyon.encode(expid) put_object({"action": "completed"}) elif action == "examine": examine(ExamineDeviceMgr, ExamineDatasetMgr, obj["file"]) put_object({"action": "completed"}) elif action == "terminate": break except: put_exception_report() finally: device_mgr.close_devices() ipc.close()
def __send(self, obj): line = pyon.encode(obj) + "\n" self.__writer.write(line.encode())
async def _handle_connection_cr(self, reader, writer): try: line = await reader.readline() if line != _init_string: return obj = { "targets": sorted(self.targets.keys()), "description": self.description } line = pyon.encode(obj) + "\n" writer.write(line.encode()) line = await reader.readline() if not line: return target_name = line.decode()[:-1] try: target = self.targets[target_name] except KeyError: return while True: line = await reader.readline() if not line: break obj = pyon.decode(line.decode()) try: if obj["action"] == "get_rpc_method_list": members = inspect.getmembers(target, inspect.ismethod) doc = { "docstring": inspect.getdoc(target), "methods": {} } for name, method in members: if name.startswith("_"): continue method = getattr(target, name) argspec = inspect.getfullargspec(method) doc["methods"][name] = (dict(argspec._asdict()), inspect.getdoc(method)) if self.builtin_terminate: doc["methods"]["terminate"] = ( { "args": ["self"], "defaults": None, "varargs": None, "varkw": None, "kwonlyargs": [], "kwonlydefaults": [], }, "Terminate the server.") obj = {"status": "ok", "ret": doc} elif obj["action"] == "call": logger.debug("calling %s", _PrettyPrintCall(obj)) if self.builtin_terminate and obj["name"] == "terminate": self._terminate_request.set() obj = {"status": "ok", "ret": None} else: method = getattr(target, obj["name"]) ret = method(*obj["args"], **obj["kwargs"]) if inspect.iscoroutine(ret): ret = await ret obj = {"status": "ok", "ret": ret} else: raise ValueError("Unknown action: {}" .format(obj["action"])) except Exception: obj = {"status": "failed", "message": traceback.format_exc()} line = pyon.encode(obj) + "\n" writer.write(line.encode()) except (ConnectionResetError, ConnectionAbortedError, BrokenPipeError): # May happens on Windows when client disconnects pass finally: writer.close()
def write_pyon(self, obj): self.write(pyon.encode(obj).encode() + b"\n")
def set_argument_value(self, value): self.setText(pyon.encode(value))
def main(): global ipc multiline_log_config(level=int(sys.argv[2])) ipc = pipe_ipc.ChildComm(sys.argv[1]) start_time = None run_time = None rid = None expid = None exp = None exp_inst = None repository_path = None device_mgr = DeviceManager(ParentDeviceDB, virtual_devices={ "scheduler": Scheduler(), "ccb": CCB() }) dataset_mgr = DatasetManager(ParentDatasetDB) import_cache.install_hook() try: while True: obj = get_object() action = obj["action"] if action == "build": start_time = time.time() rid = obj["rid"] expid = obj["expid"] if obj["wd"] is not None: # Using repository experiment_file = os.path.join(obj["wd"], expid["file"]) repository_path = obj["wd"] else: experiment_file = expid["file"] repository_path = None setup_diagnostics(experiment_file, repository_path) exp = get_exp(experiment_file, expid["class_name"]) device_mgr.virtual_devices["scheduler"].set_run_info( rid, obj["pipeline_name"], expid, obj["priority"]) start_local_time = time.localtime(start_time) dirname = os.path.join( "results", time.strftime("%Y-%m-%d", start_local_time), time.strftime("%H", start_local_time)) os.makedirs(dirname, exist_ok=True) os.chdir(dirname) argument_mgr = ProcessArgumentManager(expid["arguments"]) exp_inst = exp((device_mgr, dataset_mgr, argument_mgr, {})) put_object({"action": "completed"}) elif action == "prepare": exp_inst.prepare() put_object({"action": "completed"}) elif action == "run": run_time = time.time() exp_inst.run() put_object({"action": "completed"}) elif action == "analyze": try: exp_inst.analyze() except: # make analyze failure non-fatal, as we may still want to # write results afterwards put_exception_report() else: put_object({"action": "completed"}) elif action == "write_results": filename = "{:09}-{}.h5".format(rid, exp.__name__) with h5py.File(filename, "w") as f: dataset_mgr.write_hdf5(f) f["artiq_version"] = artiq_version f["rid"] = rid f["start_time"] = start_time f["run_time"] = run_time f["expid"] = pyon.encode(expid) put_object({"action": "completed"}) elif action == "examine": examine(ExamineDeviceMgr, ExamineDatasetMgr, obj["file"]) put_object({"action": "completed"}) elif action == "terminate": break except: put_exception_report() finally: device_mgr.close_devices() ipc.close()
def publish(self, notifier, mod): line = pyon.encode(mod) + "\n" line = line.encode() notifier_name = self._notifier_names[id(notifier)] for recipient in self._recipients[notifier_name]: recipient.put_nowait(line)