class MPMClient: """ MPM RPC Client: Will make all MPM commands accessible as Python methods. """ def __init__(self, init_mode, host, port, token=None): assert isinstance(init_mode, InitMode) print("[MPMRPC] Attempting to connect to {host}:{port}...".format( host=host, port=port )) try: self._client = RPCClient(host, port, pack_params={'use_bin_type': True}) print("[MPMRPC] Connection successful.") except Exception as ex: print("[MPMRPC] Connection refused: {}".format(ex)) raise RuntimeError("RPC connection refused.") self._remote_methods = [] if init_mode == InitMode.Hijack: assert token self._token = token if init_mode == InitMode.Claim: self._claimer = MPMClaimer(host, port) self._claimer.claim() self._token = self._claimer.token print("[MPMRPC] Getting methods...") methods = self._client.call('list_methods') for method in methods: self._add_command(*method) print("[MPMRPC] Added {} methods.".format(len(methods))) def _add_command(self, command, docs, requires_token): """ Add a command to the current session """ if not hasattr(self, command): new_command = lambda *args, **kwargs: self._rpc_template( str(command), requires_token, *args, **kwargs ) new_command.__doc__ = docs setattr(self, command, new_command) self._remote_methods.append(command) def _rpc_template(self, command, requires_token, *args, **kwargs): """ Template function to create new RPC shell commands """ if requires_token and not self._token: raise RuntimeError( "[MPMRPC] Cannot execute `{}' -- no claim available!" .format(command)) # Put token as the first argument if required: if requires_token: args = (self._token,) + args if kwargs: return self._client.call(command, *args, **kwargs) if args: return self._client.call(command, *args) return self._client.call(command)
def rpc(address, port, command, *args): if not port: port = mpm.types.MPM_RPC_PORT client = RPCClient(address, port) if args: args = [eval(arg.lstrip("=")) if arg.startswith("=") else arg for arg in args] result = client.call(command, *args) else: result = client.call(command) return result
def rpc(address, port, command, *args): if not port: port = mpm.types.MPM_RPC_PORT client = RPCClient(address, port) if args: args = [ eval(arg.lstrip("=")) if arg.startswith("=") else arg for arg in args ] result = client.call(command, *args) else: result = client.call(command) return result
def complete(self, context, prefix): from mprpc import RPCClient client = RPCClient('localhost', self.__port) response = client.call('complete', context, prefix) return [decode_completion(r) for r in response]
def segment(self, text): from mprpc import RPCClient client = RPCClient('localhost', self.__port) response = client.call('segment', text) return [decode_segment(r) for r in response]
def claim_loop(self, host, port, disc_callback): """ Run a claim loop """ client = RPCClient(host, port, pack_params={'use_bin_type': True}) self.token = client.call('claim', 'MPM Shell') try: while not self._exit_loop: client.call('reclaim', self.token) time.sleep(1) client.call('unclaim', self.token) except RPCError as ex: print("Unexpected RPC error in claimer loop!") print(str(ex)) disc_callback() self.token = None
def call(): from mprpc import RPCClient client = RPCClient('127.0.0.1', 7777) t = np.zeros(NUM_CALLS) for _ in range(NUM_CALLS): t1 = time.time() client.call('sum', 1, 2) t2 = time.time() t[_] = t2 - t1 print('stdev: {}'.format(t.std())) print('mean: {}'.format(t.mean())) print('call: {} qps'.format(1 / t.mean()))
def claim_loop(self, host, port, disc_callback): """ Run a claim loop """ client = RPCClient(host, port, pack_params={'use_bin_type': True}) self.token = client.call('claim', 'MPM Shell') try: while not self._exit_loop: client.call('reclaim', self.token) time.sleep(1) client.call('unclaim', self.token) except RPCError as ex: print("Unexpected RPC error in claimer loop!") print(str(ex)) disc_callback() self.token = None
def call_register(uuid): print("Registering peer with uuid: {}".format(uuid)) client = RPCClient(b'localhost', 18080) res = client.call(b'register', uuid) client.close() return res
def call_register(uuid): print("Registering peer with uuid: {}".format(uuid)) client = RPCClient(b'localhost', 18080) res = client.call(b'register', uuid) client.close() return res
def call(): from mprpc import RPCClient client = RPCClient('ipc://x.sock') start = time.time() [client.call('sum', 1, 2) for _ in xrange(NUM_CALLS)] print 'call: %d qps' % (NUM_CALLS / (time.time() - start))
def call(): from mprpc import RPCClient client = RPCClient('127.0.0.1', 6000) start = time.time() [client.call('sum', 1, 2) for _ in xrange(NUM_CALLS)] print 'call: %d qps' % (NUM_CALLS / (time.time() - start))
class BitFlyerMarket(BitFlyerMarketBase): def __init__(self, symbol=SYMBOL, resolution=60, port=PROXY_PORT): super().__init__(symbol, resolution) self.client = RPCClient('127.0.0.1', port) self.data = self.client.call('ohlcv', TICKERID, self.resolution, 256) def step_ohlcv(self, next_clock): d1, d0 = self.client.call('step_ohlcv', TICKERID, self.resolution, next_clock) if d1 is None: return None if d0['t'] <= self.data['t'][-1]: return None for k, v in d0.items(): self.data[k].pop(0) self.data[k][-1] = d1[k] self.data[k].append(v) return d0['t']
def call_get_schedule(uuid, num, method="smart"): print("Getting schedule for uuid: {}".format(uuid)) data = {'uuid': uuid, 'num': num, 'method': method} client = RPCClient(b'localhost', 18080) result = client.call(b'get_schedule', data) client.close() return result
def call_get_schedule(uuid, num, method="smart"): print("Getting schedule for uuid: {}".format(uuid)) data = { 'uuid':uuid, 'num':num, 'method':method } client = RPCClient(b'localhost', 18080) result = client.call(b'get_schedule', data) client.close() return result
def __exec_mprpc__(self, args): from mprpc import RPCClient, RPCPoolClient method = args[0] args = args[1] assert method in ["get", "set"] try: client = RPCClient(self.server, self.mprpc_port) result = client.call(method, args) client.close() return result except Exception as e: print(e) try: client.close() except: pass
def test_server(self, server): class Sum: def sum(self, x, y): return x + y server.load(Sum) self.g = gevent.spawn(server.run) sleep(0.1) registry = server.registry res = registry.discovery(server.name) key = registry._node_key(server.name, url.get_param("node")) host, port = res[key].split(":") client = RPCClient(str(host), int(port)) assert client.call("sum", {}, 1, 2) == 3
class Client(BaseClient): def __init__(self, host="127.0.0.1", port=9033, model="en", embeddings_path=None, verbose=False): super(Client, self).__init__(model, embeddings_path) self.host = host self.port = port self.rpc = RPCClient(host, port) self.verbose = verbose def _call(self, path, *args): return self.rpc.call(path, *args) @cachetools.func.lru_cache(maxsize=3000000) def single(self, document, attributes=None): sentences = self._call("single", document, self.model, self.embeddings_path, attributes) return SpacyClientDocument(sentences) def _bulk(self, documents, attributes): return self._call("bulk", documents, self.model, self.embeddings_path, attributes) def bulk(self, documents, batch_size=1000, attributes=None): parsed_documents = [] if len(documents) > batch_size: batches = int(math.ceil(len(documents) / batch_size)) print("Batching {} requests with batch_size {}".format( batches, batch_size)) if self.verbose: batch_iterator = tqdm.tqdm(range(batches)) else: batch_iterator = range(batches) for b in batch_iterator: docs = documents[b * batch_size:(b + 1) * batch_size] res = self._bulk(docs, attributes) parsed_documents.extend(res) else: parsed_documents = self._bulk(documents, attributes) return [SpacyClientDocument(x) for x in parsed_documents]
def _claim_loop(host, port, cmd_q, token_q): """ Process that runs a claim loop. This function should be run in its own thread. Communication to the outside thread happens with two queues: A command queue, and a token queue. The command queue is used to pass in one of these commands: claim, unclaim, or exit. The token queue is used to read back the current token. """ command = None token = None exit_loop = False client = RPCClient(host, port, pack_params={'use_bin_type': True}) try: while not exit_loop: if token and not command: client.call('reclaim', token) elif command == 'claim': if not token: token = client.call('claim', 'UHD') else: print("Already have claim") token_q.put(token) elif command == 'unclaim': if token: client.call('unclaim', token) token = None token_q.put(None) elif command == 'exit': if token: client.call('unclaim', token) token = None token_q.put(None) exit_loop = True time.sleep(1) command = None if not cmd_q.empty(): command = cmd_q.get(False) except RPCError as ex: print("Unexpected RPC error in claimer loop!") print(str(ex))
def claim_loop(self, host, port, cmd_q, token_q): """ Run a claim loop """ from mprpc import RPCClient from mprpc.exceptions import RPCError command = None token = None exit_loop = False client = RPCClient(host, port, pack_params={'use_bin_type': True}) try: while not exit_loop: if token and not command: client.call('reclaim', token) elif command == 'claim': if not token: token = client.call('claim', 'MPM Shell') else: print("Already have claim") token_q.put(token) elif command == 'unclaim': if token: client.call('unclaim', token) token = None token_q.put(None) elif command == 'exit': if token: client.call('unclaim', token) token = None token_q.put(None) exit_loop = True time.sleep(1) command = None if not cmd_q.empty(): command = cmd_q.get(False) except RPCError as ex: print("Unexpected RPC error in claimer loop!") print(str(ex))
from mprpc import RPCClient client = RPCClient('localhost', 6666 ) #o = client.call('sum', 1, 2) s = client.call('sleep', 10)
def call(): client = RPCClient('127.0.0.1', 6000) client.open() print client.call('sum', 1, 2)
from mprpc import RPCClient client = RPCClient('127.0.0.1', 6000) print client.call('add', 1000, 1000)
class MPMShell(cmd.Cmd): """ RPC Shell class. See cmd module. """ def __init__(self, host, port, claim, hijack, script): cmd.Cmd.__init__(self) self.prompt = "> " self.client = None self.remote_methods = [] self._host = host self._port = port self._device_info = None self._claimer = MPMClaimer(self._host, self._port) if host is not None: self.connect(host, port) if claim: self.claim() elif hijack: self.hijack(hijack) self.update_prompt() self._script = script if self._script: self.parse_script() def _add_command(self, command, docs, requires_token=False): """ Add a command to the current session """ cmd_name = 'do_' + command if not hasattr(self, cmd_name): new_command = lambda args: self.rpc_template( str(command), requires_token, args) new_command.__doc__ = docs setattr(self, cmd_name, new_command) self.remote_methods.append(command) def _print_response(self, response): print(re.sub("^", "< ", response, flags=re.MULTILINE)) def rpc_template(self, command, requires_token, args=None): """ Template function to create new RPC shell commands """ from mprpc.exceptions import RPCError if requires_token and \ (self._claimer is None or self._claimer.get_token() is None): self._print_response("Cannot execute `{}' -- " "no claim available!".format(command)) return False try: if args or requires_token: expanded_args = self.expand_args(args) if requires_token: expanded_args.insert(0, self._claimer.get_token()) response = self.client.call(command, *expanded_args) else: response = self.client.call(command) except RPCError as ex: self._print_response("RPC Command failed!\nError: {}".format(ex)) return False except Exception as ex: self._print_response("Unexpected exception!\nError: {}".format(ex)) return True if isinstance(response, bool): if response: self._print_response("Command succeeded.") else: self._print_response("Command failed!") else: self._print_response(str(response)) return False def get_names(self): " We need this for tab completion. " return dir(self) ########################################################################### # Cmd module specific ########################################################################### def default(self, line): self._print_response("*** Unknown syntax: %s" % line) def preloop(self): """ In script mode add Execution start marker to ease parsing script output :return: None """ if self._script: print("Execute %s" % self._script) def precmd(self, line): """ Add command prepended by "> " in scripting mode to ease parsing script output. """ if self.cmdqueue: print("> %s" % line) return line def postcmd(self, stop, line): """ Is run after every command executes. Does: - Update prompt """ self.update_prompt() return stop ########################################################################### # Internal methods ########################################################################### def connect(self, host, port): """ Launch a connection. """ from mprpc import RPCClient print("Attempting to connect to {host}:{port}...".format(host=host, port=port)) try: self.client = RPCClient(host, port, pack_params={'use_bin_type': True}) print("Connection successful.") except Exception as ex: print("Connection refused") print("Error: {}".format(ex)) return False self._host = host self._port = port print("Getting methods...") methods = self.client.call('list_methods') for method in methods: self._add_command(*method) print("Added {} methods.".format(len(methods))) print("Quering device info...") self._device_info = self.client.call('get_device_info') return True def disconnect(self): """ Clean up after a connection was closed. """ from mprpc.exceptions import RPCError self._device_info = None if self._claimer is not None: self._claimer.exit() if self.client: try: self.client.close() except RPCError as ex: print("Error while closing the connection") print("Error: {}".format(ex)) for method in self.remote_methods: delattr(self, "do_" + method) self.remote_methods = [] self.client = None self._host = None self._port = None def claim(self): " Initialize claim " print("Claiming device...") self._claimer.claim() return True def hijack(self, token): " Hijack running session " if self._claimer.hijacked: print("Claimer already active. Can't hijack.") return False print("Hijacking device...") self._claimer.hijack(token) return True def unclaim(self): """ unclaim """ self._claimer.unclaim() def update_prompt(self): """ Update prompt """ if self._device_info is None: self.prompt = '> ' else: token = self._claimer.get_token() if token is None: claim_status = '' elif self._claimer.hijacked: claim_status = ' [H]' else: claim_status = ' [C]' self.prompt = '{dev_id}{claim_status}> '.format( dev_id=self._device_info.get( 'name', self._device_info.get('serial', '?')), claim_status=claim_status, ) def parse_script(self): """ Adding script command from file pointed to by self._script. The commands are read from file one per line and added to cmdqueue of parent class. This way they will be executed instead of input from stdin. An EOF command is appended to the list to ensure the shell exits after script execution. :return: None """ try: with open(self._script, "r") as script: for command in script: self.cmdqueue.append(command.strip()) except OSError as ex: print("Failed to read script. (%s)" % ex) self.cmdqueue.append("EOF") # terminate shell after script execution def expand_args(self, args): """ Takes a string and returns a list """ if self._claimer is not None and self._claimer.get_token() is not None: args = args.replace('$T', str(self._claimer.get_token())) eval_preamble = '=' args = args.strip() if args.startswith(eval_preamble): parsed_args = eval(args.lstrip(eval_preamble)) if not isinstance(parsed_args, list): parsed_args = [parsed_args] else: parsed_args = [] for arg in args.split(): try: parsed_args.append(int(arg, 0)) continue except ValueError: pass try: parsed_args.append(float(arg)) continue except ValueError: pass parsed_args.append(arg) return parsed_args ########################################################################### # Predefined commands ########################################################################### def do_connect(self, args): """ Connect to a remote MPM server. See connect() """ host, port = split_args(args, 'localhost', MPM_RPC_PORT) port = int(port) self.connect(host, port) def do_claim(self, _): """ Spawn a claim loop """ self.claim() def do_hijack(self, token): """ Hijack a running session """ self.hijack(token) def do_unclaim(self, _): """ unclaim """ self.unclaim() def do_disconnect(self, _): """ disconnect from the RPC server """ self.disconnect() def do_import(self, args): """import a python module into the global namespace""" globals()[args] = import_module(args) # pylint: disable=invalid-name def do_EOF(self, _): """ When catching EOF, exit the program. """ print("Exiting...") self.disconnect() return True # orderly shutdown
def call(): client = RPCClient('ipc://x.sock', reconnect_delay=5) print client.call('sum', 1, 2)
class MPMShell(cmd.Cmd): """ RPC Shell class. See cmd module. """ def __init__(self, host, port, claim, hijack): cmd.Cmd.__init__(self) self.prompt = "> " self.client = None self.remote_methods = [] self._claimer = None self._host = host self._port = port self._device_info = None if host is not None: self.connect(host, port) if claim: self.claim() elif hijack: self.hijack(hijack) self.update_prompt() def _add_command(self, command, docs, requires_token=False): """ Add a command to the current session """ cmd_name = 'do_' + command if not hasattr(self, cmd_name): new_command = lambda args: self.rpc_template( str(command), requires_token, args ) new_command.__doc__ = docs setattr(self, cmd_name, new_command) self.remote_methods.append(command) def rpc_template(self, command, requires_token, args=None): """ Template function to create new RPC shell commands """ if requires_token and \ (self._claimer is None or self._claimer.token is None): print("Cannot execute `{}' -- no claim available!") return try: if args or requires_token: expanded_args = self.expand_args(args) if requires_token: expanded_args.insert(0, self._claimer.token) response = self.client.call(command, *expanded_args) else: response = self.client.call(command) except RPCError as ex: print("RPC Command failed!") print("Error: {}".format(ex)) return except Exception as ex: print("Unexpected exception!") print("Error: {}".format(ex)) return if isinstance(response, bool): if response: print("Command executed successfully!") else: print("Command failed!") else: print("==> " + str(response)) return response def get_names(self): " We need this for tab completion. " return dir(self) ########################################################################### # Cmd module specific ########################################################################### def run(self): " Go, go, go! " try: self.cmdloop() except KeyboardInterrupt: self.do_disconnect(None) exit(0) def postcmd(self, stop, line): """ Is run after every command executes. Does: - Update prompt """ self.update_prompt() ########################################################################### # Internal methods ########################################################################### def connect(self, host, port): """ Launch a connection. """ print("Attempting to connect to {host}:{port}...".format( host=host, port=port )) try: self.client = RPCClient(host, port, pack_params={'use_bin_type': True}) print("Connection successful.") except Exception as ex: print("Connection refused") print("Error: {}".format(ex)) return False self._host = host self._port = port print("Getting methods...") methods = self.client.call('list_methods') for method in methods: self._add_command(*method) print("Added {} methods.".format(len(methods))) print("Quering device info...") self._device_info = self.client.call('get_device_info') return True def disconnect(self): """ Clean up after a connection was closed. """ self._device_info = None if self._claimer is not None: self._claimer.unclaim() if self.client: try: self.client.close() except RPCError as ex: print("Error while closing the connection") print("Error: {}".format(ex)) for method in self.remote_methods: delattr(self, "do_" + method) self.remote_methods = [] self.client = None self._host = None self._port = None def claim(self): " Initialize claim " assert self.client is not None if self._claimer is not None: print("Claimer already active.") return True print("Claiming device...") self._claimer = MPMClaimer(self._host, self._port, self.unclaim_hook) return True def hijack(self, token): " Hijack running session " assert self.client is not None if self._claimer is not None: print("Claimer already active. Can't hijack.") return False print("Hijacking device...") self._claimer = MPMHijacker(token) return True def unclaim(self): """ unclaim """ if self._claimer is not None: self._claimer.unclaim() self._claimer = None def unclaim_hook(self): """ Hook """ pass def update_prompt(self): """ Update prompt """ if self._device_info is None: self.prompt = '> ' else: if self._claimer is None: claim_status = '' elif isinstance(self._claimer, MPMClaimer): claim_status = ' [C]' elif isinstance(self._claimer, MPMHijacker): claim_status = ' [H]' self.prompt = '{dev_id}{claim_status}> '.format( dev_id=self._device_info.get( 'name', self._device_info.get('serial', '?') ), claim_status=claim_status, ) def expand_args(self, args): """ Takes a string and returns a list """ if self._claimer is not None and self._claimer.token is not None: args = args.replace('$T', str(self._claimer.token)) eval_preamble = '=' args = args.strip() if args.startswith(eval_preamble): parsed_args = eval(args.lstrip(eval_preamble)) if not isinstance(parsed_args, list): parsed_args = [parsed_args] else: parsed_args = [] for arg in args.split(): try: parsed_args.append(int(arg, 0)) continue except ValueError: pass try: parsed_args.append(float(arg)) continue except ValueError: pass parsed_args.append(arg) return parsed_args ########################################################################### # Predefined commands ########################################################################### def do_connect(self, args): """ Connect to a remote MPM server. See connect() """ host, port = split_args(args, 'localhost', MPM_RPC_PORT) port = int(port) self.connect(host, port) def do_claim(self, _): """ Spawn a claim loop """ self.claim() def do_hijack(self, token): """ Hijack a running session """ self.hijack(token) def do_unclaim(self, _): """ unclaim """ self.unclaim() def do_disconnect(self, _): """ disconnect from the RPC server """ self.disconnect() def do_import(self, args): """import a python module into the global namespace""" globals()[args] = import_module(args) def do_EOF(self, _): " When catching EOF, exit the program. " print("Exiting...") self.disconnect() exit(0)
def callraspi(): client = RPCClient("133.27.171.245", 6000) print(client.call('unlock_key'))
class URLClientRemote(object): """ The URLClient takes URLs and sends them to the URLServer to get their IDs or fields We should migrate to something more efficient (but more complex!) like gRPC in the future. """ urlserver = None def connect(self): self.urlserver = RPCClient( config["URLSERVER"].split(":")[0], int(config["URLSERVER"].split(":")[1]) ) def empty(self): pass def get_id(self, url): """ Simple API to get the ID of an URL """ return self.get_ids([url])[0] def get_domain_id(self, url): """ Simple API to get the ID of a domain """ return self.get_domain_ids([url])[0] def _rpc(self, method, *args): return self.urlserver.call(method, *args) def get_ids(self, urls): """ Returns an array of 64-bit integers for each URL """ if len(urls) == 0: return [] if isinstance(urls[0], URL): urls = [u.url for u in urls] return self._rpc("get_ids", urls) def get_domain_ids(self, urls): """ Returns an array of 64-bit integers for each domain """ if len(urls) == 0: return [] if isinstance(urls[0], URL): urls = [u.url for u in urls] return self._rpc("get_domain_ids", urls) def get_metadata(self, urls): """ Returns an array of complete fields for each URL, for each main variation of those URLs """ if len(urls) == 0: return [] if not isinstance(urls[0], URL): urls = [URL(u) for u in urls] # Multiple variations of the URLs to be tested. It is slightly inefficient, but we # send the 4 variations in the same request and split them just below. urls_variations = [] for url in urls: urls_variations.extend([ url.normalized, url.normalized_without_query, url.normalized_domain, url.pld ]) res = self._rpc("get_metadata", urls_variations) ret = [] for url_i in py2_xrange(len(res) // 4): url_metadata = {} for i, key in ((0, "url"), (1, "url_without_query"), (2, "domain"), (3, "pld")): url_metadata[key] = urlserver_pb2.UrlMetadata() url_metadata[key].ParseFromString(res[(url_i * 4) + i]) ret.append(url_metadata) return ret
class URLClientRemote(object): """ The URLClient takes URLs and sends them to the URLServer to get their IDs or fields We should migrate to something more efficient (but more complex!) like gRPC in the future. """ urlserver = None def connect(self): self.urlserver = RPCClient(config["URLSERVER"].split(":")[0], int(config["URLSERVER"].split(":")[1])) def empty(self): pass def get_id(self, url): """ Simple API to get the ID of an URL """ return self.get_ids([url])[0] def get_domain_id(self, url): """ Simple API to get the ID of a domain """ return self.get_domain_ids([url])[0] def _rpc(self, method, *args): return self.urlserver.call(method, *args) def get_ids(self, urls): """ Returns an array of 64-bit integers for each URL """ if len(urls) == 0: return [] if isinstance(urls[0], URL): urls = [u.url for u in urls] return self._rpc("get_ids", urls) def get_domain_ids(self, urls): """ Returns an array of 64-bit integers for each domain """ if len(urls) == 0: return [] if isinstance(urls[0], URL): urls = [u.url for u in urls] return self._rpc("get_domain_ids", urls) def get_metadata(self, urls): """ Returns an array of complete fields for each URL, for each main variation of those URLs """ if len(urls) == 0: return [] if not isinstance(urls[0], URL): urls = [URL(u) for u in urls] # Multiple variations of the URLs to be tested. It is slightly inefficient, but we # send the 4 variations in the same request and split them just below. urls_variations = [] for url in urls: urls_variations.extend([ url.normalized, url.normalized_without_query, url.normalized_domain, url.pld ]) res = self._rpc("get_metadata", urls_variations) ret = [] for url_i in py2_xrange(len(res) // 4): url_metadata = {} for i, key in ((0, "url"), (1, "url_without_query"), (2, "domain"), (3, "pld")): url_metadata[key] = urlserver_pb2.UrlMetadata() url_metadata[key].ParseFromString(res[(url_i * 4) + i]) ret.append(url_metadata) return ret
# -*- coding: utf-8 -*- """如果想用gevent, 可以用gevent patch all """ from mprpc import RPCClient client = RPCClient('tcp://127.0.0.1:6000', reconnect_delay=5) print client.call('sum', 1, 2)
class URLClientRemote(object): """ The URLClient takes URLs and sends them to the URLServer to get their IDs or fields We should migrate to something more efficient (but more complex!) like gRPC in the future. """ urlserver = None def connect(self): self.urlserver = RPCClient( config["URLSERVER"].split(":")[0], int(config["URLSERVER"].split(":")[1]) ) def empty(self): pass def get_id(self, url): """ Simple API to get the ID of an URL """ return self.get_ids([url])[0] def get_domain_id(self, url): """ Simple API to get the ID of a domain """ return self.get_domain_ids([url])[0] def _rpc(self, method, *args): return self.urlserver.call(method, *args) def get_ids(self, urls): """ Returns an array of 64-bit integers for each URL """ if len(urls) == 0: return [] if isinstance(urls[0], URL): urls = [u.url for u in urls] return self._rpc("get_ids", urls) def get_domain_ids(self, urls): """ Returns an array of 64-bit integers for each domain """ if len(urls) == 0: return [] if isinstance(urls[0], URL): urls = [u.url for u in urls] return self._rpc("get_domain_ids", urls) def get_metadata(self, urls): """ Returns an array of complete fields for each URL """ if len(urls) == 0: return [] if isinstance(urls[0], URL): urls = [u.url for u in urls] res = self._rpc("get_metadata", urls) ret = [] for row in res: d = {} for i, f in enumerate(URLSERVER_FIELDNAMES["get_metadata"]): d[f] = row[i] ret.append(d) return ret
def call(): client = RPCClient('127.0.0.1', 6000) print client.call('sum', 1, 2)
class MPMShell(cmd.Cmd): """ RPC Shell class. See cmd module. """ def __init__(self): cmd.Cmd.__init__(self) self.prompt = "> " self.client = None self.remote_methods = [] self._claimer = None self._host = None self._port = None self._device_info = None args = parse_args() if args.host is not None: self.connect(args.host, args.port) if args.claim: self.claim() elif args.hijack: self.hijack(args.hijack) self.update_prompt() def _add_command(self, command, docs, requires_token=False): """ Add a command to the current session """ cmd_name = 'do_' + command if not hasattr(self, cmd_name): new_command = lambda args: self.rpc_template( str(command), requires_token, args ) new_command.__doc__ = docs setattr(self, cmd_name, new_command) self.remote_methods.append(command) def rpc_template(self, command, requires_token, args=None): """ Template function to create new RPC shell commands """ if requires_token and \ (self._claimer is None or self._claimer.token is None): print("Cannot execute `{}' -- no claim available!") return try: if args or requires_token: expanded_args = self.expand_args(args) if requires_token: expanded_args.insert(0, self._claimer.token) response = self.client.call(command, *expanded_args) else: response = self.client.call(command) except RPCError as ex: print("RPC Command failed!") print("Error: {}".format(ex)) return except Exception as ex: print("Unexpected exception!") print("Error: {}".format(ex)) return if isinstance(response, bool): if response: print("Command executed successfully!") return print("Command failed!") return print("==> " + str(response)) def get_names(self): " We need this for tab completion. " return dir(self) ########################################################################### # Cmd module specific ########################################################################### def run(self): " Go, go, go! " try: self.cmdloop() except KeyboardInterrupt: self.do_disconnect(None) exit(0) def postcmd(self, stop, line): """ Is run after every command executes. Does: - Update prompt """ self.update_prompt() ########################################################################### # Internal methods ########################################################################### def connect(self, host, port): """ Launch a connection. """ print("Attempting to connect to {host}:{port}...".format( host=host, port=port )) try: self.client = RPCClient(host, port, pack_params={'use_bin_type': True}) print("Connection successful.") except Exception as ex: print("Connection refused") print("Error: {}".format(ex)) return False self._host = host self._port = port print("Getting methods...") methods = self.client.call('list_methods') for method in methods: self._add_command(*method) print("Added {} methods.".format(len(methods))) print("Quering device info...") self._device_info = self.client.call('get_device_info') return True def disconnect(self): """ Clean up after a connection was closed. """ self._device_info = None if self._claimer is not None: self._claimer.unclaim() if self.client: try: self.client.close() except RPCError as ex: print("Error while closing the connection") print("Error: {}".format(ex)) for method in self.remote_methods: delattr(self, "do_" + method) self.remote_methods = [] self.client = None self._host = None self._port = None def claim(self): " Initialize claim " assert self.client is not None if self._claimer is not None: print("Claimer already active.") return True print("Claiming device...") self._claimer = MPMClaimer(self._host, self._port, self.unclaim_hook) return True def hijack(self, token): " Hijack running session " assert self.client is not None if self._claimer is not None: print("Claimer already active. Can't hijack.") return False print("Hijacking device...") self._claimer = MPMHijacker(token) return True def unclaim(self): """ unclaim """ if self._claimer is not None: self._claimer.unclaim() self._claimer = None def unclaim_hook(self): """ Hook """ pass def update_prompt(self): """ Update prompt """ if self._device_info is None: self.prompt = '> ' else: if self._claimer is None: claim_status = '' elif isinstance(self._claimer, MPMClaimer): claim_status = ' [C]' elif isinstance(self._claimer, MPMHijacker): claim_status = ' [H]' self.prompt = '{dev_id}{claim_status}> '.format( dev_id=self._device_info.get( 'name', self._device_info.get('serial', '?') ), claim_status=claim_status, ) def expand_args(self, args): """ Takes a string and returns a list """ if self._claimer is not None and self._claimer.token is not None: args = args.replace('$T', str(self._claimer.token)) eval_preamble = '=' args = args.strip() if args.startswith(eval_preamble): parsed_args = eval(args.lstrip(eval_preamble)) if not isinstance(parsed_args, list): parsed_args = [parsed_args] else: parsed_args = [] for arg in args.split(): try: parsed_args.append(int(arg, 0)) continue except ValueError: pass try: parsed_args.append(float(arg)) continue except ValueError: pass parsed_args.append(arg) return parsed_args ########################################################################### # Predefined commands ########################################################################### def do_connect(self, args): """ Connect to a remote MPM server. See connect() """ host, port = split_args(args, 'localhost', MPM_RPC_PORT) port = int(port) self.connect(host, port) def do_claim(self, _): """ Spawn a claim loop """ self.claim() def do_hijack(self, token): """ Hijack a running session """ self.hijack(token) def do_unclaim(self, _): """ unclaim """ self.unclaim() def do_disconnect(self, _): """ disconnect from the RPC server """ self.disconnect() def do_import(self, args): """import a python module into the global namespace""" globals()[args] = import_module(args) def do_EOF(self, _): " When catching EOF, exit the program. " print("Exiting...") self.disconnect() exit(0)
class Client(BaseClient): def __init__(self, host="127.0.0.1", port=9033, model="en", embeddings_path=None, verbose=False, attributes=None): super(Client, self).__init__(model, embeddings_path) self.host = host self.port = port self.rpc = RPCClient(host, port) self.verbose = verbose self.attributes = attributes self.cache = LRUCache(maxsize=3000000) def _call(self, path, *args): return self.rpc.call(path, *args) def most_similar(self, word, n=10): key = (word, n, self.model, self.embeddings_path) if key not in self.cache: result = self._call("most_similar", word, n, self.model, self.embeddings_path) self.cache[key] = result return self.cache[key] def single(self, document, attributes=None): attributes = attributes or tuple(self.attributes) key = (document, attributes, self.model, self.embeddings_path) if key not in self.cache: sentences = self._call("single", document, self.model, self.embeddings_path, attributes) result = SpacyClientDocument(sentences) self.cache[key] = result return result return self.cache[key] def _bulk(self, documents, attributes): attributes = attributes or tuple(self.attributes) done_docs = [(num, self.cache[(document, attributes)]) for num, document in enumerate(documents) if (document, attributes) in self.cache] todo_docs = [(num, document) for num, document in enumerate(documents) if (document, attributes) not in self.cache] todo_inds = [x[0] for x in todo_docs] todo_docs = [x[1] for x in todo_docs] if todo_docs: todo_docs = self._call("bulk", todo_docs, self.model, self.embeddings_path, attributes) todo_docs = [(x, SpacyClientDocument(y)) for x, y in zip(todo_inds, todo_docs)] return [x[1] for x in sorted(todo_docs + done_docs)] def bulk(self, documents, batch_size=1000, attributes=None): attributes = attributes or tuple(self.attributes) parsed_documents = [] if len(documents) > batch_size: batches = int(math.ceil(len(documents) / batch_size)) print("Batching {} requests with batch_size {}".format( batches, batch_size)) if self.verbose: batch_iterator = tqdm.tqdm(range(batches)) else: batch_iterator = range(batches) for b in batch_iterator: docs = documents[b * batch_size:(b + 1) * batch_size] res = self._bulk(docs, attributes) parsed_documents.extend(res) else: parsed_documents = self._bulk(documents, attributes) return parsed_documents def testfunc(self, document): print('Entities', [(ent.text, ent.label_) for ent in document.ents]) print("JSON FORMAT ---> ", json.dumps([(ent.text, ent.label_) for ent in document.ents]))
class URLClientRemote(object): """ The URLClient takes URLs and sends them to the URLServer to get their IDs or fields We should migrate to something more efficient (but more complex!) like gRPC in the future. """ urlserver = None def connect(self): self.urlserver = RPCClient(config["URLSERVER"].split(":")[0], int(config["URLSERVER"].split(":")[1])) def empty(self): pass def get_id(self, url): """ Simple API to get the ID of an URL """ return self.get_ids([url])[0] def get_domain_id(self, url): """ Simple API to get the ID of a domain """ return self.get_domain_ids([url])[0] def _rpc(self, method, *args): return self.urlserver.call(method, *args) def get_ids(self, urls): """ Returns an array of 64-bit integers for each URL """ if len(urls) == 0: return [] if isinstance(urls[0], URL): urls = [u.url for u in urls] return self._rpc("get_ids", urls) def get_domain_ids(self, urls): """ Returns an array of 64-bit integers for each domain """ if len(urls) == 0: return [] if isinstance(urls[0], URL): urls = [u.url for u in urls] return self._rpc("get_domain_ids", urls) def get_metadata(self, urls): """ Returns an array of complete fields for each URL """ if len(urls) == 0: return [] if isinstance(urls[0], URL): urls = [u.url for u in urls] res = self._rpc("get_metadata", urls) ret = [] for row in res: d = {} for i, f in enumerate(URLSERVER_FIELDNAMES["get_metadata"]): d[f] = row[i] ret.append(d) return ret
import numpy as np from mprpc import RPCClient client = RPCClient('127.0.0.1', 9033) with client_pool.connection() as client: vec = client.call('embedding', ['hey', 'there', 'how', 'are', 'you']) print(np.shape(vec))
#import pyjsonrpc from mprpc import RPCClient ''' from paramiko import SSHClient from scp import SCPClient ssh = SSHClient() ssh.load_system_host_keys() ssh.connect('192.168.1.16', username='******', password='******', look_for_keys=False) with SCPClient(ssh.get_transport()) as scp: scp.put("/home/pi/Desktop/pyUI/profiles/aaa/top/aaa.jpg", "/home/pi/Desktop/aaa.jpg") ''' client = RPCClient('127.0.0.1', 8080) print(client.call('add', 1, 2)) print(client.call('updateProfile', "")) print(client.call('profilepath', '/home/pi/Desktop/pyUI/profiles', 'aaa')) print(client.call('ResultTest', 5)) #client.call('Init') client.call('TakePicture', 0, False) client.call('TakePicture', 1) client.call('TakePicture', 2) client.call('CreateSamplePoint', 0, 150, 200) #client.call('Uninit') client.call('CloseServer') ''' http_client = pyjsonrpc.HttpClient( url = "http://localhost:8080/jsonrpc" #username = "******", #password = "******"
class Proxy(object): def __init__(self, uri, show=False): self.showerr = show try: self.host, self.port = uri.split(":") except: self.host = "0.0.0.0" self.port = 0 self.OK = False self.__linked = False try: self._link_ = RPCClient(self.host, int(self.port), timeout=10) self.__linked = True except: self.__linked = False if show: P_Log("[FR]ERROR [FW] No Link to {}".format(uri)) def _close(self): if self._link_.is_connected(): self._link_.close() def connect(self): try: if not self.OK: self._link_ = RPCClient(self.host, int(self.port), timeout=20) if self._link_.is_connected(): self.OK = self.__create_hooks() else: self.OK = False except: self.OK = False if self.showerr: P_Log("[FR]ERROR [FW] No Connect to {}".format(uri)) finally: return self.OK def __call__(self): return self.connect() def __str__(self): if self.OK: return "{}:{}::Connected".format(*self._link_.getpeername()) elif self.__linked: return "{}:{}::Linked".format(*self._link_.getpeername()) else: return "Disconected 1.1.1.1:1" def __create_hooks(self): try: _config = self._link_.call('G_E_T_Configuration') except: _config = {} #print("error creating hooks {}:{}".format(self.host,self.port)) hooks = [] for defs, params in _config.items(): params_def = params[0] params_call = "" for x in params[1]: params_call = params_call + "," + str(x) d = def_skel.format(defs, params_def, params_call, self.host + ":" + str(self.port)) #print(d) hooks.append((defs, d)) for defs, fun in hooks: exec(fun) self.__dict__[defs] = types.MethodType(eval(defs), self) self.functions = hooks return len(self.functions) > 0 def get_uri(self): return "{}:{}".format(self.host, self.port) def get_functions(self): return [a for a, b in self.functions] def call(self, fn, *args): try: return self._link_.call(fn, *args) except Exception as ex: #raise P_Log("[[FR]ERROR[FW]] Running call on Proxy {}".format( self.get_uri())) P_Log(str(ex)) return None
def main2(): client = RPCClient("localhost", 3000) ans = client.call("sum", 1, 2) print(ans)