class MPMClient:
"""
MPM RPC Client: Will make all MPM commands accessible as Python methods.
"""
def __init__(self, init_mode, host, port, token=None):
assert isinstance(init_mode, InitMode)
print("[MPMRPC] Attempting to connect to {host}:{port}...".format(
host=host, port=port
))
try:
self._client = RPCClient(host, port, pack_params={'use_bin_type': True})
print("[MPMRPC] Connection successful.")
except Exception as ex:
print("[MPMRPC] Connection refused: {}".format(ex))
raise RuntimeError("RPC connection refused.")
self._remote_methods = []
if init_mode == InitMode.Hijack:
assert token
self._token = token
if init_mode == InitMode.Claim:
self._claimer = MPMClaimer(host, port)
self._claimer.claim()
self._token = self._claimer.token
print("[MPMRPC] Getting methods...")
methods = self._client.call('list_methods')
for method in methods:
self._add_command(*method)
print("[MPMRPC] Added {} methods.".format(len(methods)))
def _add_command(self, command, docs, requires_token):
"""
Add a command to the current session
"""
if not hasattr(self, command):
new_command = lambda *args, **kwargs: self._rpc_template(
str(command), requires_token, *args, **kwargs
)
new_command.__doc__ = docs
setattr(self, command, new_command)
self._remote_methods.append(command)
def _rpc_template(self, command, requires_token, *args, **kwargs):
"""
Template function to create new RPC shell commands
"""
if requires_token and not self._token:
raise RuntimeError(
"[MPMRPC] Cannot execute `{}' -- no claim available!"
.format(command))
# Put token as the first argument if required:
if requires_token:
args = (self._token,) + args
if kwargs:
return self._client.call(command, *args, **kwargs)
if args:
return self._client.call(command, *args)
return self._client.call(command)
def rpc(address, port, command, *args):
if not port:
port = mpm.types.MPM_RPC_PORT
client = RPCClient(address, port)
if args:
args = [eval(arg.lstrip("=")) if arg.startswith("=") else arg for arg in args]
result = client.call(command, *args)
else:
result = client.call(command)
return result
def rpc(address, port, command, *args):
if not port:
port = mpm.types.MPM_RPC_PORT
client = RPCClient(address, port)
if args:
args = [
eval(arg.lstrip("=")) if arg.startswith("=") else arg
for arg in args
]
result = client.call(command, *args)
else:
result = client.call(command)
return result
def complete(self, context, prefix):
from mprpc import RPCClient
client = RPCClient('localhost', self.__port)
response = client.call('complete', context, prefix)
return [decode_completion(r) for r in response]
def segment(self, text):
from mprpc import RPCClient
client = RPCClient('localhost', self.__port)
response = client.call('segment', text)
return [decode_segment(r) for r in response]
def claim_loop(self, host, port, disc_callback):
"""
Run a claim loop
"""
client = RPCClient(host, port, pack_params={'use_bin_type': True})
self.token = client.call('claim', 'MPM Shell')
try:
while not self._exit_loop:
client.call('reclaim', self.token)
time.sleep(1)
client.call('unclaim', self.token)
except RPCError as ex:
print("Unexpected RPC error in claimer loop!")
print(str(ex))
disc_callback()
self.token = None
def call():
from mprpc import RPCClient
client = RPCClient('127.0.0.1', 7777)
t = np.zeros(NUM_CALLS)
for _ in range(NUM_CALLS):
t1 = time.time()
client.call('sum', 1, 2)
t2 = time.time()
t[_] = t2 - t1
print('stdev: {}'.format(t.std()))
print('mean: {}'.format(t.mean()))
print('call: {} qps'.format(1 / t.mean()))
def claim_loop(self, host, port, disc_callback):
"""
Run a claim loop
"""
client = RPCClient(host, port, pack_params={'use_bin_type': True})
self.token = client.call('claim', 'MPM Shell')
try:
while not self._exit_loop:
client.call('reclaim', self.token)
time.sleep(1)
client.call('unclaim', self.token)
except RPCError as ex:
print("Unexpected RPC error in claimer loop!")
print(str(ex))
disc_callback()
self.token = None
def call_register(uuid):
print("Registering peer with uuid: {}".format(uuid))
client = RPCClient(b'localhost', 18080)
res = client.call(b'register', uuid)
client.close()
return res
def call_register(uuid):
print("Registering peer with uuid: {}".format(uuid))
client = RPCClient(b'localhost', 18080)
res = client.call(b'register', uuid)
client.close()
return res
def call():
from mprpc import RPCClient
client = RPCClient('ipc://x.sock')
start = time.time()
[client.call('sum', 1, 2) for _ in xrange(NUM_CALLS)]
print 'call: %d qps' % (NUM_CALLS / (time.time() - start))
def call():
from mprpc import RPCClient
client = RPCClient('127.0.0.1', 6000)
start = time.time()
[client.call('sum', 1, 2) for _ in xrange(NUM_CALLS)]
print 'call: %d qps' % (NUM_CALLS / (time.time() - start))
class BitFlyerMarket(BitFlyerMarketBase):
def __init__(self, symbol=SYMBOL, resolution=60, port=PROXY_PORT):
super().__init__(symbol, resolution)
self.client = RPCClient('127.0.0.1', port)
self.data = self.client.call('ohlcv', TICKERID, self.resolution, 256)
def step_ohlcv(self, next_clock):
d1, d0 = self.client.call('step_ohlcv', TICKERID, self.resolution,
next_clock)
if d1 is None:
return None
if d0['t'] <= self.data['t'][-1]:
return None
for k, v in d0.items():
self.data[k].pop(0)
self.data[k][-1] = d1[k]
self.data[k].append(v)
return d0['t']
def call_get_schedule(uuid, num, method="smart"):
print("Getting schedule for uuid: {}".format(uuid))
data = {'uuid': uuid, 'num': num, 'method': method}
client = RPCClient(b'localhost', 18080)
result = client.call(b'get_schedule', data)
client.close()
return result
def call_get_schedule(uuid, num, method="smart"):
print("Getting schedule for uuid: {}".format(uuid))
data = {
'uuid':uuid,
'num':num,
'method':method
}
client = RPCClient(b'localhost', 18080)
result = client.call(b'get_schedule', data)
client.close()
return result
def __exec_mprpc__(self, args):
from mprpc import RPCClient, RPCPoolClient
method = args[0]
args = args[1]
assert method in ["get", "set"]
try:
client = RPCClient(self.server, self.mprpc_port)
result = client.call(method, args)
client.close()
return result
except Exception as e:
print(e)
try:
client.close()
except:
pass
def test_server(self, server):
class Sum:
def sum(self, x, y):
return x + y
server.load(Sum)
self.g = gevent.spawn(server.run)
sleep(0.1)
registry = server.registry
res = registry.discovery(server.name)
key = registry._node_key(server.name, url.get_param("node"))
host, port = res[key].split(":")
client = RPCClient(str(host), int(port))
assert client.call("sum", {}, 1, 2) == 3
class Client(BaseClient):
def __init__(self,
host="127.0.0.1",
port=9033,
model="en",
embeddings_path=None,
verbose=False):
super(Client, self).__init__(model, embeddings_path)
self.host = host
self.port = port
self.rpc = RPCClient(host, port)
self.verbose = verbose
def _call(self, path, *args):
return self.rpc.call(path, *args)
@cachetools.func.lru_cache(maxsize=3000000)
def single(self, document, attributes=None):
sentences = self._call("single", document, self.model,
self.embeddings_path, attributes)
return SpacyClientDocument(sentences)
def _bulk(self, documents, attributes):
return self._call("bulk", documents, self.model, self.embeddings_path,
attributes)
def bulk(self, documents, batch_size=1000, attributes=None):
parsed_documents = []
if len(documents) > batch_size:
batches = int(math.ceil(len(documents) / batch_size))
print("Batching {} requests with batch_size {}".format(
batches, batch_size))
if self.verbose:
batch_iterator = tqdm.tqdm(range(batches))
else:
batch_iterator = range(batches)
for b in batch_iterator:
docs = documents[b * batch_size:(b + 1) * batch_size]
res = self._bulk(docs, attributes)
parsed_documents.extend(res)
else:
parsed_documents = self._bulk(documents, attributes)
return [SpacyClientDocument(x) for x in parsed_documents]
def _claim_loop(host, port, cmd_q, token_q):
"""
Process that runs a claim loop.
This function should be run in its own thread. Communication to the outside
thread happens with two queues: A command queue, and a token queue. The
command queue is used to pass in one of these commands: claim, unclaim, or
exit.
The token queue is used to read back the current token.
"""
command = None
token = None
exit_loop = False
client = RPCClient(host, port, pack_params={'use_bin_type': True})
try:
while not exit_loop:
if token and not command:
client.call('reclaim', token)
elif command == 'claim':
if not token:
token = client.call('claim', 'UHD')
else:
print("Already have claim")
token_q.put(token)
elif command == 'unclaim':
if token:
client.call('unclaim', token)
token = None
token_q.put(None)
elif command == 'exit':
if token:
client.call('unclaim', token)
token = None
token_q.put(None)
exit_loop = True
time.sleep(1)
command = None
if not cmd_q.empty():
command = cmd_q.get(False)
except RPCError as ex:
print("Unexpected RPC error in claimer loop!")
print(str(ex))
def claim_loop(self, host, port, cmd_q, token_q):
"""
Run a claim loop
"""
from mprpc import RPCClient
from mprpc.exceptions import RPCError
command = None
token = None
exit_loop = False
client = RPCClient(host, port, pack_params={'use_bin_type': True})
try:
while not exit_loop:
if token and not command:
client.call('reclaim', token)
elif command == 'claim':
if not token:
token = client.call('claim', 'MPM Shell')
else:
print("Already have claim")
token_q.put(token)
elif command == 'unclaim':
if token:
client.call('unclaim', token)
token = None
token_q.put(None)
elif command == 'exit':
if token:
client.call('unclaim', token)
token = None
token_q.put(None)
exit_loop = True
time.sleep(1)
command = None
if not cmd_q.empty():
command = cmd_q.get(False)
except RPCError as ex:
print("Unexpected RPC error in claimer loop!")
print(str(ex))
from mprpc import RPCClient
client = RPCClient('localhost', 6666 )
#o = client.call('sum', 1, 2)
s = client.call('sleep', 10)
def call():
client = RPCClient('127.0.0.1', 6000)
client.open()
print client.call('sum', 1, 2)
from mprpc import RPCClient
client = RPCClient('127.0.0.1', 6000)
print client.call('add', 1000, 1000)
class MPMShell(cmd.Cmd):
"""
RPC Shell class. See cmd module.
"""
def __init__(self, host, port, claim, hijack, script):
cmd.Cmd.__init__(self)
self.prompt = "> "
self.client = None
self.remote_methods = []
self._host = host
self._port = port
self._device_info = None
self._claimer = MPMClaimer(self._host, self._port)
if host is not None:
self.connect(host, port)
if claim:
self.claim()
elif hijack:
self.hijack(hijack)
self.update_prompt()
self._script = script
if self._script:
self.parse_script()
def _add_command(self, command, docs, requires_token=False):
"""
Add a command to the current session
"""
cmd_name = 'do_' + command
if not hasattr(self, cmd_name):
new_command = lambda args: self.rpc_template(
str(command), requires_token, args)
new_command.__doc__ = docs
setattr(self, cmd_name, new_command)
self.remote_methods.append(command)
def _print_response(self, response):
print(re.sub("^", "< ", response, flags=re.MULTILINE))
def rpc_template(self, command, requires_token, args=None):
"""
Template function to create new RPC shell commands
"""
from mprpc.exceptions import RPCError
if requires_token and \
(self._claimer is None or self._claimer.get_token() is None):
self._print_response("Cannot execute `{}' -- "
"no claim available!".format(command))
return False
try:
if args or requires_token:
expanded_args = self.expand_args(args)
if requires_token:
expanded_args.insert(0, self._claimer.get_token())
response = self.client.call(command, *expanded_args)
else:
response = self.client.call(command)
except RPCError as ex:
self._print_response("RPC Command failed!\nError: {}".format(ex))
return False
except Exception as ex:
self._print_response("Unexpected exception!\nError: {}".format(ex))
return True
if isinstance(response, bool):
if response:
self._print_response("Command succeeded.")
else:
self._print_response("Command failed!")
else:
self._print_response(str(response))
return False
def get_names(self):
" We need this for tab completion. "
return dir(self)
###########################################################################
# Cmd module specific
###########################################################################
def default(self, line):
self._print_response("*** Unknown syntax: %s" % line)
def preloop(self):
"""
In script mode add Execution start marker to ease parsing script output
:return: None
"""
if self._script:
print("Execute %s" % self._script)
def precmd(self, line):
"""
Add command prepended by "> " in scripting mode to ease parsing script
output.
"""
if self.cmdqueue:
print("> %s" % line)
return line
def postcmd(self, stop, line):
"""
Is run after every command executes. Does:
- Update prompt
"""
self.update_prompt()
return stop
###########################################################################
# Internal methods
###########################################################################
def connect(self, host, port):
"""
Launch a connection.
"""
from mprpc import RPCClient
print("Attempting to connect to {host}:{port}...".format(host=host,
port=port))
try:
self.client = RPCClient(host,
port,
pack_params={'use_bin_type': True})
print("Connection successful.")
except Exception as ex:
print("Connection refused")
print("Error: {}".format(ex))
return False
self._host = host
self._port = port
print("Getting methods...")
methods = self.client.call('list_methods')
for method in methods:
self._add_command(*method)
print("Added {} methods.".format(len(methods)))
print("Quering device info...")
self._device_info = self.client.call('get_device_info')
return True
def disconnect(self):
"""
Clean up after a connection was closed.
"""
from mprpc.exceptions import RPCError
self._device_info = None
if self._claimer is not None:
self._claimer.exit()
if self.client:
try:
self.client.close()
except RPCError as ex:
print("Error while closing the connection")
print("Error: {}".format(ex))
for method in self.remote_methods:
delattr(self, "do_" + method)
self.remote_methods = []
self.client = None
self._host = None
self._port = None
def claim(self):
" Initialize claim "
print("Claiming device...")
self._claimer.claim()
return True
def hijack(self, token):
" Hijack running session "
if self._claimer.hijacked:
print("Claimer already active. Can't hijack.")
return False
print("Hijacking device...")
self._claimer.hijack(token)
return True
def unclaim(self):
"""
unclaim
"""
self._claimer.unclaim()
def update_prompt(self):
"""
Update prompt
"""
if self._device_info is None:
self.prompt = '> '
else:
token = self._claimer.get_token()
if token is None:
claim_status = ''
elif self._claimer.hijacked:
claim_status = ' [H]'
else:
claim_status = ' [C]'
self.prompt = '{dev_id}{claim_status}> '.format(
dev_id=self._device_info.get(
'name', self._device_info.get('serial', '?')),
claim_status=claim_status,
)
def parse_script(self):
"""
Adding script command from file pointed to by self._script.
The commands are read from file one per line and added to cmdqueue of
parent class. This way they will be executed instead of input from
stdin. An EOF command is appended to the list to ensure the shell exits
after script execution.
:return: None
"""
try:
with open(self._script, "r") as script:
for command in script:
self.cmdqueue.append(command.strip())
except OSError as ex:
print("Failed to read script. (%s)" % ex)
self.cmdqueue.append("EOF") # terminate shell after script execution
def expand_args(self, args):
"""
Takes a string and returns a list
"""
if self._claimer is not None and self._claimer.get_token() is not None:
args = args.replace('$T', str(self._claimer.get_token()))
eval_preamble = '='
args = args.strip()
if args.startswith(eval_preamble):
parsed_args = eval(args.lstrip(eval_preamble))
if not isinstance(parsed_args, list):
parsed_args = [parsed_args]
else:
parsed_args = []
for arg in args.split():
try:
parsed_args.append(int(arg, 0))
continue
except ValueError:
pass
try:
parsed_args.append(float(arg))
continue
except ValueError:
pass
parsed_args.append(arg)
return parsed_args
###########################################################################
# Predefined commands
###########################################################################
def do_connect(self, args):
"""
Connect to a remote MPM server. See connect()
"""
host, port = split_args(args, 'localhost', MPM_RPC_PORT)
port = int(port)
self.connect(host, port)
def do_claim(self, _):
"""
Spawn a claim loop
"""
self.claim()
def do_hijack(self, token):
"""
Hijack a running session
"""
self.hijack(token)
def do_unclaim(self, _):
"""
unclaim
"""
self.unclaim()
def do_disconnect(self, _):
"""
disconnect from the RPC server
"""
self.disconnect()
def do_import(self, args):
"""import a python module into the global namespace"""
globals()[args] = import_module(args)
# pylint: disable=invalid-name
def do_EOF(self, _):
"""
When catching EOF, exit the program.
"""
print("Exiting...")
self.disconnect()
return True # orderly shutdown
def call():
client = RPCClient('ipc://x.sock', reconnect_delay=5)
print client.call('sum', 1, 2)
class MPMShell(cmd.Cmd):
"""
RPC Shell class. See cmd module.
"""
def __init__(self, host, port, claim, hijack):
cmd.Cmd.__init__(self)
self.prompt = "> "
self.client = None
self.remote_methods = []
self._claimer = None
self._host = host
self._port = port
self._device_info = None
if host is not None:
self.connect(host, port)
if claim:
self.claim()
elif hijack:
self.hijack(hijack)
self.update_prompt()
def _add_command(self, command, docs, requires_token=False):
"""
Add a command to the current session
"""
cmd_name = 'do_' + command
if not hasattr(self, cmd_name):
new_command = lambda args: self.rpc_template(
str(command), requires_token, args
)
new_command.__doc__ = docs
setattr(self, cmd_name, new_command)
self.remote_methods.append(command)
def rpc_template(self, command, requires_token, args=None):
"""
Template function to create new RPC shell commands
"""
if requires_token and \
(self._claimer is None or self._claimer.token is None):
print("Cannot execute `{}' -- no claim available!")
return
try:
if args or requires_token:
expanded_args = self.expand_args(args)
if requires_token:
expanded_args.insert(0, self._claimer.token)
response = self.client.call(command, *expanded_args)
else:
response = self.client.call(command)
except RPCError as ex:
print("RPC Command failed!")
print("Error: {}".format(ex))
return
except Exception as ex:
print("Unexpected exception!")
print("Error: {}".format(ex))
return
if isinstance(response, bool):
if response:
print("Command executed successfully!")
else:
print("Command failed!")
else:
print("==> " + str(response))
return response
def get_names(self):
" We need this for tab completion. "
return dir(self)
###########################################################################
# Cmd module specific
###########################################################################
def run(self):
" Go, go, go! "
try:
self.cmdloop()
except KeyboardInterrupt:
self.do_disconnect(None)
exit(0)
def postcmd(self, stop, line):
"""
Is run after every command executes. Does:
- Update prompt
"""
self.update_prompt()
###########################################################################
# Internal methods
###########################################################################
def connect(self, host, port):
"""
Launch a connection.
"""
print("Attempting to connect to {host}:{port}...".format(
host=host, port=port
))
try:
self.client = RPCClient(host, port, pack_params={'use_bin_type': True})
print("Connection successful.")
except Exception as ex:
print("Connection refused")
print("Error: {}".format(ex))
return False
self._host = host
self._port = port
print("Getting methods...")
methods = self.client.call('list_methods')
for method in methods:
self._add_command(*method)
print("Added {} methods.".format(len(methods)))
print("Quering device info...")
self._device_info = self.client.call('get_device_info')
return True
def disconnect(self):
"""
Clean up after a connection was closed.
"""
self._device_info = None
if self._claimer is not None:
self._claimer.unclaim()
if self.client:
try:
self.client.close()
except RPCError as ex:
print("Error while closing the connection")
print("Error: {}".format(ex))
for method in self.remote_methods:
delattr(self, "do_" + method)
self.remote_methods = []
self.client = None
self._host = None
self._port = None
def claim(self):
" Initialize claim "
assert self.client is not None
if self._claimer is not None:
print("Claimer already active.")
return True
print("Claiming device...")
self._claimer = MPMClaimer(self._host, self._port, self.unclaim_hook)
return True
def hijack(self, token):
" Hijack running session "
assert self.client is not None
if self._claimer is not None:
print("Claimer already active. Can't hijack.")
return False
print("Hijacking device...")
self._claimer = MPMHijacker(token)
return True
def unclaim(self):
"""
unclaim
"""
if self._claimer is not None:
self._claimer.unclaim()
self._claimer = None
def unclaim_hook(self):
"""
Hook
"""
pass
def update_prompt(self):
"""
Update prompt
"""
if self._device_info is None:
self.prompt = '> '
else:
if self._claimer is None:
claim_status = ''
elif isinstance(self._claimer, MPMClaimer):
claim_status = ' [C]'
elif isinstance(self._claimer, MPMHijacker):
claim_status = ' [H]'
self.prompt = '{dev_id}{claim_status}> '.format(
dev_id=self._device_info.get(
'name', self._device_info.get('serial', '?')
),
claim_status=claim_status,
)
def expand_args(self, args):
"""
Takes a string and returns a list
"""
if self._claimer is not None and self._claimer.token is not None:
args = args.replace('$T', str(self._claimer.token))
eval_preamble = '='
args = args.strip()
if args.startswith(eval_preamble):
parsed_args = eval(args.lstrip(eval_preamble))
if not isinstance(parsed_args, list):
parsed_args = [parsed_args]
else:
parsed_args = []
for arg in args.split():
try:
parsed_args.append(int(arg, 0))
continue
except ValueError:
pass
try:
parsed_args.append(float(arg))
continue
except ValueError:
pass
parsed_args.append(arg)
return parsed_args
###########################################################################
# Predefined commands
###########################################################################
def do_connect(self, args):
"""
Connect to a remote MPM server. See connect()
"""
host, port = split_args(args, 'localhost', MPM_RPC_PORT)
port = int(port)
self.connect(host, port)
def do_claim(self, _):
"""
Spawn a claim loop
"""
self.claim()
def do_hijack(self, token):
"""
Hijack a running session
"""
self.hijack(token)
def do_unclaim(self, _):
"""
unclaim
"""
self.unclaim()
def do_disconnect(self, _):
"""
disconnect from the RPC server
"""
self.disconnect()
def do_import(self, args):
"""import a python module into the global namespace"""
globals()[args] = import_module(args)
def do_EOF(self, _):
" When catching EOF, exit the program. "
print("Exiting...")
self.disconnect()
exit(0)
def callraspi():
client = RPCClient("133.27.171.245", 6000)
print(client.call('unlock_key'))
class URLClientRemote(object):
""" The URLClient takes URLs and sends them to the URLServer to get their IDs or fields
We should migrate to something more efficient (but more complex!) like gRPC in the future.
"""
urlserver = None
def connect(self):
self.urlserver = RPCClient(
config["URLSERVER"].split(":")[0],
int(config["URLSERVER"].split(":")[1])
)
def empty(self):
pass
def get_id(self, url):
""" Simple API to get the ID of an URL """
return self.get_ids([url])[0]
def get_domain_id(self, url):
""" Simple API to get the ID of a domain """
return self.get_domain_ids([url])[0]
def _rpc(self, method, *args):
return self.urlserver.call(method, *args)
def get_ids(self, urls):
""" Returns an array of 64-bit integers for each URL """
if len(urls) == 0:
return []
if isinstance(urls[0], URL):
urls = [u.url for u in urls]
return self._rpc("get_ids", urls)
def get_domain_ids(self, urls):
""" Returns an array of 64-bit integers for each domain """
if len(urls) == 0:
return []
if isinstance(urls[0], URL):
urls = [u.url for u in urls]
return self._rpc("get_domain_ids", urls)
def get_metadata(self, urls):
""" Returns an array of complete fields for each URL, for each main variation of those URLs """
if len(urls) == 0:
return []
if not isinstance(urls[0], URL):
urls = [URL(u) for u in urls]
# Multiple variations of the URLs to be tested. It is slightly inefficient, but we
# send the 4 variations in the same request and split them just below.
urls_variations = []
for url in urls:
urls_variations.extend([
url.normalized,
url.normalized_without_query,
url.normalized_domain,
url.pld
])
res = self._rpc("get_metadata", urls_variations)
ret = []
for url_i in py2_xrange(len(res) // 4):
url_metadata = {}
for i, key in ((0, "url"), (1, "url_without_query"), (2, "domain"), (3, "pld")):
url_metadata[key] = urlserver_pb2.UrlMetadata()
url_metadata[key].ParseFromString(res[(url_i * 4) + i])
ret.append(url_metadata)
return ret
class URLClientRemote(object):
""" The URLClient takes URLs and sends them to the URLServer to get their IDs or fields
We should migrate to something more efficient (but more complex!) like gRPC in the future.
"""
urlserver = None
def connect(self):
self.urlserver = RPCClient(config["URLSERVER"].split(":")[0],
int(config["URLSERVER"].split(":")[1]))
def empty(self):
pass
def get_id(self, url):
""" Simple API to get the ID of an URL """
return self.get_ids([url])[0]
def get_domain_id(self, url):
""" Simple API to get the ID of a domain """
return self.get_domain_ids([url])[0]
def _rpc(self, method, *args):
return self.urlserver.call(method, *args)
def get_ids(self, urls):
""" Returns an array of 64-bit integers for each URL """
if len(urls) == 0:
return []
if isinstance(urls[0], URL):
urls = [u.url for u in urls]
return self._rpc("get_ids", urls)
def get_domain_ids(self, urls):
""" Returns an array of 64-bit integers for each domain """
if len(urls) == 0:
return []
if isinstance(urls[0], URL):
urls = [u.url for u in urls]
return self._rpc("get_domain_ids", urls)
def get_metadata(self, urls):
""" Returns an array of complete fields for each URL, for each main variation of those URLs """
if len(urls) == 0:
return []
if not isinstance(urls[0], URL):
urls = [URL(u) for u in urls]
# Multiple variations of the URLs to be tested. It is slightly inefficient, but we
# send the 4 variations in the same request and split them just below.
urls_variations = []
for url in urls:
urls_variations.extend([
url.normalized, url.normalized_without_query,
url.normalized_domain, url.pld
])
res = self._rpc("get_metadata", urls_variations)
ret = []
for url_i in py2_xrange(len(res) // 4):
url_metadata = {}
for i, key in ((0, "url"), (1, "url_without_query"), (2, "domain"),
(3, "pld")):
url_metadata[key] = urlserver_pb2.UrlMetadata()
url_metadata[key].ParseFromString(res[(url_i * 4) + i])
ret.append(url_metadata)
return ret
# -*- coding: utf-8 -*-
"""如果想用gevent, 可以用gevent patch all
"""
from mprpc import RPCClient
client = RPCClient('tcp://127.0.0.1:6000', reconnect_delay=5)
print client.call('sum', 1, 2)
class URLClientRemote(object):
""" The URLClient takes URLs and sends them to the URLServer to get their IDs or fields
We should migrate to something more efficient (but more complex!) like gRPC in the future.
"""
urlserver = None
def connect(self):
self.urlserver = RPCClient(
config["URLSERVER"].split(":")[0],
int(config["URLSERVER"].split(":")[1])
)
def empty(self):
pass
def get_id(self, url):
""" Simple API to get the ID of an URL """
return self.get_ids([url])[0]
def get_domain_id(self, url):
""" Simple API to get the ID of a domain """
return self.get_domain_ids([url])[0]
def _rpc(self, method, *args):
return self.urlserver.call(method, *args)
def get_ids(self, urls):
""" Returns an array of 64-bit integers for each URL """
if len(urls) == 0:
return []
if isinstance(urls[0], URL):
urls = [u.url for u in urls]
return self._rpc("get_ids", urls)
def get_domain_ids(self, urls):
""" Returns an array of 64-bit integers for each domain """
if len(urls) == 0:
return []
if isinstance(urls[0], URL):
urls = [u.url for u in urls]
return self._rpc("get_domain_ids", urls)
def get_metadata(self, urls):
""" Returns an array of complete fields for each URL """
if len(urls) == 0:
return []
if isinstance(urls[0], URL):
urls = [u.url for u in urls]
res = self._rpc("get_metadata", urls)
ret = []
for row in res:
d = {}
for i, f in enumerate(URLSERVER_FIELDNAMES["get_metadata"]):
d[f] = row[i]
ret.append(d)
return ret
def call():
client = RPCClient('127.0.0.1', 6000)
print client.call('sum', 1, 2)
class MPMShell(cmd.Cmd):
"""
RPC Shell class. See cmd module.
"""
def __init__(self):
cmd.Cmd.__init__(self)
self.prompt = "> "
self.client = None
self.remote_methods = []
self._claimer = None
self._host = None
self._port = None
self._device_info = None
args = parse_args()
if args.host is not None:
self.connect(args.host, args.port)
if args.claim:
self.claim()
elif args.hijack:
self.hijack(args.hijack)
self.update_prompt()
def _add_command(self, command, docs, requires_token=False):
"""
Add a command to the current session
"""
cmd_name = 'do_' + command
if not hasattr(self, cmd_name):
new_command = lambda args: self.rpc_template(
str(command), requires_token, args
)
new_command.__doc__ = docs
setattr(self, cmd_name, new_command)
self.remote_methods.append(command)
def rpc_template(self, command, requires_token, args=None):
"""
Template function to create new RPC shell commands
"""
if requires_token and \
(self._claimer is None or self._claimer.token is None):
print("Cannot execute `{}' -- no claim available!")
return
try:
if args or requires_token:
expanded_args = self.expand_args(args)
if requires_token:
expanded_args.insert(0, self._claimer.token)
response = self.client.call(command, *expanded_args)
else:
response = self.client.call(command)
except RPCError as ex:
print("RPC Command failed!")
print("Error: {}".format(ex))
return
except Exception as ex:
print("Unexpected exception!")
print("Error: {}".format(ex))
return
if isinstance(response, bool):
if response:
print("Command executed successfully!")
return
print("Command failed!")
return
print("==> " + str(response))
def get_names(self):
" We need this for tab completion. "
return dir(self)
###########################################################################
# Cmd module specific
###########################################################################
def run(self):
" Go, go, go! "
try:
self.cmdloop()
except KeyboardInterrupt:
self.do_disconnect(None)
exit(0)
def postcmd(self, stop, line):
"""
Is run after every command executes. Does:
- Update prompt
"""
self.update_prompt()
###########################################################################
# Internal methods
###########################################################################
def connect(self, host, port):
"""
Launch a connection.
"""
print("Attempting to connect to {host}:{port}...".format(
host=host, port=port
))
try:
self.client = RPCClient(host, port, pack_params={'use_bin_type': True})
print("Connection successful.")
except Exception as ex:
print("Connection refused")
print("Error: {}".format(ex))
return False
self._host = host
self._port = port
print("Getting methods...")
methods = self.client.call('list_methods')
for method in methods:
self._add_command(*method)
print("Added {} methods.".format(len(methods)))
print("Quering device info...")
self._device_info = self.client.call('get_device_info')
return True
def disconnect(self):
"""
Clean up after a connection was closed.
"""
self._device_info = None
if self._claimer is not None:
self._claimer.unclaim()
if self.client:
try:
self.client.close()
except RPCError as ex:
print("Error while closing the connection")
print("Error: {}".format(ex))
for method in self.remote_methods:
delattr(self, "do_" + method)
self.remote_methods = []
self.client = None
self._host = None
self._port = None
def claim(self):
" Initialize claim "
assert self.client is not None
if self._claimer is not None:
print("Claimer already active.")
return True
print("Claiming device...")
self._claimer = MPMClaimer(self._host, self._port, self.unclaim_hook)
return True
def hijack(self, token):
" Hijack running session "
assert self.client is not None
if self._claimer is not None:
print("Claimer already active. Can't hijack.")
return False
print("Hijacking device...")
self._claimer = MPMHijacker(token)
return True
def unclaim(self):
"""
unclaim
"""
if self._claimer is not None:
self._claimer.unclaim()
self._claimer = None
def unclaim_hook(self):
"""
Hook
"""
pass
def update_prompt(self):
"""
Update prompt
"""
if self._device_info is None:
self.prompt = '> '
else:
if self._claimer is None:
claim_status = ''
elif isinstance(self._claimer, MPMClaimer):
claim_status = ' [C]'
elif isinstance(self._claimer, MPMHijacker):
claim_status = ' [H]'
self.prompt = '{dev_id}{claim_status}> '.format(
dev_id=self._device_info.get(
'name', self._device_info.get('serial', '?')
),
claim_status=claim_status,
)
def expand_args(self, args):
"""
Takes a string and returns a list
"""
if self._claimer is not None and self._claimer.token is not None:
args = args.replace('$T', str(self._claimer.token))
eval_preamble = '='
args = args.strip()
if args.startswith(eval_preamble):
parsed_args = eval(args.lstrip(eval_preamble))
if not isinstance(parsed_args, list):
parsed_args = [parsed_args]
else:
parsed_args = []
for arg in args.split():
try:
parsed_args.append(int(arg, 0))
continue
except ValueError:
pass
try:
parsed_args.append(float(arg))
continue
except ValueError:
pass
parsed_args.append(arg)
return parsed_args
###########################################################################
# Predefined commands
###########################################################################
def do_connect(self, args):
"""
Connect to a remote MPM server. See connect()
"""
host, port = split_args(args, 'localhost', MPM_RPC_PORT)
port = int(port)
self.connect(host, port)
def do_claim(self, _):
"""
Spawn a claim loop
"""
self.claim()
def do_hijack(self, token):
"""
Hijack a running session
"""
self.hijack(token)
def do_unclaim(self, _):
"""
unclaim
"""
self.unclaim()
def do_disconnect(self, _):
"""
disconnect from the RPC server
"""
self.disconnect()
def do_import(self, args):
"""import a python module into the global namespace"""
globals()[args] = import_module(args)
def do_EOF(self, _):
" When catching EOF, exit the program. "
print("Exiting...")
self.disconnect()
exit(0)
class Client(BaseClient):
def __init__(self,
host="127.0.0.1",
port=9033,
model="en",
embeddings_path=None,
verbose=False,
attributes=None):
super(Client, self).__init__(model, embeddings_path)
self.host = host
self.port = port
self.rpc = RPCClient(host, port)
self.verbose = verbose
self.attributes = attributes
self.cache = LRUCache(maxsize=3000000)
def _call(self, path, *args):
return self.rpc.call(path, *args)
def most_similar(self, word, n=10):
key = (word, n, self.model, self.embeddings_path)
if key not in self.cache:
result = self._call("most_similar", word, n, self.model,
self.embeddings_path)
self.cache[key] = result
return self.cache[key]
def single(self, document, attributes=None):
attributes = attributes or tuple(self.attributes)
key = (document, attributes, self.model, self.embeddings_path)
if key not in self.cache:
sentences = self._call("single", document, self.model,
self.embeddings_path, attributes)
result = SpacyClientDocument(sentences)
self.cache[key] = result
return result
return self.cache[key]
def _bulk(self, documents, attributes):
attributes = attributes or tuple(self.attributes)
done_docs = [(num, self.cache[(document, attributes)])
for num, document in enumerate(documents)
if (document, attributes) in self.cache]
todo_docs = [(num, document) for num, document in enumerate(documents)
if (document, attributes) not in self.cache]
todo_inds = [x[0] for x in todo_docs]
todo_docs = [x[1] for x in todo_docs]
if todo_docs:
todo_docs = self._call("bulk", todo_docs, self.model,
self.embeddings_path, attributes)
todo_docs = [(x, SpacyClientDocument(y))
for x, y in zip(todo_inds, todo_docs)]
return [x[1] for x in sorted(todo_docs + done_docs)]
def bulk(self, documents, batch_size=1000, attributes=None):
attributes = attributes or tuple(self.attributes)
parsed_documents = []
if len(documents) > batch_size:
batches = int(math.ceil(len(documents) / batch_size))
print("Batching {} requests with batch_size {}".format(
batches, batch_size))
if self.verbose:
batch_iterator = tqdm.tqdm(range(batches))
else:
batch_iterator = range(batches)
for b in batch_iterator:
docs = documents[b * batch_size:(b + 1) * batch_size]
res = self._bulk(docs, attributes)
parsed_documents.extend(res)
else:
parsed_documents = self._bulk(documents, attributes)
return parsed_documents
def testfunc(self, document):
print('Entities', [(ent.text, ent.label_) for ent in document.ents])
print("JSON FORMAT ---> ",
json.dumps([(ent.text, ent.label_) for ent in document.ents]))
class URLClientRemote(object):
""" The URLClient takes URLs and sends them to the URLServer to get their IDs or fields
We should migrate to something more efficient (but more complex!) like gRPC in the future.
"""
urlserver = None
def connect(self):
self.urlserver = RPCClient(config["URLSERVER"].split(":")[0],
int(config["URLSERVER"].split(":")[1]))
def empty(self):
pass
def get_id(self, url):
""" Simple API to get the ID of an URL """
return self.get_ids([url])[0]
def get_domain_id(self, url):
""" Simple API to get the ID of a domain """
return self.get_domain_ids([url])[0]
def _rpc(self, method, *args):
return self.urlserver.call(method, *args)
def get_ids(self, urls):
""" Returns an array of 64-bit integers for each URL """
if len(urls) == 0:
return []
if isinstance(urls[0], URL):
urls = [u.url for u in urls]
return self._rpc("get_ids", urls)
def get_domain_ids(self, urls):
""" Returns an array of 64-bit integers for each domain """
if len(urls) == 0:
return []
if isinstance(urls[0], URL):
urls = [u.url for u in urls]
return self._rpc("get_domain_ids", urls)
def get_metadata(self, urls):
""" Returns an array of complete fields for each URL """
if len(urls) == 0:
return []
if isinstance(urls[0], URL):
urls = [u.url for u in urls]
res = self._rpc("get_metadata", urls)
ret = []
for row in res:
d = {}
for i, f in enumerate(URLSERVER_FIELDNAMES["get_metadata"]):
d[f] = row[i]
ret.append(d)
return ret
import numpy as np
from mprpc import RPCClient
client = RPCClient('127.0.0.1', 9033)
with client_pool.connection() as client:
vec = client.call('embedding', ['hey', 'there', 'how', 'are', 'you'])
print(np.shape(vec))
#import pyjsonrpc
from mprpc import RPCClient
'''
from paramiko import SSHClient
from scp import SCPClient
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.connect('192.168.1.16', username='******', password='******', look_for_keys=False)
with SCPClient(ssh.get_transport()) as scp:
scp.put("/home/pi/Desktop/pyUI/profiles/aaa/top/aaa.jpg", "/home/pi/Desktop/aaa.jpg")
'''
client = RPCClient('127.0.0.1', 8080)
print(client.call('add', 1, 2))
print(client.call('updateProfile', ""))
print(client.call('profilepath', '/home/pi/Desktop/pyUI/profiles', 'aaa'))
print(client.call('ResultTest', 5))
#client.call('Init')
client.call('TakePicture', 0, False)
client.call('TakePicture', 1)
client.call('TakePicture', 2)
client.call('CreateSamplePoint', 0, 150, 200)
#client.call('Uninit')
client.call('CloseServer')
'''
http_client = pyjsonrpc.HttpClient(
url = "http://localhost:8080/jsonrpc"
#username = "******",
#password = "******"
class Proxy(object):
def __init__(self, uri, show=False):
self.showerr = show
try:
self.host, self.port = uri.split(":")
except:
self.host = "0.0.0.0"
self.port = 0
self.OK = False
self.__linked = False
try:
self._link_ = RPCClient(self.host, int(self.port), timeout=10)
self.__linked = True
except:
self.__linked = False
if show:
P_Log("[FR]ERROR [FW] No Link to {}".format(uri))
def _close(self):
if self._link_.is_connected():
self._link_.close()
def connect(self):
try:
if not self.OK:
self._link_ = RPCClient(self.host, int(self.port), timeout=20)
if self._link_.is_connected():
self.OK = self.__create_hooks()
else:
self.OK = False
except:
self.OK = False
if self.showerr:
P_Log("[FR]ERROR [FW] No Connect to {}".format(uri))
finally:
return self.OK
def __call__(self):
return self.connect()
def __str__(self):
if self.OK:
return "{}:{}::Connected".format(*self._link_.getpeername())
elif self.__linked:
return "{}:{}::Linked".format(*self._link_.getpeername())
else:
return "Disconected 1.1.1.1:1"
def __create_hooks(self):
try:
_config = self._link_.call('G_E_T_Configuration')
except:
_config = {}
#print("error creating hooks {}:{}".format(self.host,self.port))
hooks = []
for defs, params in _config.items():
params_def = params[0]
params_call = ""
for x in params[1]:
params_call = params_call + "," + str(x)
d = def_skel.format(defs, params_def, params_call,
self.host + ":" + str(self.port))
#print(d)
hooks.append((defs, d))
for defs, fun in hooks:
exec(fun)
self.__dict__[defs] = types.MethodType(eval(defs), self)
self.functions = hooks
return len(self.functions) > 0
def get_uri(self):
return "{}:{}".format(self.host, self.port)
def get_functions(self):
return [a for a, b in self.functions]
def call(self, fn, *args):
try:
return self._link_.call(fn, *args)
except Exception as ex:
#raise
P_Log("[[FR]ERROR[FW]] Running call on Proxy {}".format(
self.get_uri()))
P_Log(str(ex))
return None
def main2():
client = RPCClient("localhost", 3000)
ans = client.call("sum", 1, 2)
print(ans)
