def execnet_gateways(self):
# TODO: Eventlet hangs on Mac OS X with popen.
# execnet.set_execmodel("eventlet", "thread")
gw = self.gateway
if not gw:
for _ in range(self.job_num):
yield execnet.makegateway(
"" if "FOWLER_PYTHON" not in os.environ else "popen//python={}".format(os.environ["FOWLER_PYTHON"])
)
for gateway_spec in gw:
if "*" in gateway_spec:
num, spec = gateway_spec.split("*")
num = int(num)
group = execnet.Group()
group.defaultspec = spec
xspec = execnet.XSpec(spec)
master_spec = "ssh={xspec.via}//" "id={xspec.via}//" "python={xspec.python}" "".format(xspec=xspec)
logger.debug("Connecting to master %s to create %s gateways.", master_spec, num)
group.makegateway(master_spec)
for _ in range(num):
yield group.makegateway()
else:
yield execnet.makegateway()
def turn(responses):
if turn_way == "left":
z = 1
elif turn_way == "right":
z = 1
else:
responses.put("failure")
return
action_code = """
import rospy, time
from geometry_msgs.msg import Twist
rospy.init_node('move')
p = rospy.Publisher('/base_controller/command', Twist)
turn = Twist()
turn.linear.x = 0;
turn.linear.y = 0; turn.linear.z = 0;
turn.angular.x = 0; turn.angular.y = 0;
turn.angular.z = %i;
times = 0
while 1:
times += 1
p.publish(turn)
rospy.sleep(%s)
if times == %i:
break
twist = Twist()
p.publish(twist)
channel.send("success")
""" % (z, "0.1", 40)
gw = execnet.makegateway("popen//python=python2.7")
channel = gw.remote_exec(action_code)
if channel.receive() == "success":
responses.put("success")
def create_connection(self, hostname, connection_id, niceness): """Create a new connection to a prepared host.""" gateway_cmd = "ssh="+hostname+"//id="+connection_id+"//nice="+ niceness gw = execnet.makegateway(gateway_cmd) # keep track of how we created this gw so we can re-create it if necessary. self.worker_attrs_[gw] = (hostname, connection_id, niceness) return gw
def _start_cluster_simulation(self, subdomains, cluster=None):
"""Starts a simulation on a cluster of nodes."""
if cluster is None:
try:
cluster = imp.load_source('cluster', self.config.cluster_spec)
except IOError:
cluster = imp.load_source('cluster',
os.path.expanduser('~/.sailfish/{0}'.format(self.config.cluster_spec)))
self._cluster_gateways = []
self._node_subdomains = split_subdomains_between_nodes(cluster.nodes, subdomains)
import execnet
for _, node in zip(self._node_subdomains, cluster.nodes):
self._cluster_gateways.append(execnet.makegateway(node.host))
# Copy files to remote nodes if necessary.
if self.config.cluster_sync:
local, dest = self.config.cluster_sync.split(':')
assert dest[0] != '/', 'Only relative paths are supported on remote nodes.'
rsync = execnet.RSync(local)
for gw in self._cluster_gateways:
rsync.add_target(gw, dest)
rsync.send()
subdomain_id_to_addr = {}
for node_id, subdomains in enumerate(self._node_subdomains):
for subdomain in subdomains:
subdomain_id_to_addr[subdomain.id] = cluster.nodes[node_id].addr
self._cluster_channels = []
import sys
for i, (node, gw) in enumerate(zip(cluster.nodes, self._cluster_gateways)):
# Assign specific GPUs from this node, as defined by the cluster
# config file.
node_config = copy.copy(self.config)
node_config.gpus = cluster.nodes[i].gpus
for k, v in node.settings.items():
setattr(node_config, k, v)
self._cluster_channels.append(
gw.remote_exec(_start_cluster_machine_master,
args=pickle.dumps((node_config, self._node_subdomains[i])),
main_script=sys.argv[0],
lb_class_name=self._lb_class.__name__,
subdomain_addr_map=subdomain_id_to_addr,
iface=node.iface))
ports = {}
for channel in self._cluster_channels:
data = channel.receive()
# If a string is received, print it to help with debugging.
if type(data) is str:
print(data)
else:
ports.update(data)
for channel in self._cluster_channels:
channel.send(ports)
def connect(self):
if self._cache_key not in RPC_CACHE:
ctrl = self.runner._ploy_ctrl
instance = ctrl.instances[self.host]
if hasattr(instance, '_status'):
if instance._status() != 'running':
raise errors.AnsibleError("Instance '%s' unavailable." % instance.config_id)
try:
ssh_info = instance.init_ssh_key(user=self.user)
except instance.paramiko.SSHException as e:
raise errors.AnsibleError("Couldn't validate fingerprint for '%s': %s" % (instance.config_id, e))
spec = execnet.XSpec('ssh')
ssh_args = instance.ssh_args_from_info(ssh_info)
if utils.VERBOSITY > 3:
ssh_args += ["-vvv"]
spec.ssh = SSHArgs(ssh_args)
vars = self.runner.inventory.get_variables(self.host)
spec.python = vars.get('ansible_python_interpreter', 'python')
gw = execnet.makegateway(spec)
try:
channel = gw.remote_exec(remote)
except IOError as e:
raise errors.AnsibleError("Couldn't open execnet channel for '%s': %s" % (instance.config_id, e))
RPC_CACHE[self._cache_key] = RPCWrapper(channel)
self.rpc = RPC_CACHE[self._cache_key]
return self
def map(mod, args, specs=[('popen', 2)]):
gateways = []
channels = []
for spec, count in specs:
for i in range(count):
gw = execnet.makegateway(spec)
gateways.append(gw)
channels.append(gw.remote_exec(mod))
cyc = itertools.cycle(channels)
for i, arg in enumerate(args):
channel = next(cyc)
channel.send((i, arg))
mch = execnet.MultiChannel(channels)
queue = mch.make_receive_queue()
l = len(args)
results = [None] * l
for j in range(l):
channel, (i, result) = queue.get()
results[i] = result
for gw in gateways:
gw.exit()
return results
def _start_cluster_simulation(self, subdomains, cluster=None):
"""Starts a simulation on a cluster of nodes."""
if cluster is None:
try:
cluster = imp.load_source('cluster', self.config.cluster_spec)
except IOError:
cluster = imp.load_source('cluster',
os.path.expanduser('~/.sailfish/{0}'.format(self.config.cluster_spec)))
self._cluster_gateways = []
self._node_subdomains = split_subdomains_between_nodes(cluster.nodes, subdomains)
import execnet
for _, node in zip(self._node_subdomains, cluster.nodes):
self._cluster_gateways.append(execnet.makegateway(node.host))
# Copy files to remote nodes if necessary.
if self.config.cluster_sync:
local, dest = self.config.cluster_sync.split(':')
assert dest[0] != '/', 'Only relative paths are supported on remote nodes.'
rsync = execnet.RSync(local)
for gw in self._cluster_gateways:
rsync.add_target(gw, dest)
rsync.send()
subdomain_id_to_addr = {}
for node_id, subdomains in enumerate(self._node_subdomains):
for subdomain in subdomains:
subdomain_id_to_addr[subdomain.id] = cluster.nodes[node_id].addr
self._cluster_channels = []
import sys
for i, (node, gw) in enumerate(zip(cluster.nodes, self._cluster_gateways)):
# Assign specific GPUs from this node, as defined by the cluster
# config file.
node_config = copy.copy(self.config)
node_config.gpus = cluster.nodes[i].gpus
for k, v in node.settings.items():
setattr(node_config, k, v)
self._cluster_channels.append(
gw.remote_exec(_start_cluster_machine_master,
args=pickle.dumps((node_config, self._node_subdomains[i])),
main_script=sys.argv[0],
lb_class_name=self._lb_class.__name__,
subdomain_addr_map=subdomain_id_to_addr,
iface=node.iface))
ports = {}
for channel in self._cluster_channels:
data = channel.receive()
# If a string is received, print it to help with debugging.
if type(data) is str:
print(data)
else:
ports.update(data)
for channel in self._cluster_channels:
channel.send(ports)
def map(mod, args, options=[('popen', 2)]): #2 = no of processes to spawn
gateways = []
channels = []
for option, count in options:
for i in range(count):
gw = execnet.makegateway(option)
gateways.append(gw)
channels.append(gw.remote_exec(mod))
cyc = itertools.cycle(channels)
for i, arg in enumerate(args):
channel = cyc.next()
channel.send((i, arg))
mch = execnet.MultiChannel(channels)
queue = mch.make_receive_queue()
l = len(args)
results = [None] * l
for j in range(l):
if j + 1 <= sys.maxint:
print "\nProcessing document no. %d..." % (j + 1)
channel, (i, result) = queue.get()
results[i] = result
for gw in gateways:
gw.exit()
return results
def exec_method_in_sub_process():
gw = execnet.makegateway()
channel = gw.remote_exec(multiplier, factor = 10)
channel.send(2)
out = channel.receive()
gw.exit()
print("\nOut:{0}\n".format(out))
def execnet_master_main(install_hook=True, callback=None, gw=None, slave_main=execnet_slave_main):
callback = callback or handle_ch
exit_event.clear() # to allow multiple test cases to run despite our gross global state
# This directory contains the 'baz' module. Adding it to sys.path means that 'baz' will be importable on the
# master, but not (normally) on any of the slaves.
baz_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), '../test/not-in-default-path')
if baz_path not in sys.path:
sys.path.append(baz_path)
# Create another Python interpreter to be our slave. (This could just as well be any other remote interpreter that
# execnet supports.)
gw = gw or execnet.makegateway()
# This is all of the setup that we need to do! This creates a channel to the gateway (and inserts itself into the
# slave's import machinery) as well as a callback to service requests on this side.
if install_hook:
execnet_importhook.install_import_hook(gw)
# Now we can go about our business.
ch = gw.remote_exec(slave_main)
ch.setcallback(partial(callback, ch), ENDMARKER)
# When execnet_slave_main() finishes, our callback (handle_ch) will get ENDMARKER and set() the event, so that we
# know to let the master end too.
exit_event.wait()
def node_recon(nodelist, interactive=True):
"""grab system information from a list of hosts and create or update
nodes' db entries.
"""
import execnet
from dirt.tasks import system_info
from dirt.core.db import db
nodes = db.get_nodes()
for node in nodelist:
log.write("Connecting to host %s" % node)
try:
gw = execnet.makegateway("ssh=%s" % node)
except execnet.HostNotFound:
log.write("Host not found: %s" % node)
continue
log.write("Connected to host %s" % node)
ch = gw.remote_exec(system_info)
sys_info = ch.receive()
# update the db
if sys_info["fqdn"] in nodes:
d = nodes[sys_info["fqdn"]]
d["sys_info"] = sys_info
d["enabled"] = True
else:
d = {"type": "node", "fqdn": sys_info["fqdn"], "sys_info": sys_info}
log.write("Adding new node %(fqdn)s to database" % d)
d["enabled"] = settings.node_enable_default
db.save(d)
def map(mod, args, options=[('popen', 2)]): #2 = no of processes to spawn
gateways = []
channels = []
for option, count in options:
for i in range(count):
gw = execnet.makegateway(option)
gateways.append(gw)
channels.append(gw.remote_exec(mod))
cyc = itertools.cycle(channels)
for i, arg in enumerate(args):
channel = cyc.next()
channel.send((i, arg))
mch = execnet.MultiChannel(channels)
queue = mch.make_receive_queue()
l = len(args)
results = [None] * l
for j in range(l):
if j+1 <= sys.maxint:
print "\nProcessing document no. %d..." %(j+1)
channel, (i, result) = queue.get()
results[i] = result
for gw in gateways:
gw.exit()
return results
def load_workers(self, workers):
workers_dict = helpers.x_to_dict[type(workers)](workers)
for worker in workers_dict["workers"]:
worker.setdefault("exec_id",0)
## if not configured to be in a pool, have a pool of your own.
pool_id = worker.setdefault("pool", [worker["id"]])
ctrl_sock = self.ctx().socket(zmq.REQ)
ctrl_lock = threading.Lock()
ctrl_lock.acquire()
self.workers[worker["id"]] = (ctrl_sock, ctrl_lock, worker)
if "execnet_address" in worker:
channel = execnet.makegateway(worker["execnet_address"]).remote_exec(remote_agent)
channel.send(worker["control_address"])
#print "{id:s}: connecting to {control_address:s}...".format(**worker)
ctrl_sock.connect(worker["control_address"])
#print worker["id"], "connected"
self._send_ctrl(worker, {"instruction":"set_attr","attr":"name", "value": worker["id"]})
ctrl_lock.release()
pool = self.add_pool(pool_id)
pool[2].add(worker["id"])
ctrl_lock.acquire()
self._send_ctrl(worker, {"instruction": "task_attach", "remote": pool[1]})
self._send_ctrl(worker, {"instruction": "results_attach", "remote": self.config["result_bind"]})
ctrl_lock.release()
ctrl_lock.acquire()
ctrl_lock.release()
def map(mod, args, specs=[('popen', 2)]):
gateways = []
channels = []
for spec, count in specs:
for i in range(count):
gw = execnet.makegateway(spec)
gateways.append(gw)
channels.append(gw.remote_exec(mod))
cyc = itertools.cycle(channels)
for i, arg in enumerate(args):
channel = next(cyc)
channel.send((i, arg))
mch = execnet.MultiChannel(channels)
queue = mch.make_receive_queue()
l = len(args)
results = [None] * l
for j in range(l):
channel, (i, result) = queue.get()
results[i] = result
for gw in gateways:
gw.exit()
return results
def add_prole(ident):
''' for a given host, try to spawn the crack module on that host
if it succeeds, add the channel to chans, the gateway to gateways
and put this host in the task_tracker'''
host = ident.split("_")[0]
try:
# establish connection to 'host'
gw = execnet.makegateway("ssh="+host+"//id="+ident+"//python=python3//chdir=./execnet/crackr/")
# run crack module on 'host'
ch = gw.remote_exec(crack)
ch.send((passwd, max_task_size))
# put this chan, gw into a list of them
chans[ident] = ch
gateways[ident] = gw
# task_tracker keeps track of what set of strings this machine/core
# is working on so that if it fails these strings won't be skipped
task_tracker[ident] = 0
if VERBOSE: print("added " + ident)
return 0
except:
if VERBOSE: print("fail to add " + ident)
return -1
def getinfo(sshname, ssh_config=None, loginfo=sys.stdout):
if ssh_config:
spec = "ssh=-F %s %s" % (ssh_config, sshname)
else:
spec += "ssh=%s" % sshname
debug("connecting to", repr(spec))
try:
gw = execnet.makegateway(spec)
except IOError:
error("could not get sshgatway", sshname)
else:
ri = RemoteInfo(gw)
#print "%s info:" % sshname
prefix = sshname.upper() + " "
print >>loginfo, prefix, "fqdn:", ri.getfqdn()
for attr in (
"sys.platform",
"sys.version_info",
):
loginfo.write("%s %s: " %(prefix, attr,))
loginfo.flush()
value = ri.getmodattr(attr)
loginfo.write(str(value))
loginfo.write("\n")
loginfo.flush()
memswap = ri.getmemswap()
if memswap:
mem,swap = memswap
print >>loginfo, prefix, "Memory:", mem, "Swap:", swap
cpuinfo = ri.getcpuinfo()
if cpuinfo:
numcpu, model = cpuinfo
print >>loginfo, prefix, "number of cpus:", numcpu
print >>loginfo, prefix, "cpu model", model
return ri
def distribute_with_execnet(config, workloads):
import src.jobs.execnet_recieve as run_jobs_remote
# 3. Run jobs on remote servers:
comms = []
for i, server in enumerate(config.servers):
gw = execnet.makegateway(
f"ssh={server.address}//python={server.python}//chdir={server.cwd}"
)
ch = gw.remote_exec(run_jobs_remote)
ch.send(pickle.dumps((workloads[i], i, config)))
comms += [(gw, ch)]
# 4. Gather direct results
new_population = []
for gw, ch in comms:
received = ch.receive()
result = pickle.loads(received)
if isinstance(result, Exception):
raise result
new_population += result
# 5. Waiting for results and shutting down:
mch = execnet.MultiChannel([ch for _, ch in comms])
mch.waitclose()
for gw, _ in comms:
gw.exit()
return new_population
def scan(locations, deserialize=False, scanengine_root_dir=None):
"""
Scan the list of paths at `location` and return the results as an iterable
of JSON strings. If `deserialize` is True the iterable contains a python data
instead.
Each location is scanned independently.
"""
if not scanengine_root_dir:
scanengine_root_dir = abspath(normpath(__file__))
scanengine_root_dir = dirname(dirname(dirname(scanengine_root_dir)))
python2 = join(scanengine_root_dir, 'bin', 'python')
spec = 'popen//python={python2}'.format(**locals())
gateway = execnet.makegateway(spec) # NOQA
channel = gateway.remote_exec(scanserv)
for location in locations:
# build a mapping of options to use for this scan
scan_kwargs = dict(
location=location,
license=True,
license_text=True,
copyright=True,
info=True,
processes=0,
)
channel.send(scan_kwargs) # execute func-call remotely
results = channel.receive()
if deserialize:
results = json.loads(results, object_pairs_hook=OrderedDict)
yield results
def load_workers(self, workers):
workers_dict = helpers.x_to_dict[type(workers)](workers)
for worker in workers_dict["workers"]:
## if not configured to be in a pool, have a pool of your own.
pool_ids = worker.setdefault("pools", [worker["id"]])
ctrl_sock = self.ctx.socket(zmq.REQ)
ctrl_lock = multiprocessing.Lock()
ctrl_lock.acquire()
self.workers[worker["id"]] = (ctrl_sock, ctrl_lock, worker)
if "execnet_address" in worker:
channel = execnet.makegateway(worker["execnet_address"]).remote_exec(remote_agent)
channel.send(worker["control_address"])
ctrl_sock.connect(worker["control_address"])
ctrl_sock.send_json({"cmd":[{"instruction":"set_attr","attr":"name", "value": worker["id"]}]})
print ctrl_sock.recv_json()
ctrl_lock.release()
for pool_id in pool_ids:
pool = self.add_pool(pool_id)
pool[2].add(worker["id"])
ctrl_lock.acquire()
ctrl_sock.send_json({"cmd":[{"instruction": "task_attach", "remote": pool[1]} ] } )
print "task_attach:", ctrl_sock.recv_json()
ctrl_sock.send_json({"cmd":[{"instruction": "results_attach", "remote": self.config["result_bind"]} ] } )
print "results_attach:", ctrl_sock.recv_json()
ctrl_lock.release()
ctrl_lock.acquire()
ctrl_lock.release()
def _connect(self):
if self._cache_key not in ploy_ansible.RPC_CACHE:
ctrl = self._play_context._ploy_ctrl
instance = ctrl.instances[self.host]
if hasattr(instance, '_status'):
if instance._status() != 'running':
raise errors.AnsibleError("Instance '%s' unavailable." %
instance.config_id)
try:
ssh_info = instance.init_ssh_key(user=self.user)
except paramiko.SSHException as e:
raise errors.AnsibleError(
"Couldn't validate fingerprint for '%s': %s" %
(instance.config_id, e))
spec = execnet.XSpec('ssh')
ssh_args = instance.ssh_args_from_info(ssh_info)
if display.verbosity > 3:
ssh_args += ["-vvv"]
spec.ssh = SSHArgs(ssh_args)
spec.python = instance.config.get('ansible_python_interpreter',
'python')
gw = execnet.makegateway(spec)
try:
channel = gw.remote_exec(ploy_ansible.remote)
except IOError as e:
raise errors.AnsibleError(
"Couldn't open execnet channel for '%s': %s" %
(instance.config_id, e))
ploy_ansible.RPC_CACHE[self._cache_key] = RPCWrapper(channel)
self.rpc = ploy_ansible.RPC_CACHE[self._cache_key]
return self
def createChannels(self):
'''Create a list of channels used for communication between coordinator and worker'''
clients = self.config['default']['clients']
assert len(clients) > 0
LOG.info("Invoking benchmark framework on %d clients" % len(clients))
import benchmark
remoteCall = benchmark
channels = []
# Create fake channel that invokes the worker directly in
# the same process
if self.config['default']['direct']:
self.config['default']['clientprocs'] = 1
ch = DirectChannel()
channels.append(ch)
# Otherwise create SSH channels to client nodes
else:
# Print a header message in the logfile to indicate that we're starting
# a new benchmark run
LOG.info("Executor Log File: %s" %
self.config['default']['logfile'])
with open(self.config['default']['logfile'], "a") as fd:
header = "%s BENCHMARK - %s\n\n" % (self.benchmark.upper(),
datetime.now())
header += "%s" % (pformat(self.config))
fd.write('*' * 100 + '\n')
for line in header.split('\n'):
fd.write('* ' + line + '\n')
fd.write('*' * 100 + '\n')
## WITH
totalClients = len(clients) * self.config['default']['clientprocs']
start = time.time()
for node in clients:
cmd = 'ssh=' + node
cmd += r"//chdir=" + self.config['default']['path']
LOG.debug(cmd)
LOG.debug("# of Client Processes: %s" %
self.config['default']['clientprocs'])
for i in range(int(self.config['default']['clientprocs'])):
LOG.debug("Invoking %s on %s" % (remoteCall, node))
gw = execnet.makegateway(cmd)
ch = gw.remote_exec(remoteCall)
channels.append(ch)
now = time.time()
if (now - start) > 10 and int(
(len(channels) / float(totalClients)) * 100) % 25 == 0:
LOG.debug("Started Client Threads %d / %d" %
(len(channels), totalClients))
## FOR (processes)
## FOR (node)
# IF
LOG.info("Created %d client processes on %d nodes" %
(len(channels), len(clients)))
LOG.debug(channels)
return channels
def call_python_version(Version, Module, Function, ArgumentList):
gw = execnet.makegateway("popen//python=python%s" % Version)
channel = gw.remote_exec("""
from %s import %s as the_function
channel.send(the_function(*channel.receive()))
""" % (Module, Function))
channel.send(ArgumentList)
return channel.receive()
def get_load(hostname):
try:
load_module = __import__('dirt.tasks.load', fromlist=['dirt.tasks'])
gw = execnet.makegateway('ssh=%s' % hostname)
ch = gw.remote_exec(load_module)
return ch.receive()['load']
except Exception:
return None
def gateway(request):
try:
gw = execnet.makegateway('popen//python=%s' % request.param)
except FileNotFoundError:
pytest.skip('%s is not installed')
gw.reconfigure(py2str_as_py3str=False, py3str_as_py2str=False)
return gw
def printver(python_executable):
gw = execnet.makegateway(
'popen//python={}'.format(python_executable))
channel = gw.remote_exec("""
import platform
channel.send(platform.python_version())
""")
return channel.receive()
def main():
gw = execnet.makegateway("popen//python=/usr/local/bin/jython")
channel = gw.remote_exec("""
import try_write_xml as twx
twx.write()
""")
def call_python3(module, func, arg_list):
# Adapted from https://stackoverflow.com/a/44965570
gw = execnet.makegateway("popen//python=python3")
channel = gw.remote_exec("""
from %s import %s as the_function
channel.send(the_function(*channel.receive()))
""" % (module, func))
channel.send(arg_list)
return channel.receive()
def setup(self, ):
self.testdir.chdir()
#import os ; os.environ['EXECNET_DEBUG'] = "2"
self.gateway = execnet.makegateway()
self.config = config = self.testdir.parseconfigure()
putevent = self.use_callback and self.events.put or None
self.slp = SlaveController(None, self.gateway, config, putevent)
self.request.addfinalizer(self.slp.ensure_teardown)
self.slp.setup()
def setup(self, ):
self.testdir.chdir()
# import os ; os.environ['EXECNET_DEBUG'] = "2"
self.gateway = execnet.makegateway()
self.config = config = self.testdir.parseconfigure()
putevent = self.use_callback and self.events.put or None
self.slp = SlaveController(None, self.gateway, config, putevent)
self.request.addfinalizer(self.slp.ensure_teardown)
self.slp.setup()
def on_ssh_select(self, host_string):
import execnet
venv_paths = sublime.load_settings(SETTINGS_FILE).get("python_virtualenv_paths", [])
try:
gw = execnet.makegateway("ssh=" + host_string)
ch = gw.remote_exec(VENV_SCAN_CODE)
except Exception, e:
sublime.error_message(repr(e))
return
def create_group(hostfile):
hosts = ["ssh=" + h for h in readlines(hostfile)]
gateways = []
for h in hosts:
try:
gateways.append(execnet.makegateway(h))
print("{0} connected".format(h))
except:
print("Could not open gateway:", sys.exc_info()[0])
return gateways
def exec_code_string_in_sub_proc():
"""source is a string: execute source string remotely with a channel put into the global namespace.
"""
gw = execnet.makegateway()
channel = gw.remote_exec("channel.send(channel.receive() + 1)")
channel.send(1)
out = channel.receive()
gw.exit()
print("\nout: {0}\n".format(out))
def test_hrsync_one_host(self, source: Path, dest: Path) -> None:
gw = execnet.makegateway("popen//chdir=%s" % dest)
finished = []
rsync = HostRSync(source)
rsync.add_target_host(gw, finished=lambda: finished.append(1))
source.joinpath("hello.py").write_text("world")
rsync.send()
gw.exit()
assert dest.joinpath(source.name, "hello.py").exists()
assert len(finished) == 1
def test_default_group(self):
oldlist = list(execnet.default_group)
gw = execnet.makegateway("popen")
try:
newlist = list(execnet.default_group)
assert len(newlist) == len(oldlist) + 1
assert gw in newlist
assert gw not in oldlist
finally:
gw.exit()
def __init__(self):
bpy.types.Operator.__init__(self)
self.spec = "popen//cmodel=eventle//python=python"
self.gateway = execnet.makegateway(self.spec)
self.channels = None
self.queue = None
self.timer = None
self.tasks = Queue()
self.terminated = 0
self.individus = []
def main(argv=None):
if not argv:
argv = sys.argv
options, remotesrc, localdest = RRSync.parse_options(argv)
roptions = options.delete and dict(delete=True) or {}
gw = execnet.makegateway(options.xspec)
channel = gw.remote_exec(rrsync_remote)
channel.send((str(remotesrc), str(localdest), roptions))
execnet.rsync_remote.serve_rsync(channel)
print "<< Done >>"
def execute_steps(self, loop, steps):
if self._gw is None:
self._gw = execnet.makegateway('ssh=%s' % self.host_string)
channel = self._gw.remote_exec(reverse_some_text)
channel.send('sent from the master')
result = yield from loop.run_in_executor(None, channel.receive)
logger.info('Recieved the result: %s' % result)
def execute(self, uriAndArg, fn, **kwargs): channels = [] for (uri, args) in uriAndArg: print "Creating gateway to " + uri gw = execnet.makegateway(uri) channels.append(gw.remote_exec(fn, args=args, **kwargs)) multi = execnet.multi.MultiChannel(channels) multi.waitclose() execnet.default_group.terminate()
def test_default_group(self):
oldlist = list(execnet.default_group)
gw = execnet.makegateway("popen")
try:
newlist = list(execnet.default_group)
assert len(newlist) == len(oldlist) + 1
assert gw in newlist
assert gw not in oldlist
finally:
gw.exit()
def on_ssh_select(self, host_string):
import execnet
venv_paths = sublime.load_settings(SETTINGS_FILE).get(
"python_virtualenv_paths", [])
try:
gw = execnet.makegateway("ssh=" + host_string)
ch = gw.remote_exec(VENV_SCAN_CODE)
except Exception, e:
sublime.error_message(repr(e))
return
def connectGateway(username, ipaddress):
global gw, channel
print('runrules: connectGateway: opening gateway to ' + ipaddress)
try:
# ASG
# Establish the ssh connection with the target VM along with sudo privileges for non-root user,
# by doing this we dont need to make the changes for those commands who needed a sudo privileges to run on target VM.
sshport = getsshport(username, ipaddress)
username, ip = ipaddress.split('@')
if "root" in username:
gw = execnet.makegateway(
"ssh=" + ipaddress + " -p " + str(sshport) +
" -o ServerAliveInterval=30 -o StrictHostKeyChecking=no -o PasswordAuthentication=no -A"
)
else:
gw = execnet.makegateway(
"ssh=" + ipaddress + " -p " + str(sshport) +
" -o ServerAliveInterval=30 -o StrictHostKeyChecking=no -o PasswordAuthentication=no -A sudo"
)
print("runrules: gw = " + str(gw))
except Exception as e:
print(
"runrules: connectGateway: Unable to open ssh gateway to target: "
+ ipaddress)
print("runrules: connectGateway: Exception => " + str(e.args))
return None
# Now gw is a valid gateway
print('runrules: connectGateway: sending channel code')
try:
channel = gw.remote_exec("""
# This is the stub program sent to the target machine
username = '******' """ + inspect.getsource(commonstub) +
inspect.getsource(initgatewaycode))
except Exception as e:
print("Error creating gateway" + str(e))
channel = None
print("runrules: Channel = ", str(channel))
return channel
def _run(self, slave_main, install_hook=True, gw_spec=None):
msgs = []
def handler(ch, item):
if item is ENDMARKER:
return exit_event.set()
msgs.append(item)
gw = execnet.makegateway(gw_spec)
execnet_master_main(install_hook=install_hook, callback=handler, slave_main=slave_main, gw=gw)
return msgs
def test_hrsync_one_host(self, mysetup):
source, dest = mysetup.source, mysetup.dest
gw = execnet.makegateway("popen//chdir=%s" % dest)
finished = []
rsync = HostRSync(source)
rsync.add_target_host(gw, finished=lambda: finished.append(1))
source.join("hello.py").write("world")
rsync.send()
gw.exit()
assert dest.join(source.basename, "hello.py").check()
assert len(finished) == 1
def handle(self, *args, **options):
# options
year=options['year']
week=options['week']
threshold=options['threshold']
# Create a new interpreter
gw=execnet.makegateway("popen//python=python2.7")
# Multiline varaible string which is executed as a python2.7 script in a separate shell
cmd = '''
import os
my_file = os.path.join(os.path.expanduser("~"), "venv/nflgame/nflgame/bin/activate_this.py")
execfile(my_file,dict(__file__=my_file))
import nflgame
def score_player(player):
scoring = {
'passing_yds' : lambda x : x*.04,
'passing_tds' : lambda x : x*4,
'passing_twoptm' : lambda x : x*2,
'rushing_yds' : lambda x : x*.1,
'rushing_tds' : lambda x : x*6,
'kickret_tds' : lambda x : x*6,
'rushing_twoptm' : lambda x : x*2,
'receiving_tds' : lambda x : x*6,
'receiving_yds' : lambda x : x*.1,
'receiving_rec' : lambda x : x*.5,
'receiving_twoptm' : lambda x : x*2,
'fumbles_lost' : lambda x : x*-2,
'passing_ints' : lambda x : x*-2,
}
score = 0
for stat in player.stats:
if stat in scoring:
score += scoring[stat](getattr(player,stat))
return score
players=nflgame.combine_game_stats(nflgame.games(%(year)s,%(week)s))
response = {}
for p in players:
for stat in p.stats:
resposnse[p.guess_position[p.name]] = stat
channel.send(repr(response))
''' % {'year':year[0],'week':week[0],'threshold':threshold[0]}
# Execture the string
channel=gw.remote_exec(cmd)
# Get the response
players = channel.receive()
# For now, just print
# TODO - After stats.models are available, store the DB with results
self.stdout.write(players)
def __init__(self, encoding, connection_string=None, activate_file="", ps1=">>> "):
super(ExecnetRepl, self).__init__(encoding, u"python", "\n", False)
self._connections_string = connection_string
self._ps1 = ps1
self._gw = execnet.makegateway(connection_string)
remote_code = REMOTE_CODE.format(ps1=ps1, activate_file=activate_file)
self._channel = self._gw.remote_exec(remote_code)
self.output = Queue()
self._channel.setcallback(self.output.put, endmarker=None)
self._alive = True
self._killed = False
def test_jedi_sub(interpreter):
gw = execnet.makegateway('popen//python={}'.format(interpreter))
rcmd = tmpl.substitute(
src='"""{}\n"""'.format(JEDI_SOURCES[0])
)
print(rcmd)
channel = gw.remote_exec(rcmd)
completions = channel.receive()
assert isinstance(completions, list)
assert completions[0]['name'] == 'date'
assert completions[0]['completion'] == 'te'
def sync(self, src, dst, uris): rsync = execnet.RSync(src) # populate the rsync for uri in uris: print "syncing to " + uri gw = execnet.makegateway(uri) rsync.add_target(gw, dst) # execute it rsync.send() # done, so close all execnet.default_group.terminate()
def test_np(self):
gw = execnet.makegateway("popen//python=python")
channel = gw.remote_exec("""
from numpy import *
a = array([2,3,4])
channel.send(a.size)
""")
for item in channel:
print item
def call_python_version(Version, Module, Function, ArgumentList):
gw = execnet.makegateway("popen//python=python%s" % Version)
channel = gw.remote_exec("""
import numpy as np
from %s import %s as the_function
dict=*channel.receive()
image=np.asarray(dict['image']).reshape(dict['shape'])
channel.send(the_function(image))
""" % (Module, Function))
channel.send(ArgumentList)
return channel.receive()
def test_hrsync_one_host(self, mysetup):
source, dest = mysetup.source, mysetup.dest
gw = execnet.makegateway("popen//chdir=%s" % dest)
finished = []
rsync = HostRSync(source)
rsync.add_target_host(gw, finished=lambda: finished.append(1))
source.join("hello.py").write("world")
rsync.send()
gw.exit()
assert dest.join(source.basename, "hello.py").check()
assert len(finished) == 1
def __init__(self, encoding, connection_string=None, activate_file="", ps1=">>> "):
super(ExecnetRepl, self).__init__(encoding, u"python", "\n", False)
self._connections_string = connection_string
self._ps1 = ps1
self._gw = execnet.makegateway(connection_string)
remote_code = REMOTE_CODE.format(ps1=ps1, activate_file=activate_file)
self._channel = self._gw.remote_exec(remote_code)
self.output = Queue()
self._channel.setcallback(self.output.put, endmarker=None)
self._alive = True
self._killed = False
def callPy(Version, Module, Function, ArgumentList):
try:
gw = execnet.makegateway("popen//python=python%s" % Version)
channel = gw.remote_exec("""
from %s import %s as the_function
channel.send(the_function(*channel.receive()))
""" % (Module, Function))
channel.send(ArgumentList)
return channel.receive()
except:
print("Unable to execute the Python script.")
return False
def get_rssi(address):
version = '2.7'
module_ = 'bluetooth_proximity.bt_rssi'
function_ = 'get_rssi'
gateway = execnet.makegateway('popen//python=python{}'.format(version))
channel = gateway.remote_exec('''
from {} import {} as the_function
channel.send(the_function(*channel.receive()))
'''.format(module_, function_))
channel.send([address])
return channel.receive()
def createChannels(self):
'''Create a list of channels used for communication between coordinator and worker'''
clients = self.config['default']['clients']
assert len(clients) > 0
LOG.info("Invoking benchmark framework on %d clients" % len(clients))
import benchmark
remoteCall = benchmark
channels=[]
# Create fake channel that invokes the worker directly in
# the same process
if self.config['default']['direct']:
self.config['default']['clientprocs'] = 1
ch = DirectChannel()
channels.append(ch)
# Otherwise create SSH channels to client nodes
else:
# Print a header message in the logfile to indicate that we're starting
# a new benchmark run
LOG.info("Executor Log File: %s" % self.config['default']['logfile'])
with open(self.config['default']['logfile'], "a") as fd:
header = "%s BENCHMARK - %s\n\n" % (self.benchmark.upper(), datetime.now())
header += "%s" % (pformat(self.config))
fd.write('*'*100 + '\n')
for line in header.split('\n'):
fd.write('* ' + line + '\n')
fd.write('*'*100 + '\n')
## WITH
totalClients = len(clients) * self.config['default']['clientprocs']
start = time.time()
for node in clients:
cmd = 'ssh='+ node
cmd += r"//chdir=" + self.config['default']['path']
LOG.debug(cmd)
LOG.debug("# of Client Processes: %s" % self.config['default']['clientprocs'])
for i in range(int(self.config['default']['clientprocs'])):
LOG.debug("Invoking %s on %s" % (remoteCall, node))
gw = execnet.makegateway(cmd)
ch = gw.remote_exec(remoteCall)
channels.append(ch)
now = time.time()
if (now - start) > 10 and int((len(channels) / float(totalClients))*100) % 25 == 0:
LOG.debug("Started Client Threads %d / %d" % (len(channels), totalClients))
## FOR (processes)
## FOR (node)
# IF
LOG.info("Created %d client processes on %d nodes" % (len(channels), len(clients)))
LOG.debug(channels)
return channels
def __call__(self, *args):
gw = execnet.makegateway("popen//python=python%s" % self.version)
script = """
import sys
outchan = channel.gateway.newchannel()
sys.stdout = outchan.makefile("w")
channel.send(outchan)
errchan = channel.gateway.newchannel()
sys.stderr = errchan.makefile("w")
channel.send(errchan)
"""
if self.path is not None:
script += """
sys.path.append('%s')
""" % (os.path.abspath(self.path))
script += """
from %s import %s as the_function
result = the_function(*channel.receive())
channel.send(result)
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
try:
sys.stdout.flush()
sys.stdout.close()
except:
pass
try:
sys.stderr.flush()
sys.stderr.close()
except:
pass
outchan.close()
errchan.close()
""" % (self.module, self.function_name)
channel = gw.remote_exec(script)
outchan = channel.receive()
errchan = channel.receive()
outchan.setcallback(lambda data: sys.stdout.write(str(data)))
errchan.setcallback(lambda data: sys.stderr.write(str(data)))
channel.send(args)
result = channel.receive()
# wait for proper shutdown
# https://bitbucket.org/hpk42/execnet/issues/26/redirecting-stdout-stderr-in-the-remote
outchan.waitclose()
errchan.waitclose()
return result
def get_topology(case_path_dict, debug_print=True):
gw = execnet.makegateway("popen//python={}".format(Python27Path))
channel = gw.remote_exec("import sys; from definitions import TOPOLOGY_DIR; sys.path.append(TOPOLOGY_DIR)")
channel = gw.remote_exec(pssetopology)
channel.send((
case_path_dict
))
end = channel.receive()
with open("temp", 'rb') as f:
topology = pickle.load(f, encoding='latin-1')
gw.exit()
return topology
