def test_socket_installvia(self):
group = execnet.Group()
group.makegateway("popen//id=p1")
gw = group.makegateway("socket//installvia=p1//id=s1")
assert gw.id == "s1"
assert gw.remote_status()
group.terminate()
def __init__(self,**options):
data=numpy.load('datasets/facemodel/attributes.npz')
self._fields=( '5_o_Clock_Shadow', 'Arched_Eyebrows', 'Attractive', 'Bags_Under_Eyes', 'Bald', 'Bangs', 'Big_Lips', 'Big_Nose', 'Black_Hair', 'Blond_Hair', 'Blurry', 'Brown_Hair', 'Bushy_Eyebrows', 'Chubby', 'Double_Chin', 'Eyeglasses', 'Goatee', 'Gray_Hair', 'Heavy_Makeup', 'High_Cheekbones', 'Male', 'Mouth_Slightly_Open', 'Mustache', 'Narrow_Eyes', 'No_Beard', 'Oval_Face', 'Pale_Skin', 'Pointy_Nose', 'Receding_Hairline', 'Rosy_Cheeks', 'Sideburns', 'Smiling', 'Straight_Hair', 'Wavy_Hair', 'Wearing_Earrings', 'Wearing_Hat', 'Wearing_Lipstick', 'Wearing_Necklace', 'Wearing_Necktie', 'Young', 'asian', 'baby', 'black', 'brown_eyes', 'child', 'color_photo', 'eyes_open', 'flash', 'flushed_face', 'frowning', 'fully_visible_forehead', 'harsh_lighting', 'indian', 'middle_aged', 'mouth_closed', 'mouth_wide_open', 'no_eyewear', 'obstructed_forehead', 'outdoor', 'partially_visible_forehead', 'posed_photo', 'round_face', 'round_jaw', 'senior', 'shiny_skin', 'soft_lighting', 'square_face', 'strong_nose_mouth_lines', 'sunglasses', 'teeth_not_visible', 'white' )
self.filelist=tuple(data['filelist'])
self._scores=data['scores']
self._landmarks=data['landmarks']
self._binary_score=(self._scores>=0)
# mark 70% as confident
self._confident=numpy.zeros_like(self._scores,dtype=numpy.bool)
self._confident[self._scores>numpy.percentile(self._scores[self._scores>=0],30,axis=0)]=True
self._confident[self._scores<numpy.percentile(self._scores[self._scores<0],70,axis=0)]=True
self.map_name={k:i for i,k in enumerate(self.filelist)}
self.rev_map_name={i:k for i,k in enumerate(self.filelist)}
# only attributes related to faces for nearest neighbor distance
self.distance_idx=[self._fields.index(x) for x in [ '5_o_Clock_Shadow', 'Arched_Eyebrows', 'Bags_Under_Eyes', 'Bald', 'Bangs', 'Big_Lips', 'Big_Nose', 'Black_Hair', 'Blond_Hair', 'Brown_Hair', 'Bushy_Eyebrows', 'Eyeglasses', 'Goatee', 'Gray_Hair', 'Heavy_Makeup', 'Male', 'Mouth_Slightly_Open', 'Mustache', 'Narrow_Eyes', 'No_Beard', 'Pale_Skin', 'Pointy_Nose', 'Receding_Hairline', 'Sideburns', 'Smiling', 'Straight_Hair', 'Wavy_Hair', 'Wearing_Lipstick', 'Young', 'asian', 'baby', 'black', 'brown_eyes', 'child', 'eyes_open', 'frowning', 'fully_visible_forehead', 'indian', 'middle_aged', 'mouth_closed', 'mouth_wide_open', 'no_eyewear', 'obstructed_forehead', 'partially_visible_forehead', 'senior', 'shiny_skin', 'strong_nose_mouth_lines', 'sunglasses', 'teeth_not_visible', 'white' ]]
# start classifer in another process since Torch and Caffe cannot
# coexist in the same process
import facemodel_worker
import execnet
self.group=execnet.Group()
self.gw=self.group.makegateway()
self.channel=self.gw.remote_exec(facemodel_worker)
self.responseid=0
self.channel.send((('ping',self.responseid),{}))
args,kwargs=self.channel.receive()
self.raise_remote_errors(args,kwargs)
assert args[0]=='okay'
assert args[1]==self.responseid
self.responseid+=1
def create_ui(self):
self.tester = None
self._group = execnet.Group()
self.source_tree.set_columns([Column(title='Result', cells=[
Cell('icon', use_stock=True, expand=False),
Cell('short_name', title='status',),
])])
self.source_tree.set_headers_visible(False)
self.clear()
def test_multichannel_send_each(self):
gm = execnet.Group(["popen"] * 2)
mc = gm.remote_exec("""
import os
channel.send(channel.receive() + 1)
""")
mc.send_each(41)
l = mc.receive_each()
assert l == [42, 42]
def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"):
self.config = config
self._nodesready = py.std.threading.Event()
self.trace = self.config.trace.get("nodemanager")
self.log = create_sublogger('nodeman')
self.group = execnet.Group()
self.specs = self._getxspecs()
for spec in self.specs:
self.group.allocate_id(spec)
def __init__(self, specs, hook, defaultchdir="pyexecnetcache"):
self.specs = []
self.hook = hook
self.group = execnet.Group()
for spec in specs:
if not isinstance(spec, execnet.XSpec):
spec = execnet.XSpec(spec)
if not spec.chdir and not spec.popen:
spec.chdir = defaultchdir
self.specs.append(spec)
def test_Group_execmodel_setting(self):
gm = execnet.Group()
gm.set_execmodel("thread")
assert gm.execmodel.backend == "thread"
assert gm.remote_execmodel.backend == "thread"
gm._gateways.append(1)
try:
with pytest.raises(ValueError):
gm.set_execmodel("eventlet")
assert gm.execmodel.backend == "thread"
finally:
gm._gateways.pop()
def __init__(self, hosts, user):
""" Register datacollectors of Vertica
Arguments:
hosts: list of host name or IP.
"""
self.hosts = hosts
self.user = user
self.executors = execnet.Group()
# create executors in parallel
self.initExecutersParallel()
def test_multichannel_receive_queue_for_two_subprocesses(self):
gm = execnet.Group(["popen"] * 2)
mc = gm.remote_exec("""
import os
channel.send(os.getpid())
""")
queue = mc.make_receive_queue()
ch, item = queue.get(timeout=10)
ch2, item2 = queue.get(timeout=10)
assert ch != ch2
assert ch.gateway != ch2.gateway
assert item != item2
mc.waitclose()
def __init__(self, settings=None):
if settings is None:
settings = self.default_settings
if type(settings) is file:
self.conf = json.load(settings)
elif type(settings) is str:
self.conf = json.loads(settings)
elif type(settings) is dict:
self.conf = settings
else:
raise (TypeError(
"settings must be None, JSON string or JSON-parseable file"))
self.gateways = execnet.Group()
self.gws = []
for json_host in self.conf["hosts"]:
self.add_gateway(json_host)
def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"):
self.config = config
self._nodesready = py.std.threading.Event()
self.trace = self.config.trace.get("nodemanager")
self.group = execnet.Group()
if specs is None:
specs = self._getxspecs()
self.specs = []
for spec in specs:
if not isinstance(spec, execnet.XSpec):
spec = execnet.XSpec(spec)
if not spec.chdir and not spec.popen:
spec.chdir = defaultchdir
self.group.allocate_id(spec)
self.specs.append(spec)
self.roots = self._getrsyncdirs()
def make_gateway(node, core, work_dir, endmarker):
''' Generate a fixed-length (1) group of gateway groups for cores
in remote nodes. Each gateway group contains a singel gateway,
which can be terminated later, making room for a new gateway.
'''
def node_id(node):
return re.findall(r'\d+', node)[0]
group = execnet.Group()
group.makegateway("ssh={0}//id={0}_{1}//chdir={2}".format(
node, core, work_dir))
mch = group.remote_exec(job)
queue = mch.make_receive_queue(endmarker=endmarker)
# Either a channel is closed by job-completion (mch.send_each(None))
# or by force-termination, the 2nd term of queue.get() is endmarker.
return group, mch, queue
def _receiveData(self):
group = execnet.Group()
data = {}
for config in self.m_nodes:
if config['cached']:
continue
node = config['node']
logging.info("Receiving data from {}".format(node))
gateway = self._getExecnetGateway(node, config['via'])
try:
group.makegateway(gateway)
except execnet.HostNotFound as e:
raise ScannerError("Failed to connect to remote host: " +
str(e))
config['data'] = group[node].remote_exec(squinnie.probe).receive()
def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"):
self.config = config
self.trace = self.config.trace.get("nodemanager")
self.testrunuid = self.config.getoption("testrunuid")
if self.testrunuid is None:
self.testrunuid = uuid.uuid4().hex
self.group = execnet.Group()
if specs is None:
specs = self._getxspecs()
self.specs = []
for spec in specs:
if not isinstance(spec, execnet.XSpec):
spec = execnet.XSpec(spec)
if not spec.chdir and not spec.popen:
spec.chdir = defaultchdir
self.group.allocate_id(spec)
self.specs.append(spec)
self.roots = self._getrsyncdirs()
self.rsyncoptions = self._getrsyncoptions()
self._rsynced_specs = set()
def getCrowbarConfig(self):
"""
Creates a connection to the configured entry node and retrieves a
machine listing from crowbar running there.
Returns a dictionary of the form {
"entry": ["machine1", "machine2", ...]
}
"""
if not self.m_entry_node:
raise squinnie.errors.ScannerError(
"entry node for scanning crowbar network is required")
group = execnet.Group()
master = group.makegateway(
"id=master//python=python{}//ssh=root@{}".format(
2, self.m_entry_node))
cmd = "crowbar machines list"
exec_cmd = """
import subprocess
channel.send(subprocess.check_output('{}'))
""".format(cmd)
try:
crowbar_output = master.remote_exec(exec_cmd).receive()
except execnet.RemoteError as e:
raise squinnie.errors.ScannerError(
"Failed to run crowbar on {}:\n\n{}".format(
self.m_entry_node, e))
node_lines = crowbar_output.splitlines()
# filter out empty lines
node_lines = list(filter(None, node_lines))
try:
node_lines.remove(self.m_entry_node)
except ValueError:
raise squinnie.errors.ScannerError(
"entry node was not found in returned crowbar data")
return {self.m_entry_node: [str(item) for item in node_lines]}
def createExecuter(self, i) :
gw = None
s = socket.socket()
try:
# check connectivity
s.settimeout(3)
s.connect((self.hosts[i], 22))
tg = execnet.Group()
pythonPath = '/usr/bin/python2' # Note: other server maybe has different installation location than sys.executable, just use default.
gw = tg.makegateway("ssh=%s@%s//id=%s//python=%s" % (self.user, self.hosts[i], self.hosts[i], pythonPath))
tg._unregister(gw)
del gw._group
except Exception:
msg = 'ssh port 22 of server "%s" with user "%s" is not accessible! Ignore it, but you can not access newest info of this node.' % (self.hosts[i], self.user)
print '\nERROR: %s' % msg
logger.exception(msg)
finally:
s.close()
return gw
def createExecuter(self, i):
gw = None
s = socket.socket()
try:
# check connectivity
s.settimeout(3)
s.connect((self.hostIPs[i], 22))
tg = execnet.Group()
gw = tg.makegateway(
"ssh=%s@%s//id=%s//python=/opt/vertica/oss/python/bin/python" %
(self.vAdminOSUser, self.hostIPs[i], self.nodeNames[i]))
tg._unregister(gw)
del gw._group
except Exception:
msg = "ssh port 22 of Vertica node %s(%s) with user %s is not accessible! Ignore it, but you can not access newest info of this node." % (
self.nodeNames[i], self.hostIPs[i], self.vAdminOSUser)
print "\nERROR: %s" % msg
logger.exception(msg)
finally:
s.close()
return gw
def __init__(self,
vDbName='',
vMetaFile='/opt/vertica/config/admintools.conf',
vAdminOSUser='******'):
""" Register datacollectors of Vertica
Arguments:
vDbName: string
Vertica database name, if ignored, fist database in vMetaFile(/opt/vertica/config/admintools.conf) will be choosed.
vMetaFile string
Vertica database meta file
vAdminOSUser string
Vertica Administrator OS username
"""
# get configurations(vDbName, path, node, host) of Vertica database
self.vDbName, self.catPath, self.nodeNames, self.hostIPs = getVerticaDBConfig(
vDbName, vMetaFile)
self.vAdminOSUser = vAdminOSUser
self.executors = execnet.Group()
# create executors in parallel
self.initExecutersParallel()
def execnet_gateways(self):
# TODO: Eventlet hangs on Mac OS X with popen.
# execnet.set_execmodel("eventlet", "thread")
gw = self.gateway
if not gw:
for _ in range(self.job_num):
yield execnet.makegateway('' if 'FOWLER_PYTHON' not in
os.environ else 'popen//python={}'.
format(os.environ['FOWLER_PYTHON']))
for gateway_spec in gw:
if '*' in gateway_spec:
num, spec = gateway_spec.split('*')
num = int(num)
group = execnet.Group()
group.defaultspec = spec
xspec = execnet.XSpec(spec)
master_spec = ('ssh={xspec.via}//'
'id={xspec.via}//'
'python={xspec.python}'
''.format(xspec=xspec))
logger.debug(
'Connecting to master %s to create %s gateways.',
master_spec,
num,
)
group.makegateway(master_spec)
for _ in range(num):
yield group.makegateway()
else:
yield execnet.makegateway()
def _make_gateway(self, hostname):
self.group = execnet.Group()
gateway = self.group.makegateway(
self._make_connection_string(hostname))
gateway.reconfigure(py2str_as_py3str=False, py3str_as_py2str=False)
return gateway
def group():
g = execnet.Group()
yield g
g.terminate(timeout=1)
def get_nodes_specs(nodes,
python=None,
chdir=None,
virtualenv_path=None,
mem_per_process=None,
max_processes=None,
rsync_max_processes=None,
rsync_bandwidth_limit=None,
config=None):
"""Get nodes specs.
Get list of node names, connect to each of them, get the system information, produce the list of node specs out of
that information filtering non-connectable nodes and nodes which don't comply the requirements.
Executed on the master node side.
:param nodes: `list` of node names in form [[<username>@]<hostname>, ...]
:type nodes: list
:param python: python executable name to use on the remote side
:type python: str
:param chdir: relative path where to run (and sync) tests on the remote side
:type chdir: str
:param virtualenv_path: relative path to the virtualenv to activate on the remote test node
:type virtualenv_path: str
:param mem_per_process: optional amount of memory per process needed, in megabytest
:type mem_per_process: int
:param max_processes: optional maximum number of processes per test node
:type max_processes: int
:param rsync_max_processes: optional maximum number of rsync processes
:type rsync_max_processes: int
:param rsync_bandwidth_limit: optional bandwidth limit per rsync process in kilobytes per second
:type rsync_bandwidth_limit: int
:param config: pytest config object
:type config: pytest.Config
:return: `list` of test gateway specs for all test nodes which confirm given requirements
in form ['1*ssh=<node>//id=<hostname>:<index>', ...]
:rtype: list
"""
group = execnet.Group()
try:
if virtualenv_path:
nm = NodeManager(config, specs=[])
virtualenv_path = os.path.relpath(virtualenv_path)
node_specs = []
node_caps = {}
root_dir = getrootdir(config, '')
nodes = list(unique_everseen(nodes))
print('Detected root dir: {0}'.format(root_dir))
rsync = RSync(root_dir,
chdir,
includes=config.getini("rsyncdirs"),
jobs=rsync_max_processes or len(nodes),
bwlimit=rsync_bandwidth_limit,
bandwidth_limit=rsync_bandwidth_limit,
**nm.rsyncoptions)
print('Detecting connectable test nodes...')
for node in nodes:
host = node.split('@')[1] if '@' in node else node
spec = 'ssh={node}//id={host}//chdir={chdir}//python={python}'.format(
node=node, host=host, chdir=chdir, python=python)
try:
make_gateway(group, spec)
except Exception: # pylint: disable=W0703
continue
rsync.add_target_host(node)
node_specs.append((node, host))
if node_specs:
print('Found {0} connectable test nodes: {1}'.format(
len(node_specs), rsync.targets))
else:
pytest.exit('None of the given test nodes are connectable')
print('RSyncing directory structure')
rsync.send()
print('RSync finished')
develop_eggs = get_develop_eggs(root_dir, config)
group.remote_exec(patches.activate_env,
virtualenv_path=virtualenv_path,
develop_eggs=develop_eggs).waitclose()
multi_channel = group.remote_exec(get_node_capabilities)
try:
caps = multi_channel.receive_each(True)
for ch, cap in caps:
node_caps[ch.gateway.id] = cap
finally:
multi_channel.waitclose()
return list(
chain.from_iterable(
get_node_specs(
node,
hst,
node_caps[hst],
python=os.path.join(chdir, virtualenv_path, 'bin', python),
chdir=chdir,
mem_per_process=mem_per_process,
max_processes=max_processes) for node, hst in node_specs))
finally:
try:
group.terminate()
except Exception: # pylint: disable=W0703
pass
def main():
if len(sys.argv) != 2:
print >> sys.stderr, 'Usage: python screenshot_analyze.py directory'
sys.exit(1)
subdirs = os.listdir(sys.argv[1])
prog_levels = []
for level in range(0,101,10):
prog_levels.append([])
for meshdir in subdirs:
meshfiles = os.listdir(os.path.join(sys.argv[1], meshdir))
orig_ss = None
prog_ss_levels = []
for meshfile in meshfiles:
if fnmatch.fnmatch(meshfile, '*orig.zip.png'):
orig_ss = os.path.join(sys.argv[1], meshdir, meshfile)
for level in range(0,101,10):
for meshfile in meshfiles:
if fnmatch.fnmatch(meshfile, '*progressive.%d.png' % level):
prog_ss_levels.append(os.path.join(sys.argv[1], meshdir, meshfile))
if orig_ss is None or len(prog_ss_levels) != 11:
print 'SKIPPING', meshdir
for i, prog_ss in enumerate(prog_ss_levels):
prog_levels[i].append((orig_ss, prog_ss, meshdir))
tasks = []
for i, level in enumerate(prog_levels):
for orig_ss, prog_ss, meshdir in level:
tasks.append((i, meshdir, open(orig_ss, 'rb').read(), open(prog_ss, 'rb').read()))
group = execnet.Group()
todo = [27,28,29,30,31,32,33,34,35,36,37,38,40,41,42,43,44]
for i in todo:
for core in range(8):
group.makegateway("ssh=sns%d.cs.princeton.edu//id=sns%d-%d" % (i,i,core))
mch = group.remote_exec(process_item)
q = mch.make_receive_queue(endmarker=-1)
terminated = 0
while True:
if tasks and tasks != -1:
print 'Tasks left', len(tasks), 'terminated', terminated
channel, item = q.get()
if item == -1:
terminated += 1
print "terminated %s" % channel.gateway.id
if terminated == len(mch):
print "got all results, terminating"
break
continue
if item != "ready":
print "other side %s returned" % (channel.gateway.id)
level, meshdir, data = item
f = open(os.path.join(sys.argv[1], meshdir, 'results-%d.txt' % level), 'w')
for d in data:
f.write("%f\n" % d)
f.close()
if not tasks:
print "no tasks remain, sending termination request to all"
mch.send_each(None)
tasks = -1
if tasks and tasks != -1:
task = tasks.pop()
channel.send(task)
print "sent task to %s" % (channel.gateway.id)
def group(request):
group = execnet.Group()
request.addfinalizer(group.terminate)
return group
def makegateway(request):
group = execnet.Group()
request.addfinalizer(lambda: group.terminate(0.5))
return group.makegateway
def pypy_render(request):
w = int(request.GET.get('w', 320))
h = int(request.GET.get('h', 240))
channel = pypy.remote_exec("""
from py_mandel import mandelbrot
w, h = channel.receive()
img = mandelbrot(w, h)
channel.send(img)
""")
channel.send((w, h))
img = channel.receive()
return HttpResponse(img, content_type="image/bmp")
# setup execnet and pypy child
#
# The setup is done here so that a single pypy-c is started and reused for all
# requests. sys.path and cwd are set up so that py_mandel can be imported
# from remote_exec
import execnet
mygroup = execnet.Group()
pypy = mygroup.makegateway("popen//python=pypy-c")
pypy.remote_exec("""
import sys
import os
os.chdir("mandelbrot")
sys.path.insert(0, '')
""")
