def test_socket_installvia(self): group = execnet.Group() group.makegateway("popen//id=p1") gw = group.makegateway("socket//installvia=p1//id=s1") assert gw.id == "s1" assert gw.remote_status() group.terminate()
def __init__(self,**options): data=numpy.load('datasets/facemodel/attributes.npz') self._fields=( '5_o_Clock_Shadow', 'Arched_Eyebrows', 'Attractive', 'Bags_Under_Eyes', 'Bald', 'Bangs', 'Big_Lips', 'Big_Nose', 'Black_Hair', 'Blond_Hair', 'Blurry', 'Brown_Hair', 'Bushy_Eyebrows', 'Chubby', 'Double_Chin', 'Eyeglasses', 'Goatee', 'Gray_Hair', 'Heavy_Makeup', 'High_Cheekbones', 'Male', 'Mouth_Slightly_Open', 'Mustache', 'Narrow_Eyes', 'No_Beard', 'Oval_Face', 'Pale_Skin', 'Pointy_Nose', 'Receding_Hairline', 'Rosy_Cheeks', 'Sideburns', 'Smiling', 'Straight_Hair', 'Wavy_Hair', 'Wearing_Earrings', 'Wearing_Hat', 'Wearing_Lipstick', 'Wearing_Necklace', 'Wearing_Necktie', 'Young', 'asian', 'baby', 'black', 'brown_eyes', 'child', 'color_photo', 'eyes_open', 'flash', 'flushed_face', 'frowning', 'fully_visible_forehead', 'harsh_lighting', 'indian', 'middle_aged', 'mouth_closed', 'mouth_wide_open', 'no_eyewear', 'obstructed_forehead', 'outdoor', 'partially_visible_forehead', 'posed_photo', 'round_face', 'round_jaw', 'senior', 'shiny_skin', 'soft_lighting', 'square_face', 'strong_nose_mouth_lines', 'sunglasses', 'teeth_not_visible', 'white' ) self.filelist=tuple(data['filelist']) self._scores=data['scores'] self._landmarks=data['landmarks'] self._binary_score=(self._scores>=0) # mark 70% as confident self._confident=numpy.zeros_like(self._scores,dtype=numpy.bool) self._confident[self._scores>numpy.percentile(self._scores[self._scores>=0],30,axis=0)]=True self._confident[self._scores<numpy.percentile(self._scores[self._scores<0],70,axis=0)]=True self.map_name={k:i for i,k in enumerate(self.filelist)} self.rev_map_name={i:k for i,k in enumerate(self.filelist)} # only attributes related to faces for nearest neighbor distance self.distance_idx=[self._fields.index(x) for x in [ '5_o_Clock_Shadow', 'Arched_Eyebrows', 'Bags_Under_Eyes', 'Bald', 'Bangs', 'Big_Lips', 'Big_Nose', 'Black_Hair', 'Blond_Hair', 'Brown_Hair', 'Bushy_Eyebrows', 'Eyeglasses', 'Goatee', 'Gray_Hair', 'Heavy_Makeup', 'Male', 'Mouth_Slightly_Open', 'Mustache', 'Narrow_Eyes', 'No_Beard', 'Pale_Skin', 'Pointy_Nose', 'Receding_Hairline', 'Sideburns', 'Smiling', 'Straight_Hair', 'Wavy_Hair', 'Wearing_Lipstick', 'Young', 'asian', 'baby', 'black', 'brown_eyes', 'child', 'eyes_open', 'frowning', 'fully_visible_forehead', 'indian', 'middle_aged', 'mouth_closed', 'mouth_wide_open', 'no_eyewear', 'obstructed_forehead', 'partially_visible_forehead', 'senior', 'shiny_skin', 'strong_nose_mouth_lines', 'sunglasses', 'teeth_not_visible', 'white' ]] # start classifer in another process since Torch and Caffe cannot # coexist in the same process import facemodel_worker import execnet self.group=execnet.Group() self.gw=self.group.makegateway() self.channel=self.gw.remote_exec(facemodel_worker) self.responseid=0 self.channel.send((('ping',self.responseid),{})) args,kwargs=self.channel.receive() self.raise_remote_errors(args,kwargs) assert args[0]=='okay' assert args[1]==self.responseid self.responseid+=1
def create_ui(self): self.tester = None self._group = execnet.Group() self.source_tree.set_columns([Column(title='Result', cells=[ Cell('icon', use_stock=True, expand=False), Cell('short_name', title='status',), ])]) self.source_tree.set_headers_visible(False) self.clear()
def test_multichannel_send_each(self): gm = execnet.Group(["popen"] * 2) mc = gm.remote_exec(""" import os channel.send(channel.receive() + 1) """) mc.send_each(41) l = mc.receive_each() assert l == [42, 42]
def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"): self.config = config self._nodesready = py.std.threading.Event() self.trace = self.config.trace.get("nodemanager") self.log = create_sublogger('nodeman') self.group = execnet.Group() self.specs = self._getxspecs() for spec in self.specs: self.group.allocate_id(spec)
def __init__(self, specs, hook, defaultchdir="pyexecnetcache"): self.specs = [] self.hook = hook self.group = execnet.Group() for spec in specs: if not isinstance(spec, execnet.XSpec): spec = execnet.XSpec(spec) if not spec.chdir and not spec.popen: spec.chdir = defaultchdir self.specs.append(spec)
def test_Group_execmodel_setting(self): gm = execnet.Group() gm.set_execmodel("thread") assert gm.execmodel.backend == "thread" assert gm.remote_execmodel.backend == "thread" gm._gateways.append(1) try: with pytest.raises(ValueError): gm.set_execmodel("eventlet") assert gm.execmodel.backend == "thread" finally: gm._gateways.pop()
def __init__(self, hosts, user): """ Register datacollectors of Vertica Arguments: hosts: list of host name or IP. """ self.hosts = hosts self.user = user self.executors = execnet.Group() # create executors in parallel self.initExecutersParallel()
def test_multichannel_receive_queue_for_two_subprocesses(self): gm = execnet.Group(["popen"] * 2) mc = gm.remote_exec(""" import os channel.send(os.getpid()) """) queue = mc.make_receive_queue() ch, item = queue.get(timeout=10) ch2, item2 = queue.get(timeout=10) assert ch != ch2 assert ch.gateway != ch2.gateway assert item != item2 mc.waitclose()
def __init__(self, settings=None): if settings is None: settings = self.default_settings if type(settings) is file: self.conf = json.load(settings) elif type(settings) is str: self.conf = json.loads(settings) elif type(settings) is dict: self.conf = settings else: raise (TypeError( "settings must be None, JSON string or JSON-parseable file")) self.gateways = execnet.Group() self.gws = [] for json_host in self.conf["hosts"]: self.add_gateway(json_host)
def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"): self.config = config self._nodesready = py.std.threading.Event() self.trace = self.config.trace.get("nodemanager") self.group = execnet.Group() if specs is None: specs = self._getxspecs() self.specs = [] for spec in specs: if not isinstance(spec, execnet.XSpec): spec = execnet.XSpec(spec) if not spec.chdir and not spec.popen: spec.chdir = defaultchdir self.group.allocate_id(spec) self.specs.append(spec) self.roots = self._getrsyncdirs()
def make_gateway(node, core, work_dir, endmarker): ''' Generate a fixed-length (1) group of gateway groups for cores in remote nodes. Each gateway group contains a singel gateway, which can be terminated later, making room for a new gateway. ''' def node_id(node): return re.findall(r'\d+', node)[0] group = execnet.Group() group.makegateway("ssh={0}//id={0}_{1}//chdir={2}".format( node, core, work_dir)) mch = group.remote_exec(job) queue = mch.make_receive_queue(endmarker=endmarker) # Either a channel is closed by job-completion (mch.send_each(None)) # or by force-termination, the 2nd term of queue.get() is endmarker. return group, mch, queue
def _receiveData(self): group = execnet.Group() data = {} for config in self.m_nodes: if config['cached']: continue node = config['node'] logging.info("Receiving data from {}".format(node)) gateway = self._getExecnetGateway(node, config['via']) try: group.makegateway(gateway) except execnet.HostNotFound as e: raise ScannerError("Failed to connect to remote host: " + str(e)) config['data'] = group[node].remote_exec(squinnie.probe).receive()
def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"): self.config = config self.trace = self.config.trace.get("nodemanager") self.testrunuid = self.config.getoption("testrunuid") if self.testrunuid is None: self.testrunuid = uuid.uuid4().hex self.group = execnet.Group() if specs is None: specs = self._getxspecs() self.specs = [] for spec in specs: if not isinstance(spec, execnet.XSpec): spec = execnet.XSpec(spec) if not spec.chdir and not spec.popen: spec.chdir = defaultchdir self.group.allocate_id(spec) self.specs.append(spec) self.roots = self._getrsyncdirs() self.rsyncoptions = self._getrsyncoptions() self._rsynced_specs = set()
def getCrowbarConfig(self): """ Creates a connection to the configured entry node and retrieves a machine listing from crowbar running there. Returns a dictionary of the form { "entry": ["machine1", "machine2", ...] } """ if not self.m_entry_node: raise squinnie.errors.ScannerError( "entry node for scanning crowbar network is required") group = execnet.Group() master = group.makegateway( "id=master//python=python{}//ssh=root@{}".format( 2, self.m_entry_node)) cmd = "crowbar machines list" exec_cmd = """ import subprocess channel.send(subprocess.check_output('{}')) """.format(cmd) try: crowbar_output = master.remote_exec(exec_cmd).receive() except execnet.RemoteError as e: raise squinnie.errors.ScannerError( "Failed to run crowbar on {}:\n\n{}".format( self.m_entry_node, e)) node_lines = crowbar_output.splitlines() # filter out empty lines node_lines = list(filter(None, node_lines)) try: node_lines.remove(self.m_entry_node) except ValueError: raise squinnie.errors.ScannerError( "entry node was not found in returned crowbar data") return {self.m_entry_node: [str(item) for item in node_lines]}
def createExecuter(self, i) : gw = None s = socket.socket() try: # check connectivity s.settimeout(3) s.connect((self.hosts[i], 22)) tg = execnet.Group() pythonPath = '/usr/bin/python2' # Note: other server maybe has different installation location than sys.executable, just use default. gw = tg.makegateway("ssh=%s@%s//id=%s//python=%s" % (self.user, self.hosts[i], self.hosts[i], pythonPath)) tg._unregister(gw) del gw._group except Exception: msg = 'ssh port 22 of server "%s" with user "%s" is not accessible! Ignore it, but you can not access newest info of this node.' % (self.hosts[i], self.user) print '\nERROR: %s' % msg logger.exception(msg) finally: s.close() return gw
def createExecuter(self, i): gw = None s = socket.socket() try: # check connectivity s.settimeout(3) s.connect((self.hostIPs[i], 22)) tg = execnet.Group() gw = tg.makegateway( "ssh=%s@%s//id=%s//python=/opt/vertica/oss/python/bin/python" % (self.vAdminOSUser, self.hostIPs[i], self.nodeNames[i])) tg._unregister(gw) del gw._group except Exception: msg = "ssh port 22 of Vertica node %s(%s) with user %s is not accessible! Ignore it, but you can not access newest info of this node." % ( self.nodeNames[i], self.hostIPs[i], self.vAdminOSUser) print "\nERROR: %s" % msg logger.exception(msg) finally: s.close() return gw
def __init__(self, vDbName='', vMetaFile='/opt/vertica/config/admintools.conf', vAdminOSUser='******'): """ Register datacollectors of Vertica Arguments: vDbName: string Vertica database name, if ignored, fist database in vMetaFile(/opt/vertica/config/admintools.conf) will be choosed. vMetaFile string Vertica database meta file vAdminOSUser string Vertica Administrator OS username """ # get configurations(vDbName, path, node, host) of Vertica database self.vDbName, self.catPath, self.nodeNames, self.hostIPs = getVerticaDBConfig( vDbName, vMetaFile) self.vAdminOSUser = vAdminOSUser self.executors = execnet.Group() # create executors in parallel self.initExecutersParallel()
def execnet_gateways(self): # TODO: Eventlet hangs on Mac OS X with popen. # execnet.set_execmodel("eventlet", "thread") gw = self.gateway if not gw: for _ in range(self.job_num): yield execnet.makegateway('' if 'FOWLER_PYTHON' not in os.environ else 'popen//python={}'. format(os.environ['FOWLER_PYTHON'])) for gateway_spec in gw: if '*' in gateway_spec: num, spec = gateway_spec.split('*') num = int(num) group = execnet.Group() group.defaultspec = spec xspec = execnet.XSpec(spec) master_spec = ('ssh={xspec.via}//' 'id={xspec.via}//' 'python={xspec.python}' ''.format(xspec=xspec)) logger.debug( 'Connecting to master %s to create %s gateways.', master_spec, num, ) group.makegateway(master_spec) for _ in range(num): yield group.makegateway() else: yield execnet.makegateway()
def _make_gateway(self, hostname): self.group = execnet.Group() gateway = self.group.makegateway( self._make_connection_string(hostname)) gateway.reconfigure(py2str_as_py3str=False, py3str_as_py2str=False) return gateway
def group(): g = execnet.Group() yield g g.terminate(timeout=1)
def get_nodes_specs(nodes, python=None, chdir=None, virtualenv_path=None, mem_per_process=None, max_processes=None, rsync_max_processes=None, rsync_bandwidth_limit=None, config=None): """Get nodes specs. Get list of node names, connect to each of them, get the system information, produce the list of node specs out of that information filtering non-connectable nodes and nodes which don't comply the requirements. Executed on the master node side. :param nodes: `list` of node names in form [[<username>@]<hostname>, ...] :type nodes: list :param python: python executable name to use on the remote side :type python: str :param chdir: relative path where to run (and sync) tests on the remote side :type chdir: str :param virtualenv_path: relative path to the virtualenv to activate on the remote test node :type virtualenv_path: str :param mem_per_process: optional amount of memory per process needed, in megabytest :type mem_per_process: int :param max_processes: optional maximum number of processes per test node :type max_processes: int :param rsync_max_processes: optional maximum number of rsync processes :type rsync_max_processes: int :param rsync_bandwidth_limit: optional bandwidth limit per rsync process in kilobytes per second :type rsync_bandwidth_limit: int :param config: pytest config object :type config: pytest.Config :return: `list` of test gateway specs for all test nodes which confirm given requirements in form ['1*ssh=<node>//id=<hostname>:<index>', ...] :rtype: list """ group = execnet.Group() try: if virtualenv_path: nm = NodeManager(config, specs=[]) virtualenv_path = os.path.relpath(virtualenv_path) node_specs = [] node_caps = {} root_dir = getrootdir(config, '') nodes = list(unique_everseen(nodes)) print('Detected root dir: {0}'.format(root_dir)) rsync = RSync(root_dir, chdir, includes=config.getini("rsyncdirs"), jobs=rsync_max_processes or len(nodes), bwlimit=rsync_bandwidth_limit, bandwidth_limit=rsync_bandwidth_limit, **nm.rsyncoptions) print('Detecting connectable test nodes...') for node in nodes: host = node.split('@')[1] if '@' in node else node spec = 'ssh={node}//id={host}//chdir={chdir}//python={python}'.format( node=node, host=host, chdir=chdir, python=python) try: make_gateway(group, spec) except Exception: # pylint: disable=W0703 continue rsync.add_target_host(node) node_specs.append((node, host)) if node_specs: print('Found {0} connectable test nodes: {1}'.format( len(node_specs), rsync.targets)) else: pytest.exit('None of the given test nodes are connectable') print('RSyncing directory structure') rsync.send() print('RSync finished') develop_eggs = get_develop_eggs(root_dir, config) group.remote_exec(patches.activate_env, virtualenv_path=virtualenv_path, develop_eggs=develop_eggs).waitclose() multi_channel = group.remote_exec(get_node_capabilities) try: caps = multi_channel.receive_each(True) for ch, cap in caps: node_caps[ch.gateway.id] = cap finally: multi_channel.waitclose() return list( chain.from_iterable( get_node_specs( node, hst, node_caps[hst], python=os.path.join(chdir, virtualenv_path, 'bin', python), chdir=chdir, mem_per_process=mem_per_process, max_processes=max_processes) for node, hst in node_specs)) finally: try: group.terminate() except Exception: # pylint: disable=W0703 pass
def main(): if len(sys.argv) != 2: print >> sys.stderr, 'Usage: python screenshot_analyze.py directory' sys.exit(1) subdirs = os.listdir(sys.argv[1]) prog_levels = [] for level in range(0,101,10): prog_levels.append([]) for meshdir in subdirs: meshfiles = os.listdir(os.path.join(sys.argv[1], meshdir)) orig_ss = None prog_ss_levels = [] for meshfile in meshfiles: if fnmatch.fnmatch(meshfile, '*orig.zip.png'): orig_ss = os.path.join(sys.argv[1], meshdir, meshfile) for level in range(0,101,10): for meshfile in meshfiles: if fnmatch.fnmatch(meshfile, '*progressive.%d.png' % level): prog_ss_levels.append(os.path.join(sys.argv[1], meshdir, meshfile)) if orig_ss is None or len(prog_ss_levels) != 11: print 'SKIPPING', meshdir for i, prog_ss in enumerate(prog_ss_levels): prog_levels[i].append((orig_ss, prog_ss, meshdir)) tasks = [] for i, level in enumerate(prog_levels): for orig_ss, prog_ss, meshdir in level: tasks.append((i, meshdir, open(orig_ss, 'rb').read(), open(prog_ss, 'rb').read())) group = execnet.Group() todo = [27,28,29,30,31,32,33,34,35,36,37,38,40,41,42,43,44] for i in todo: for core in range(8): group.makegateway("ssh=sns%d.cs.princeton.edu//id=sns%d-%d" % (i,i,core)) mch = group.remote_exec(process_item) q = mch.make_receive_queue(endmarker=-1) terminated = 0 while True: if tasks and tasks != -1: print 'Tasks left', len(tasks), 'terminated', terminated channel, item = q.get() if item == -1: terminated += 1 print "terminated %s" % channel.gateway.id if terminated == len(mch): print "got all results, terminating" break continue if item != "ready": print "other side %s returned" % (channel.gateway.id) level, meshdir, data = item f = open(os.path.join(sys.argv[1], meshdir, 'results-%d.txt' % level), 'w') for d in data: f.write("%f\n" % d) f.close() if not tasks: print "no tasks remain, sending termination request to all" mch.send_each(None) tasks = -1 if tasks and tasks != -1: task = tasks.pop() channel.send(task) print "sent task to %s" % (channel.gateway.id)
def group(request): group = execnet.Group() request.addfinalizer(group.terminate) return group
def makegateway(request): group = execnet.Group() request.addfinalizer(lambda: group.terminate(0.5)) return group.makegateway
def pypy_render(request): w = int(request.GET.get('w', 320)) h = int(request.GET.get('h', 240)) channel = pypy.remote_exec(""" from py_mandel import mandelbrot w, h = channel.receive() img = mandelbrot(w, h) channel.send(img) """) channel.send((w, h)) img = channel.receive() return HttpResponse(img, content_type="image/bmp") # setup execnet and pypy child # # The setup is done here so that a single pypy-c is started and reused for all # requests. sys.path and cwd are set up so that py_mandel can be imported # from remote_exec import execnet mygroup = execnet.Group() pypy = mygroup.makegateway("popen//python=pypy-c") pypy.remote_exec(""" import sys import os os.chdir("mandelbrot") sys.path.insert(0, '') """)