def _poll_mode(self): """ Read and write to device in polling mode. """ pi = select.poll() po = select.poll() for fd in self.in_files: pi.register(fd, select.POLLIN) for fd in self.out_files: po.register(fd, select.POLLOUT) while not self.exit_thread.isSet(): data = b"" t_out = self.out_files readyf = pi.poll(1.0) for i in readyf: data += os.read(i[0], self.cachesize) if data != b"": while ((len(t_out) != len(readyf)) and not self.exit_thread.isSet()): readyf = po.poll(1.0) for desc in t_out: os.write(desc, data)
def _setup_poll_function(self): try: select.poll() except: self._poll_function = asyncore.poll else: self._poll_function = asyncore.poll2
def startSkulltag(self, deadArg): self.skulltag = subprocess.Popen(['/usr/games/skulltag/skulltag-server', '+sv_markchatlines 1']+self.args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=0) self.stdinPoll = select.poll() self.stdinPoll.register(self.skulltag.stdout, select.POLLIN) self.stdoutPoll = select.poll() self.stdoutPoll.register(self.skulltag.stdout, select.POLLOUT) thread.start_new(self.rwLoop, (None,))
def __init__(self, nnpy_socket): self.send = nnpy_socket.send self.recv = nnpy_socket.recv self.close = nnpy_socket.close self.setsockopt = nnpy_socket.setsockopt self.getsockopt = nnpy_socket.getsockopt # construct poll function try: read_p = select.poll() read_fd = nnpy_socket.getsockopt(nnpy.SOL_SOCKET, nnpy.RCVFD) read_p.register(read_fd, select.POLLIN) def check_readable(): return read_p.poll(0) != [] self.check_readable = check_readable except: self.check_readable = None try: write_p = select.poll() write_fd = nnpy_socket.getsockopt(nnpy.SOL_SOCKET, nnpy.SNDFD) write_p.register(write_fd, select.POLLOUT) def check_writeable(): return write_p.poll(0) != [] self.check_writeable = check_writeable except: self.check_writeable = None
def test_poll(self): import select class A(object): def __int__(self): return 3 select.poll().poll(A()) # assert did not crash
def waitForNode( self, node ): "Wait for a node to finish, and print its output." # Pollers nodePoller = poll() nodePoller.register( node.stdout ) bothPoller = poll() bothPoller.register( self.stdin ) bothPoller.register( node.stdout ) while True: try: bothPoller.poll() # XXX BL: this doesn't quite do what we want. if False and self.inputFile: key = self.inputFile.read( 1 ) if key is not '': node.write(key) else: self.inputFile = None if isReadable( self.inPoller ): key = self.stdin.read( 1 ) node.write( key ) if isReadable( nodePoller ): data = node.monitor() output( data ) if not node.waiting: break except KeyboardInterrupt: node.sendInt()
def waitForNode( self, node ): "Wait for a node to finish, and print its output." # Pollers nodePoller = poll() nodePoller.register( node.stdout ) bothPoller = poll() bothPoller.register( self.stdin, POLLIN ) bothPoller.register( node.stdout, POLLIN ) if self.isatty(): # Buffer by character, so that interactive # commands sort of work quietRun( 'stty -icanon min 1' ) while True: try: bothPoller.poll() # XXX BL: this doesn't quite do what we want. if False and self.inputFile: key = self.inputFile.read( 1 ) if key is not '': node.write(key) else: self.inputFile = None if isReadable( self.inPoller ): key = self.stdin.read( 1 ) node.write( key ) if isReadable( nodePoller ): data = node.monitor() output( data ) if not node.waiting: break except KeyboardInterrupt: node.sendInt()
def __init__(self, fd): self.fd = fd self._r, self._w = os.pipe() self._wp = select.poll() self._wp.register(self._w, select.POLLOUT) self._rp = select.poll() self._rp.register(self.fd, select.POLLIN) self._rp.register(self._r, select.POLLIN)
def __init__(self): ImmortalThread.__init__(self, name="Redusa Async Core") self._use_poll = 0 try: select.poll() self._use_poll = 1 except: msglog.log('broadway', msglog.types.INFO, 'Platform does not support poll().') return
def main(): args = aux.parse_args("Echo server with poll-processing model.") server_socket = aux.listening_socket(args.port) server_fileno = server_socket.fileno() fileno_to_data = {} fileno_to_socket = {} read_poll = select.poll() read_poll.register(server_fileno, select.POLLIN) while True: for fileno, eventmask in read_poll.poll(50): if fileno == server_fileno: client_socket, _ = server_socket.accept() client_fileno = client_socket.fileno() fileno_to_data[client_fileno] = "" fileno_to_socket[client_fileno] = client_socket read_poll.register(client_fileno, select.POLLIN) else: client_socket = fileno_to_socket[fileno] data = client_socket.recv(1024) if not data: read_poll.unregister(fileno) del fileno_to_data[fileno] del fileno_to_socket[fileno] client_socket.close() else: fileno_to_data[fileno] += data check_writability = [f for f, d in fileno_to_data.iteritems() if d] if not check_writability: continue write_poll = select.poll() for fileno in check_writability: write_poll.register(fileno, select.POLLOUT) for fileno, eventmask in write_poll.poll(50): if eventmask & (select.POLLERR | select.POLLHUP): read_poll.unregister(fileno) fileno_to_socket[fileno].close() del fileno_to_data[fileno] del fileno_to_socket[fileno] continue client_socket = fileno_to_socket[fileno] data = fileno_to_data[fileno] n = client_socket.send(data) if n > 0: fileno_to_data[fileno] = data[n:]
def test_poll(self): import select readend, writeend = self.getpair() try: class A(object): def __int__(self): return readend.fileno() select.poll().poll(A()) # assert did not crash finally: readend.close() writeend.close()
def test_poll(self): import select if not hasattr(select, 'poll'): skip("no select.poll() on this platform") readend, writeend = self.getpair() try: class A(object): def __int__(self): return readend.fileno() select.poll().poll(A()) # assert did not crash finally: readend.close() writeend.close()
def test_select_not_ready_with_poll(self): s = self._MockSocket() callback = self.mox.CreateMockAnything() poll = self.mox.CreateMockAnything() select.poll().AndReturn(poll) poll.register(s.fileno(), select.POLLIN) poll.poll(1000).AndReturn([]) self.mox.ReplayAll() self.select_thread.add_socket(s, callback) self.select_thread._select() self.mox.VerifyAll()
def configure(self, config): CompositeNode.configure(self, config) set_attribute(self, 'ncounters', REQUIRED, config, int) set_attribute(self, 'nDIs', REQUIRED, config, int) set_attribute(self, 'nrelays', REQUIRED, config, int) set_attribute(self, 'ndallas_busses', REQUIRED, config, int) set_attribute(self, 'nGPIOs', REQUIRED, config, int) # Open the avr devices. self.avr = None self.avroob = None try: self.avr = open("/dev/avr", "r+") self.avroob = open("/dev/avroob", "r") self.p = select.poll() self.p.register(self.avroob, select.POLLIN) avr_maj_ver = ord(self.invoke_message('\x17\x00\x00')[0]) if (avr_maj_ver < 2) and (self.nGPIOs > 0): self.nGPIOs = 0 msglog.log('mpx',msglog.types.ERR,'No GPIOs created; AVR version is %s; should be 2.x or greater.' \ % self.version()) # Attach the counters, relays and dallas busses to the AVR. config_list = (('mpx.ion.host.avr.counter', 'counter', self.ncounters), ('mpx.ion.host.avr.di', 'DI', self.nDIs), ('mpx.ion.host.avr.relay', 'relay', self.nrelays), ('mpx.ion.host.avr.dallasbus', 'dallas', self.ndallas_busses), ('mpx.ion.host.avr.gpio', 'gpio', self.nGPIOs)) for module,prefix,count in config_list: for i in range(1,count+1): name = prefix + str(i) config = {'name':name, 'id':i, 'avr':self, 'parent':self} ion = mpx.lib.factory(module) ion.configure(config) except: msglog.log('broadway',msglog.types.ERR,"Failed to open avr device.") msglog.exception() self.p = select.poll() if self.avr: self.avr.close() self.avr = None if self.avroob: self.avroob.close() self.avroob = None pass return
def is_connection_dropped(conn): # Platform-specific """ Returns True if the connection is dropped and should be closed. :param conn: :class:`httplib.HTTPConnection` object. Note: For platforms like AppEngine, this will always return ``False`` to let the wxplatform handle connection recycling transparently for us. """ sock = getattr(conn, 'sock', False) if sock is False: # Platform-specific: AppEngine return False if sock is None: # Connection already closed (such as by httplib). return True if not poll: if not select: # Platform-specific: AppEngine return False try: return select([sock], [], [], 0.0)[0] except socket.error: return True # This version is better on platforms that support it. p = poll() p.register(sock, POLLIN) for (fno, ev) in p.poll(0.0): if fno == sock.fileno(): # Either data is buffered (bad), or the connection is dropped. return True
def startShell( self ): if self.shell: error( "%s: shell is already running" ) return subprocess.call(["docker stop "+self.name], shell=True, stdout=output) subprocess.call(["docker rm -f "+self.name], shell=True, stdout=output) cmd = ["docker","run","--privileged","-h",self.name ,"--name="+self.name,"-v", "/vagrant:/home/ubuntu"] if self.dargs is not None: cmd.extend([self.dargs]) cmd.extend(["--net='none'",self.image, self.startString]) self.shell = Popen( cmd, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True ) self.stdin = self.shell.stdin self.stdout = self.shell.stdout self.pid = self.shell.pid self.pollOut = select.poll() self.pollOut.register( self.stdout ) self.outToNode[ self.stdout.fileno() ] = self self.inToNode[ self.stdin.fileno() ] = self self.execed = False self.lastCmd = None self.lastPid = None self.readbuf = '' self.waiting = False call("sleep 1", shell=True) pid_cmd = ["docker","inspect","--format='{{ .State.Pid }}'",""+self.name] pidp = Popen( pid_cmd, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=False ) ps_out = pidp.stdout.readlines() self.pid = int(ps_out[0])
def poll2(timeout=0.0, map=None): # Use the poll() support added to the select module in Python 2.0 if map is None: map = socket_map if timeout is not None: # timeout is in milliseconds timeout = int(timeout*1000) pollster = select.poll() if map: for fd, obj in map.items(): flags = 0 if obj.readable(): flags |= select.POLLIN | select.POLLPRI if obj.writable(): flags |= select.POLLOUT if flags: # Only check for exceptions if object was either readable # or writable. flags |= select.POLLERR | select.POLLHUP | select.POLLNVAL pollster.register(fd, flags) try: r = pollster.poll(timeout) except select.error, err: if err.args[0] != EINTR: raise r = [] for fd, flags in r: obj = map.get(fd) if obj is None: continue readwrite(obj, flags)
def run(self): """Start the heartbeat thread.""" # The first heartbeat happens immediately self.log.info('starting heartbeater') interval = 0 self.agent.set_agent_advertise_addr() self.reader, self.writer = os.pipe() p = select.poll() p.register(self.reader, select.POLLIN) try: while True: if p.poll(interval * 1000): if os.read(self.reader, 1) == 'a': break self.do_heartbeat() interval_multiplier = random.uniform( self.min_jitter_multiplier, self.max_jitter_multiplier) interval = self.agent.heartbeat_timeout * interval_multiplier log_msg = 'sleeping before next heartbeat, interval: {0}' self.log.info(log_msg.format(interval)) finally: os.close(self.reader) os.close(self.writer) self.reader = None self.writer = None
def __init__(self): self.poll = select.poll() self.pipetrick = os.pipe() self.pendingWakeup = False self.runningPoll = False self.nextHandleID = 1 self.nextTimerID = 1 self.handles = [] self.timers = [] self.quit = False # The event loop can be used from multiple threads at once. # Specifically while the main thread is sleeping in poll() # waiting for events to occur, another thread may come along # and add/update/remove a file handle, or timer. When this # happens we need to interrupt the poll() sleep in the other # thread, so that it'll see the file handle / timer changes. # # Using OS level signals for this is very unreliable and # hard to implement correctly. Thus we use the real classic # "self pipe" trick. A anonymous pipe, with one end registered # with the event loop for input events. When we need to force # the main thread out of a poll() sleep, we simple write a # single byte of data to the other end of the pipe. debug("Self pipe watch %d write %d" %(self.pipetrick[0], self.pipetrick[1])) self.poll.register(self.pipetrick[0], select.POLLIN)
def __init__(self, servername, sshcommand="ssh", username=None, ssh_options=[], debug=0, debugopts=dict(), maxopenfiles=10): self.debug = DebugMode(debug) self.debugopts = debugopts self.requests = dict() self.collectors = dict() self.queue = list() self.wantwrite = list() self.requestid_try_next = 17 self.semaphores = {'openfile': maxopenfiles} commandline = [sshcommand] if ssh_options: commandline.extend(ssh_options) # those defaults are after the user-supplied ones so they can be overridden. # (earlier ones win with ssh). commandline.extend(["-oProtocol 2", # "-oLogLevel DEBUG", "-oForwardX11 no", "-oForwardAgent no", "-oPermitLocalCommand no", "-oClearAllForwardings yes"]) if username: commandline.extend(["-l", username]) commandline.extend(["-s", "--", servername, "sftp"]) self.connection = subprocess.Popen(commandline, close_fds = True, stdin = subprocess.PIPE, stdout = subprocess.PIPE, bufsize = 0) self.poll = select.poll() self.poll.register(self.connection.stdout, select.POLLIN) self.inbuffer = bytes() self.send(INIT.bin(self, 3)) t,b = self.getpacket() if t != VERSION.id: raise SftpUnexpectedAnswerException(b, "INIT")
def ext_reduce(red_in, red_out, params): import select p = select.poll() eof = select.POLLHUP | select.POLLNVAL | select.POLLERR p.register(out_fd, select.POLLIN | eof) p.register(in_fd, select.POLLOUT | eof) MAX_NUM_OUTPUT = MAX_NUM_OUTPUT tt = 0 while True: for fd, event in p.poll(): if event & (select.POLLNVAL | select.POLLERR): raise DiscoError("Pipe to the external process failed") elif event & select.POLLIN: num = struct.unpack("I", out_fd.read(4))[0] if num > MAX_NUM_OUTPUT: raise DiscoError("External output limit "\ "exceeded: %d > %d" %\ (num, MAX_NUM_OUTPUT)) for i in range(num): red_out.add(*unpack_kv()) tt += 1 elif event & select.POLLOUT: try: msg = pack_kv(*red_in.next()) in_fd.write(msg) in_fd.flush() except StopIteration: p.unregister(in_fd) in_fd.close() else: return
def test_threaded_poll(self): r, w = os.pipe() self.addCleanup(os.close, r) self.addCleanup(os.close, w) rfds = [] for i in range(10): fd = os.dup(r) self.addCleanup(os.close, fd) rfds.append(fd) pollster = select.poll() for fd in rfds: pollster.register(fd, select.POLLIN) t = threading.Thread(target=pollster.poll) t.start() try: time.sleep(0.5) # trigger ufds array reallocation for fd in rfds: pollster.unregister(fd) pollster.register(w, select.POLLOUT) self.assertRaises(RuntimeError, pollster.poll) finally: # and make the call to poll() from the thread return os.write(w, b'spam') t.join()
def register(): ''' This function creates a select.poll object that can be used in the same manner as signal.pause(). The poll object returns each time a signal was received by the process. This function has to be called from the main thread. ''' global _signal_poller global _signal_read_fd if _signal_poller is not None: raise RuntimeError('register was already called') read_fd, write_fd = os.pipe() # Python c-level signal handler requires that the write end will be in # non blocking mode filecontrol.set_non_blocking(write_fd) # Set the read pipe end to non-blocking too, just in case. filecontrol.set_non_blocking(read_fd) # Prevent subproccesses we execute from inheriting the pipes. filecontrol.set_close_on_exec(write_fd) filecontrol.set_close_on_exec(read_fd) signal.set_wakeup_fd(write_fd) poller = select.poll() poller.register(read_fd, select.POLLIN) _signal_poller = poller _signal_read_fd = read_fd
def receive(sdef, slen=SLEN): try: sdef.setblocking(1) poller = select.poll() poller.register(sdef, READ_OR_ERROR) ready = poller.poll(LTIMEOUT*1000) if not ready: # logical timeout return "*" fd, flag = ready[0] if (flag & ( select.POLLHUP | select.POLLERR | select.POLLNVAL)): # No need to read raise RuntimeError("Socket POLLHUP") if (flag & (select.POLLIN|select.POLLPRI)): data = sdef.recv(slen) if not data: # POLLIN and POLLHUP are not exclusive. We can have both. raise RuntimeError("Socket EOF") data = int(data) # receive length elif (flag & (select.POLLERR | select.POLLHUP | select.POLLNVAL)): raise RuntimeError("Socket error {}".format(flag)) else: raise RuntimeError("Socket Unexpected Error") chunks = [] bytes_recd = 0 while bytes_recd < data: ready = poller.poll(LTIMEOUT*1000) if not ready: raise RuntimeError("Socket Timeout2") fd, flag = ready[0] if (flag & ( select.POLLHUP | select.POLLERR | select.POLLNVAL)): # No need to read raise RuntimeError("Socket POLLHUP2") if (flag & (select.POLLIN|select.POLLPRI)): chunk = sdef.recv(min(data - bytes_recd, 2048)) if not chunk: raise RuntimeError("Socket EOF2") chunks.append(chunk) bytes_recd = bytes_recd + len(chunk) elif (flag & (select.POLLERR | select.POLLHUP | select.POLLNVAL)): raise RuntimeError("Socket Error {}".format(flag)) else: raise RuntimeError("Socket Unexpected Error") poller.unregister(sdef) segments = b''.join(chunks).decode("utf-8") return json.loads(segments) except Exception as e: """ exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] print(exc_type, fname, exc_tb.tb_lineno) """ # Final cleanup try: poller.unregister(sdef) except Exception as e2: pass #print ("Exception unregistering: {}".format(e2)) raise RuntimeError("Connections: {}".format(e))
def run(self): log.debug("button thread started") # hard coded p9.12 f = '/sys/class/gpio/gpio60/value' # open & read the file fh = open(f) fh.read() fh.seek(0) poller = select.poll() poller.register(fh, select.POLLPRI | select.POLLERR) # wait until something changes and quit while True: events = poller.poll(1) if len(events): val = fh.read().strip() fh.seek(0) log.debug("%s = %s" % (f,val)) if val == "1": log.debug("button event %s" % events) break # check override try: open(override) log.debug("button override") os.unlink(override) break except IOError: pass # finish fh.close() log.debug("button thread finished")
def wait(self, timeout=None): """Wait for a notification. @type timeout: float or None @param timeout: Waiting timeout (can be None) """ self._check_owned() self._check_unnotified() self._nwaiters += 1 try: if self._poller is None: (self._read_fd, self._write_fd) = os.pipe() self._poller = select.poll() self._poller.register(self._read_fd, select.POLLHUP) wait_fn = self._waiter_class(self._poller, self._read_fd) state = self._release_save() try: # Wait for notification wait_fn(timeout) finally: # Re-acquire lock self._acquire_restore(state) finally: self._nwaiters -= 1 if self._nwaiters == 0: self._Cleanup()
def _wait_for_logcat_idle(self, seconds=1): lines = 0 # Clear logcat # os.system('{} logcat -s {} -c'.format(adb, DEVICE)); self.target.clear_logcat() # Dump logcat output logcat_cmd = 'adb -s {} logcat'.format(self.target.adb_name) logcat = Popen(logcat_cmd, shell=True, stdout=PIPE) logcat_poll = select.poll() logcat_poll.register(logcat.stdout, select.POLLIN) # Monitor logcat until it's idle for the specified number of [s] self._log.info('Waiting for system to be almost idle') self._log.info(' i.e. at least %d[s] of no logcat messages', seconds) while True: poll_result = logcat_poll.poll(seconds * 1000) if not poll_result: break lines = lines + 1 line = logcat.stdout.readline(1024) if lines % 1000: self._log.debug(' still waiting...') if lines > 1e6: self._log.warning('device logcat seems quite busy, ' 'continuing anyway... ') break
def SingleWaitForFdCondition(fdobj, event, timeout): """Waits for a condition to occur on the socket. Immediately returns at the first interruption. @type fdobj: integer or object supporting a fileno() method @param fdobj: entity to wait for events on @type event: integer @param event: ORed condition (see select module) @type timeout: float or None @param timeout: Timeout in seconds @rtype: int or None @return: None for timeout, otherwise occured conditions """ check = (event | select.POLLPRI | select.POLLNVAL | select.POLLHUP | select.POLLERR) if timeout is not None: # Poller object expects milliseconds timeout *= 1000 poller = select.poll() poller.register(fdobj, event) try: # TODO: If the main thread receives a signal and we have no timeout, we # could wait forever. This should check a global "quit" flag or something # every so often. io_events = poller.poll(timeout) except select.error, err: if err[0] != errno.EINTR: raise io_events = []
def can_poll_device(): """ Test if it's possible to use poll() on a device such as a pty. This is known to fail on Darwin. @rtype: bool @returns: True if poll() on a device succeeds, False otherwise. """ global _can_poll_device if _can_poll_device is not None: return _can_poll_device if not hasattr(select, "poll"): _can_poll_device = False return _can_poll_device try: dev_null = open('/dev/null', 'rb') except IOError: _can_poll_device = False return _can_poll_device p = select.poll() p.register(dev_null.fileno(), PollConstants.POLLIN) invalid_request = False for f, event in p.poll(): if event & PollConstants.POLLNVAL: invalid_request = True break dev_null.close() _can_poll_device = not invalid_request return _can_poll_device
def __init__(self): if 'poll' in dir(select): self.use_poll = True self.poller = select.poll() else: self.use_poll = False self.targets = {}
def _communicate(self, input): if self.stdin: # Flush stdio buffer. This might block, if the user has # been writing to .stdin in an uncontrolled fashion. self.stdin.flush() if not input: self.stdin.close() stdout = None # Return stderr = None # Return fd2file = {} fd2output = {} poller = select.poll() def register_and_append(file_obj, eventmask): poller.register(file_obj.fileno(), eventmask) fd2file[file_obj.fileno()] = file_obj def close_unregister_and_remove(fd): poller.unregister(fd) fd2file[fd].close() fd2file.pop(fd) if self.stdin and input: register_and_append(self.stdin, select.POLLOUT) select_POLLIN_POLLPRI = select.POLLIN | select.POLLPRI if self.stdout: register_and_append(self.stdout, select_POLLIN_POLLPRI) fd2output[self.stdout.fileno()] = stdout = [] if self.stderr: register_and_append(self.stderr, select_POLLIN_POLLPRI) fd2output[self.stderr.fileno()] = stderr = [] input_offset = 0 while fd2file: try: ready = poller.poll() except select.error, e: if e.args[0] == errno.EINTR: continue raise for fd, mode in ready: if mode & select.POLLOUT: chunk = input[input_offset:input_offset + _PIPE_BUF] try: input_offset += os.write(fd, chunk) except OSError as e: if e.errno == errno.EPIPE: close_unregister_and_remove(fd) else: raise else: if input_offset >= len(input): close_unregister_and_remove(fd) elif mode & select_POLLIN_POLLPRI: data = os.read(fd, 4096) if not data: close_unregister_and_remove(fd) fd2output[fd].append(data) else: # Ignore hang up or errors. close_unregister_and_remove(fd)
def __init__(self): self.IN, self.OUT, self.EOF = select.POLLIN, select.POLLOUT, select.POLLHUP self.poller = select.poll() Handler.__init__(self)
def __init__(self, timeout=TIMEOUT, channel=channel): super(Poll, self).__init__(timeout, channel=channel) self._map = {} self._poller = poll()
def _pt_thread_linux(self, context): poll = select.poll() poll.register(context.fileno(), select.POLLIN) poll.poll(-1) wx.CallAfter(self.ExitMainLoop, True)
def safe_communicate(proc, input, outlimit=None, errlimit=None): if outlimit is None: outlimit = 10485760 if errlimit is None: errlimit = outlimit if proc.stdin: # Flush stdio buffer. This might block, if the user has # been writing to .stdin in an uncontrolled fashion. proc.stdin.flush() if not input: proc.stdin.close() stdout = None # Return stderr = None # Return fd2file = {} fd2output = {} fd2length = {} fd2limit = {} poller = select.poll() def register_and_append(file_obj, eventmask): poller.register(file_obj.fileno(), eventmask) fd2file[file_obj.fileno()] = file_obj def close_unregister_and_remove(fd): poller.unregister(fd) fd2file[fd].close() fd2file.pop(fd) if proc.stdin and input: register_and_append(proc.stdin, select.POLLOUT) select_POLLIN_POLLPRI = select.POLLIN | select.POLLPRI if proc.stdout: register_and_append(proc.stdout, select_POLLIN_POLLPRI) fd2output[proc.stdout.fileno()] = stdout = [] fd2length[proc.stdout.fileno()] = 0 fd2limit[proc.stdout.fileno()] = outlimit if proc.stderr: register_and_append(proc.stderr, select_POLLIN_POLLPRI) fd2output[proc.stderr.fileno()] = stderr = [] fd2length[proc.stderr.fileno()] = 0 fd2limit[proc.stderr.fileno()] = errlimit input_offset = 0 while fd2file: try: ready = poller.poll() except select.error as e: if e.args[0] == errno.EINTR: continue raise for fd, mode in ready: if mode & select.POLLOUT: chunk = input[input_offset:input_offset + _PIPE_BUF] try: input_offset += os.write(fd, chunk) except OSError as e: if e.errno == errno.EPIPE: close_unregister_and_remove(fd) else: raise else: if input_offset >= len(input): close_unregister_and_remove(fd) elif mode & select_POLLIN_POLLPRI: data = os.read(fd, 4096) if not data: close_unregister_and_remove(fd) fd2output[fd].append(data) fd2length[fd] += len(data) if fd2length[fd] > fd2limit[fd]: if stdout is not None: stdout = b''.join(stdout) if stderr is not None: stderr = b''.join(stderr) raise OutputLimitExceeded([ 'stderr', 'stdout' ][proc.stdout is not None and proc.stdout.fileno() == fd], stdout, stderr) else: # Ignore hang up or errors. close_unregister_and_remove(fd) # All data exchanged. Translate lists into strings. if stdout is not None: stdout = b''.join(stdout) if stderr is not None: stderr = b''.join(stderr) proc.wait() return stdout, stderr
def step_impl(context, timeout): """Call scenarios.py to run a test scenario referenced by the scenario identifier""" # Run the scenario (implemented in scenarios.py) burpprocess = start_burp(context) timeout = int(timeout) scan_start_time = time.time() # Note the scan start time run_scenario(context.scenario_id, context.burp_proxy_address, burpprocess) # Wait for end of scan or timeout re_abandoned = re.compile( "^abandoned") # Regex to match abandoned scan statuses re_finished = re.compile( "^(abandoned|finished)") # Regex to match finished scans proxydict = { 'http': 'http://' + context.burp_proxy_address, 'https': 'https://' + context.burp_proxy_address } while True: # Loop until timeout or all scan tasks finished # Get scan item status list try: requests.get("http://localhost:1111", proxies=proxydict, timeout=1) except requests.exceptions.ConnectionError as error: kill_subprocess(burpprocess) assert False, "Could not communicate with headless-scanner-driver over %s (%s)" % ( context.burp_proxy_address, error.reason) # Burp extensions' stdout buffers will fill with a lot of results, and # it hangs, so we time out here and just proceed with reading the output. except requests.Timeout: pass proxy_message = read_next_json(burpprocess) # Go through scan item statuses statuses if proxy_message is None: # Extension did not respond kill_subprocess(burpprocess) assert False, "Timed out retrieving scan status information from " \ "Burp Suite over %s" % context.burp_proxy_address finished = True if proxy_message == []: # No scan items were started by extension kill_subprocess(burpprocess) assert False, "No scan items were started by Burp. Check web test case and suite scope." for status in proxy_message: if not re_finished.match(status): finished = False # In some test setups, abandoned scans are failures, and this has been set if hasattr(context, 'fail_on_abandoned_scans'): if re_abandoned.match(status): kill_subprocess(burpprocess) assert False, "Burp Suite reports an abandoned scan, " \ "but you wanted all scans to succeed. DNS " \ "problem or non-Target Scope hosts " \ "targeted in a test scenario?" if finished is True: # All scan statuses were in state "finished" break if (time.time() - scan_start_time) > (timeout * 60): kill_subprocess(burpprocess) assert False, "Scans did not finish in %s minutes, timed out. Scan statuses were: %s" % ( timeout, proxy_message) time.sleep(10) # Poll again in 10 seconds # Retrieve scan results and request clean exit try: requests.get("http://localhost:1112", proxies=proxydict, timeout=1) except requests.exceptions.ConnectionError as error: kill_subprocess(burpprocess) assert False, "Could not communicate with headless-scanner-driver over %s (%s)" % ( context.burp_proxy_address, error.reason) # Burp extensions' stdout buffers will fill with a lot of results, and # it hangs, so we time out here and just proceed with reading the output. except requests.Timeout: pass proxy_message = read_next_json(burpprocess) if proxy_message is None: kill_subprocess(burpprocess) assert False, "Timed out retrieving scan results from Burp Suite over %s" % context.burp_proxy_address context.results = proxy_message # Store results for baseline delta checking # Wait for Burp to exit poll = select.poll() poll.register(burpprocess.stdout, select.POLLNVAL | select.POLLHUP) # pylint: disable-msg=E1101 descriptors = poll.poll(10000) if descriptors == []: kill_subprocess(burpprocess) assert False, "Burp Suite clean exit took more than 10 seconds, killed" assert True
def run(self): global timeout, w, tuples, regexes, json_pending, last_push, config fp = {} if osname == "linux": w = watcher.AutoWatcher() for path in config.get('Analyzer', 'paths').split(","): try: print("Recursively monitoring " + path.strip() + "...") w.add_all(path.strip(), inotify.IN_ALL_EVENTS) except OSError as err: pass if not w.num_watches(): print("No paths to analyze, nothing to do!") sys.exit(1) poll = select.poll() poll.register(w, select.POLLIN) timeout = None threshold = watcher.Threshold(w, 256) inodes = {} inodes_path = {} xes = connect_es(config) while True: events = poll.poll(timeout) nread = 0 if threshold() or not events: #print('reading,', threshold.readable(), 'bytes available') for evt in w.read(0): nread += 1 # The last thing to do to improve efficiency here would be # to coalesce similar events before passing them up to a # higher level. # For example, it's overwhelmingly common to have a stream # of inotify events contain a creation, followed by # multiple modifications of the created file. # Recognising this pattern (and others) and coalescing # these events into a single creation event would reduce # the number of trips into our app's presumably more # computationally expensive upper layers. masks = inotify.decode_mask(evt.mask) #print(masks) path = evt.fullpath #print(repr(evt.fullpath), ' | '.join(masks)) try: if not u'IN_ISDIR' in masks: if (u'IN_MOVED_FROM' in masks) and (path in filehandles): print( "File moved, closing original handle") try: filehandles[path].close() except Exception as err: print(err) del filehandles[path] inode = inodes_path[path] del inodes[inode] elif (not u'IN_DELETE' in masks) and ( not path in filehandles) and ( path.find(".gz") == -1): try: print("Opening " + path) idata = os.stat(path) inode = idata.st_ino if not inode in inodes: filehandles[path] = open(path, "r") print("Started watching " + path) filehandles[path].seek(0, 2) inodes[inode] = path inodes_path[path] = inode except Exception as err: print(err) try: filehandles[path].close() except Exception as err: print(err) del filehandles[path] inode = inodes_path[path] del inodes[inode] # First time we've discovered this file? if u'IN_CLOSE_NOWRITE' in masks and not path in filehandles: pass # New file created in a folder we're watching?? elif u'IN_CREATE' in masks: pass # File truncated? elif u'IN_CLOSE_WRITE' in masks and path in filehandles: # print(path + " truncated!") filehandles[path].seek(0, 2) # File contents modified? elif u'IN_MODIFY' in masks and path in filehandles: # print(path + " was modified") rd = 0 data = "" #print("Change in " + path) try: while True: line = filehandles[path].readline() if not line: #filehandles[path].seek(0,2) break else: rd += len(line) data += line #print("Read %u bytes from %s" % (rd, path)) parseLine(path, data) except Exception as err: try: print("Could not utilize " + path + ", closing.." + err) filehandles[path].close() except Exception as err: print(err) del filehandles[path] inode = inodes_path[path] del inodes[inode] # File deleted? (close handle) elif u'IN_DELETE' in masks: if path in filehandles: print("Closed " + path) try: filehandles[path].close() except Exception as err: print(err) del filehandles[path] inode = inodes_path[path] del inodes[inode] print("Stopped watching " + path) else: pass except Exception as err: print(err) for x in json_pending: if (time.time() > (last_push[x] + 15)) or len(json_pending[x]) >= 500: if not x in fp: fp[x] = True print("First push for " + x + "!") t = NodeThread() t.assign(json_pending[x], x, xes) t.start() json_pending[x] = [] last_push[x] = time.time() if nread: #print('plugging back in') timeout = None poll.register(w, select.POLLIN) else: #print('unplugging,', threshold.readable(), 'bytes available') timeout = 1000 poll.unregister(w) if osname == "freebsd": xes = connect_es(config) observer = Observer() for path in paths: observer.schedule(BSDHandler(), path, recursive=True) syslog.syslog(syslog.LOG_INFO, "Recursively monitoring " + path.strip() + "...") observer.start() try: while True: for x in json_pending: if not x in last_push: last_push[x] = time.time() if len(json_pending[x]) > 0 and ( (time.time() > (last_push[x] + 15)) or len(json_pending[x]) >= 500): if not x in fp: fp[x] = True syslog.syslog(syslog.LOG_INFO, "First push for " + x + "!") t = NodeThread() t.assign(json_pending[x], x, xes) t.start() json_pending[x] = [] last_push[x] = time.time() time.sleep(0.5) except KeyboardInterrupt: observer.stop() observer.join()
def wait(conn, criteria): poll = select.poll() poll.register(conn.fileno(), select.POLLIN) while not criteria(): poll.poll() conn.process()
dev.set_led(3, dev.get_led(4)) dev.set_led(4, dev.get_led(4) == False) print("capacity:", dev.get_battery(), "%") print("devtype:", dev.get_devtype()) print("extension:", dev.get_extension()) except SystemError as e: print("ooops", e) exit(1) dev.set_mp_normalization(10, 20, 30, 40) x, y, z, factor = dev.get_mp_normalization() print("mp", x, y, z, factor) # read some values p = poll() p.register(fd, POLLIN) evt = xwiimote.event() n = 0 while n < 2: p.poll() try: dev.dispatch(evt) if evt.type == xwiimote.EVENT_KEY: code, state = evt.get_key() print("Key:", code, ", State:", state) n += 1 elif evt.type == xwiimote.EVENT_GONE: print("Gone") n = 2 elif evt.type == xwiimote.EVENT_WATCH:
def run(self): ''' Look at our incoming queue, and receiv it ''' self.fd = self.connection.fileno( ) # file descriptor for this connection self.pollme = select.poll() # register the fd # self.pollme.register(self.fd, self.pollAll) self.pollme.register(self.fd, self.pollRecv) debug("ready to receive data") while True: data = None try: if self.endNow.isSet() or self.onExit.isSet(): debug("receive thread received halt event") break # block and wait for data or error condition pollOutput = self.pollme.poll(None) # reply is list of tuples with fd and reason for ourfd, reason in pollOutput: # if data arrives if reason & self.pollRecv: debug("recv reason: " + str(reason) + ", attempting to read") data = self.connection.recv(RECV_BUFFER_SIZE) if log.logFlag: debug("received " + str(len(data)) + " bytes on socket") if len(data): debug("string received was: " + str(data[0:32])) # at this point we should have data # place it on queue if len(data): self.recvq.put(data) else: # zero-length receive means socket is irrevocably closed for reading debug("received zero bytes, this socket closed") self.onExit.set() break # Check to see if there were other codes as well if reason & self.mask: # returned because of other errors if reason & select.POLLHUP: debug("recv halt due to hangup") self.onExit.set() break elif reason & select.POLLERR: debug("recv halt due to error") self.onExit.set() break elif reason & select.POLLNVAL: debug("recv halt due to invalid file descriptor") self.onExit.set() break else: # ya never know... debug("recv halt due to unknown code " + str(reason)) self.onExit.set() break if self.onExit.isSet(): break # out of while loop except Exception, inst: self.onExit.set() # send Event() that we are finished debug("receive thread over because " + str(inst)) break
def _RunCmdPipe(cmd, env, via_shell, cwd, interactive, timeout, noclose_fds, input_fd, postfork_fn=None, _linger_timeout=constants.CHILD_LINGER_TIMEOUT): """Run a command and return its output. @type cmd: string or list @param cmd: Command to run @type env: dict @param env: The environment to use @type via_shell: bool @param via_shell: if we should run via the shell @type cwd: string @param cwd: the working directory for the program @type interactive: boolean @param interactive: Run command interactive (without piping) @type timeout: int @param timeout: Timeout after the programm gets terminated @type noclose_fds: list @param noclose_fds: list of additional (fd >=3) file descriptors to leave open for the child process @type input_fd: C{file}-like object or numeric file descriptor @param input_fd: File descriptor for process' standard input @type postfork_fn: Callable receiving PID as parameter @param postfork_fn: Function run after fork but before timeout @rtype: tuple @return: (out, err, status) """ # pylint: disable=R0101 poller = select.poll() if interactive: stderr = None stdout = None else: stderr = subprocess.PIPE stdout = subprocess.PIPE if input_fd: stdin = input_fd elif interactive: stdin = None else: stdin = subprocess.PIPE if noclose_fds: preexec_fn = lambda: CloseFDs(noclose_fds) close_fds = False else: preexec_fn = None close_fds = True child = subprocess.Popen(cmd, shell=via_shell, stderr=stderr, stdout=stdout, stdin=stdin, close_fds=close_fds, env=env, cwd=cwd, preexec_fn=preexec_fn) if postfork_fn: postfork_fn(child.pid) out = StringIO() err = StringIO() linger_timeout = None if timeout is None: poll_timeout = None else: poll_timeout = utils_algo.RunningTimeout(timeout, True).Remaining msg_timeout = ("Command %s (%d) run into execution timeout, terminating" % (cmd, child.pid)) msg_linger = ("Command %s (%d) run into linger timeout, killing" % (cmd, child.pid)) timeout_action = _TIMEOUT_NONE # subprocess: "If the stdin argument is PIPE, this attribute is a file object # that provides input to the child process. Otherwise, it is None." assert (stdin == subprocess.PIPE) ^ (child.stdin is None), \ "subprocess' stdin did not behave as documented" if not interactive: if child.stdin is not None: child.stdin.close() poller.register(child.stdout, select.POLLIN) poller.register(child.stderr, select.POLLIN) fdmap = { child.stdout.fileno(): (out, child.stdout), child.stderr.fileno(): (err, child.stderr), } for fd in fdmap: utils_wrapper.SetNonblockFlag(fd, True) while fdmap: if poll_timeout: pt = poll_timeout() * 1000 if pt < 0: if linger_timeout is None: logging.warning(msg_timeout) if child.poll() is None: timeout_action = _TIMEOUT_TERM utils_wrapper.IgnoreProcessNotFound(os.kill, child.pid, signal.SIGTERM) linger_timeout = \ utils_algo.RunningTimeout(_linger_timeout, True).Remaining pt = linger_timeout() * 1000 if pt < 0: break else: pt = None pollresult = utils_wrapper.RetryOnSignal(poller.poll, pt) for fd, event in pollresult: if event & select.POLLIN or event & select.POLLPRI: data = fdmap[fd][1].read() # no data from read signifies EOF (the same as POLLHUP) if not data: poller.unregister(fd) del fdmap[fd] continue fdmap[fd][0].write(data) if (event & select.POLLNVAL or event & select.POLLHUP or event & select.POLLERR): poller.unregister(fd) del fdmap[fd] if timeout is not None: assert callable(poll_timeout) # We have no I/O left but it might still run if child.poll() is None: _WaitForProcess(child, poll_timeout()) # Terminate if still alive after timeout if child.poll() is None: if linger_timeout is None: logging.warning(msg_timeout) timeout_action = _TIMEOUT_TERM utils_wrapper.IgnoreProcessNotFound(os.kill, child.pid, signal.SIGTERM) lt = _linger_timeout else: lt = linger_timeout() _WaitForProcess(child, lt) # Okay, still alive after timeout and linger timeout? Kill it! if child.poll() is None: timeout_action = _TIMEOUT_KILL logging.warning(msg_linger) utils_wrapper.IgnoreProcessNotFound(os.kill, child.pid, signal.SIGKILL) out = out.getvalue() err = err.getvalue() status = child.wait() return out, err, status, timeout_action
def asynch_copy (src, dst): size = src.get_size () # This is our reading position in the source. soff = 0 # This callback is called when any pread from the source # has completed. writes = [] def read_completed (buf, offset, error): global bytes_read bytes_read += buf.size () wr = (buf, offset) writes.append (wr) # By returning 1 here we auto-retire the pread command. return 1 # This callback is called when any pwrite to the destination # has completed. def write_completed (buf, error): global bytes_written bytes_written += buf.size () # By returning 1 here we auto-retire the pwrite command. return 1 # The main loop which runs until we have finished reading and # there are no more commands in flight. while soff < size or dst.aio_in_flight () > 0: # If we're able to submit more reads from the source # then do so now. if soff < size and src.aio_in_flight () < max_reads_in_flight: bufsize = min (bs, size - soff) buf = nbd.Buffer (bufsize) # NB: Python lambdas are BROKEN. # https://stackoverflow.com/questions/2295290 src.aio_pread (buf, soff, lambda err, buf=buf, soff=soff: read_completed (buf, soff, err)) soff += bufsize # If there are any write commands waiting to be issued # to the destination, send them now. for buf, offset in writes: # See above link about broken Python lambdas. dst.aio_pwrite (buf, offset, lambda err, buf=buf: write_completed (buf, err)) writes = [] poll = select.poll () sfd = src.aio_get_fd () dfd = dst.aio_get_fd () sevents = 0 devents = 0 if src.aio_get_direction () & nbd.AIO_DIRECTION_READ: sevents += select.POLLIN if src.aio_get_direction () & nbd.AIO_DIRECTION_WRITE: sevents += select.POLLOUT if dst.aio_get_direction () & nbd.AIO_DIRECTION_READ: devents += select.POLLIN if dst.aio_get_direction () & nbd.AIO_DIRECTION_WRITE: devents += select.POLLOUT poll.register (sfd, sevents) poll.register (dfd, devents) for (fd, revents) in poll.poll (): # The direction of each handle can change since we # slept in the select. if fd == sfd and revents & select.POLLIN and \ src.aio_get_direction () & nbd.AIO_DIRECTION_READ: src.aio_notify_read () elif fd == sfd and revents & select.POLLOUT and \ src.aio_get_direction () & nbd.AIO_DIRECTION_WRITE: src.aio_notify_write () elif fd == dfd and revents & select.POLLIN and \ dst.aio_get_direction () & nbd.AIO_DIRECTION_READ: dst.aio_notify_read () elif fd == dfd and revents & select.POLLOUT and \ dst.aio_get_direction () & nbd.AIO_DIRECTION_WRITE: dst.aio_notify_write ()
#!/usr/bin/python import socket import select server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) server_socket.bind(('', 2007)) server_socket.listen(5) # server_socket.setblocking(0) poll = select.poll() # epoll() should work the same poll.register(server_socket.fileno(), select.POLLIN) connections = {} while True: events = poll.poll(10000) # 10 seconds for fileno, event in events: if fileno == server_socket.fileno(): (client_socket, client_address) = server_socket.accept() print "got connection from", client_address # client_socket.setblocking(0) poll.register(client_socket.fileno(), select.POLLIN) connections[client_socket.fileno()] = client_socket elif event & select.POLLIN: client_socket = connections[fileno] data = client_socket.recv(4096) if data: client_socket.send(data) # sendall() partial? else: poll.unregister(fileno) client_socket.close()
import socket, select s = socket.socket() host = socket.gethostname() port = 1234 s.bind((host, port)) fdmap = {s.fileno(): s} s.listen(5) p = select.poll() p.register(s) while True: events = p.poll() for fd, event in events: if fdmap[fd] is s: c, addr = s.accept() print 'Got connection from', addr p.register(c) fdmap[c.fileno()] = c elif event & select.POLLIN: data = fdmap[fd].recv(1024) if not data: # No data -- connection closed print fdmap[fd].getpeername(), 'disconnected' p.unregister(fd) del fdmap[fd] else: print data
def main() -> None: parser = argparse.ArgumentParser() parser.add_argument('-p', '--port', type=int, action='store', dest='port', default=9007, required=False, help='set port number') parser.add_argument('-i', '--ip', type=str, action='store', dest='ip', default='127.0.0.1', required=False, help='set ip address') parser.add_argument('-t', '--tokenfile', type=str, action='store', dest='tokenfile', default=expanduser('~') + '/.localslackirc', required=False, help='set the token file') parser.add_argument('-u', '--nouserlist', action='store_true', dest='nouserlist', required=False, help='don\'t display userlist') parser.add_argument('-j', '--autojoin', action='store_true', dest='autojoin', required=False, help="Automatically join all remote channels") parser.add_argument( '-o', '--override', action='store_true', dest='overridelocalip', required=False, help='allow non 127. addresses, this is potentially dangerous') parser.add_argument( '--rc-url', type=str, action='store', dest='rc_url', default=None, required=False, help= 'The rocketchat URL. Setting this changes the mode from slack to rocketchat' ) args = parser.parse_args() # Exit if their chosden ip isn't local. User can override with -o if they so dare if not args.ip.startswith('127') and not args.overridelocalip: exit('supplied ip isn\'t local\nlocalslackirc has no encryption or ' \ 'authentication, it\'s recommended to only allow local connections\n' \ 'you can override this with -o') if 'PORT' in environ: port = int(environ['PORT']) else: port = args.port if 'TOKEN' in environ: token = environ['TOKEN'] else: try: with open(args.tokenfile) as f: token = f.readline().strip() except (FileNotFoundError, PermissionError): exit(f'Unable to open the token file {args.tokenfile}') if args.rc_url: sl_client = rocket.Rocket( args.rc_url, token) # type: Union[slack.Slack, rocket.Rocket] else: sl_client = slack.Slack(token) sl_events = sl_client.events_iter() serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) serversocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) serversocket.bind((args.ip, port)) serversocket.listen(1) poller = select.poll() while True: s, _ = serversocket.accept() ircclient = Client(s, sl_client, nouserlist=args.nouserlist, autojoin=args.autojoin) poller.register(s.fileno(), select.POLLIN) if sl_client.fileno is not None: poller.register(sl_client.fileno, select.POLLIN) # Main loop timeout = 2 while True: s_event = poller.poll(timeout) # type: List[Tuple[int,int]] sl_event = next(sl_events) if s_event: text = s.recv(1024) if len(text) == 0: break #FIXME handle the case when there is more to be read for i in text.split(b'\n')[:-1]: i = i.strip() if i: ircclient.command(i) while sl_event: print(sl_event) ircclient.slack_event(sl_event) sl_event = next(sl_events)
def _expect_with_poll(self, expect_list, timeout=None): """Read until one from a list of a regular expressions matches. This method uses select.poll() to implement the timeout. """ re = None expect_list = expect_list[:] indices = range(len(expect_list)) for i in indices: if not hasattr(expect_list[i], "search"): if not re: import re expect_list[i] = re.compile(expect_list[i]) call_timeout = timeout if timeout is not None: from time import time time_start = time() self.process_rawq() m = None for i in indices: m = expect_list[i].search(self.cookedq) if m: e = m.end() text = self.cookedq[:e] self.cookedq = self.cookedq[e:] break if not m: poller = select.poll() poll_in_or_priority_flags = select.POLLIN | select.POLLPRI poller.register(self, poll_in_or_priority_flags) while not m and not self.eof: try: ready = poller.poll(call_timeout) except OSError as e: if e.errno == errno.EINTR: if timeout is not None: elapsed = time() - time_start call_timeout = timeout-elapsed continue raise for fd, mode in ready: if mode & poll_in_or_priority_flags: self.fill_rawq() self.process_rawq() for i in indices: m = expect_list[i].search(self.cookedq) if m: e = m.end() text = self.cookedq[:e] self.cookedq = self.cookedq[e:] break if timeout is not None: elapsed = time() - time_start if elapsed >= timeout: break call_timeout = timeout-elapsed poller.unregister(self) if m: return (i, m, text) text = self.read_very_lazy() if not text and self.eof: raise EOFError return (-1, None, text)
def handle_process_output(process, stdout_handler, stderr_handler, finalizer): """Registers for notifications to lean that process output is ready to read, and dispatches lines to the respective line handlers. We are able to handle carriage returns in case progress is sent by that mean. For performance reasons, we only apply this to stderr. This function returns once the finalizer returns :return: result of finalizer :param process: subprocess.Popen instance :param stdout_handler: f(stdout_line_string), or None :param stderr_hanlder: f(stderr_line_string), or None :param finalizer: f(proc) - wait for proc to finish""" fdmap = { process.stdout.fileno(): (stdout_handler, [b'']), process.stderr.fileno(): (stderr_handler, [b'']) } def _parse_lines_from_buffer(buf): line = b'' bi = 0 lb = len(buf) while bi < lb: char = _bchr(buf[bi]) bi += 1 if char in (b'\r', b'\n') and line: yield bi, line line = b'' else: line += char # END process parsed line # END while file is not done reading # end def _read_lines_from_fno(fno, last_buf_list): buf = os.read(fno, mmap.PAGESIZE) buf = last_buf_list[0] + buf bi = 0 for bi, line in _parse_lines_from_buffer(buf): yield line # for each line to parse from the buffer # keep remainder last_buf_list[0] = buf[bi:] def _dispatch_single_line(line, handler): line = line.decode(defenc) if line and handler: handler(line) # end dispatch helper # end single line helper def _dispatch_lines(fno, handler, buf_list): lc = 0 for line in _read_lines_from_fno(fno, buf_list): _dispatch_single_line(line, handler) lc += 1 # for each line return lc # end def _deplete_buffer(fno, handler, buf_list, wg=None): lc = 0 while True: line_count = _dispatch_lines(fno, handler, buf_list) lc += line_count if line_count == 0: break # end deplete buffer if buf_list[0]: _dispatch_single_line(buf_list[0], handler) lc += 1 # end if wg: wg.done() return lc # end if hasattr(select, 'poll'): # poll is preferred, as select is limited to file handles up to 1024 ... . This could otherwise be # an issue for us, as it matters how many handles our own process has poll = select.poll() READ_ONLY = select.POLLIN | select.POLLPRI | select.POLLHUP | select.POLLERR CLOSED = select.POLLHUP | select.POLLERR poll.register(process.stdout, READ_ONLY) poll.register(process.stderr, READ_ONLY) closed_streams = set() while True: # no timeout try: poll_result = poll.poll() except select.error as e: if e.args[0] == errno.EINTR: continue raise # end handle poll exception for fd, result in poll_result: if result & CLOSED: closed_streams.add(fd) else: _dispatch_lines(fd, *fdmap[fd]) # end handle closed stream # end for each poll-result tuple if len(closed_streams) == len(fdmap): break # end its all done # end endless loop # Depelete all remaining buffers for fno, (handler, buf_list) in fdmap.items(): _deplete_buffer(fno, handler, buf_list) # end for each file handle for fno in fdmap.keys(): poll.unregister(fno) # end don't forget to unregister ! else: # Oh ... probably we are on windows. select.select() can only handle sockets, we have files # The only reliable way to do this now is to use threads and wait for both to finish # Since the finalizer is expected to wait, we don't have to introduce our own wait primitive # NO: It's not enough unfortunately, and we will have to sync the threads wg = WaitGroup() for fno, (handler, buf_list) in fdmap.items(): wg.add(1) t = threading.Thread( target=lambda: _deplete_buffer(fno, handler, buf_list, wg)) t.start() # end # NOTE: Just joining threads can possibly fail as there is a gap between .start() and when it's # actually started, which could make the wait() call to just return because the thread is not yet # active wg.wait() # end return finalizer(process)
def server(): not_len = 0 try: import socket, select serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) serversocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) serversocket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) keepalive = serversocket.getsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE) print "socket.SO_KEEPALIVE", keepalive serversocket.bind((HOST, PORT)) serversocket.listen(1) serversocket.setblocking(0) spoll = select.poll() spoll.register(serversocket.fileno(), select.POLLIN) connections = {}; requests = {}; responses = {}; times = {} while True: ffno = 0 events = spoll.poll(1) for fileno, event in events: intime = int(time.time()) if fileno == serversocket.fileno(): connection, address = serversocket.accept() print "Connect:", connection.fileno(), address connection.setblocking(0) spoll.register(connection.fileno(), select.POLLIN) connections[connection.fileno()] = connection requests[connection.fileno()] = '' elif event & select.POLLIN: try: requests[fileno] = connections[fileno].recv(1024) if len(requests[fileno]): agis_data_capture (fileno, requests[fileno]) times[fileno] = intime not_len = 0 else: not_len += 1 print "Close", fileno, not_len ffno = fileno connections[fileno].close() spoll.unregister(fileno) except socket.error: print "socket:", fileno, "error:" pexcept ("error") connections[fileno].close() spoll.unregister(fileno) elif event & select.POLLOUT: print "send", responses[fileno] byteswritten = connections[fileno].send(responses[fileno]) responses[fileno] = responses[fileno][byteswritten:] if len(responses[fileno]) == 0: spoll.modify(fileno, 0) connections[fileno].shutdown(socket.SHUT_RDWR) elif event & select.POLLHUP: ffno = fileno print "\tevent & select.POLLHUP" if not_len > 100: print time.strftime("\tBreack %Y-%m-%d %T", time.localtime(intime)) break time.sleep(.01) if not keepalive and not ffno: for fn in times: if intime - times[fn] > 900: #1800: print "POLL.unregister", fn ffno = fn break if ffno and times.has_key(ffno): print "\tffno", ffno, time.strftime("%Y-%m-%d %T", time.localtime(intime)) del times[ffno] try: connections[ffno].close() spoll.unregister(ffno) del connections[ffno] except KeyError: print "\t<exceptions.KeyError>" except socket.error: pexcept ('server') except: pexcept ('server') finally: serversocket.close()
def __init__(self): self.timeout = None self.poll_object = select.poll()
def _remote_call_int(self): rsize = pow(2, 20) sockets = [ self.process.stderr.fileno(), self.process.stdout.fileno(), self.pipe_read_ctos ] # l.debug("polling sockets: " + repr(sockets)) pp = select.poll() for ss in sockets: pp.register(ss) state = 1 to_recv = 1 + 8 buf = b'' ttype = "n" try: while True: fds_tuples = pp.poll() self.process.poll() # set the return code ret_code = self.process.returncode # l.debug("poll results: " + repr(fds_tuples) + ", " + repr(ret_code)) if len(fds_tuples) == 2 and \ (self.process.stderr.fileno(), select.EPOLLHUP) in fds_tuples and \ (self.process.stdout.fileno(), select.EPOLLHUP) in fds_tuples: # select.EPOLLHUP --> 16, select.EPOLLIN -> 1 # stderr and stdout have been closed and nothing else is available if ret_code is not None: estr = "JYTHON DIED %d\n%s" % (ret_code, self.get_client_std()) raise JythonClientException(estr) else: estr = "JYTHON SOCKET CLOSED\n%s" % ( self.get_client_std()) self.process.kill() self.process.poll() raise JythonClientException(estr) fds = [ f[0] for f in fds_tuples if (f[1] & select.EPOLLIN) != 0 ] # time.sleep(0.1) if self.process.stderr.fileno() in fds: tstr = os.read(self.process.stderr.fileno(), rsize) self.client_stderr += str(tstr) if self.process.stdout.fileno() in fds: tstr = os.read(self.process.stdout.fileno(), rsize) self.client_stdout += str(tstr) if self.pipe_read_ctos in fds: # os.read wants as readn at most a signed int readn = min(to_recv, pow(2, 30)) tstr = os.read(self.pipe_read_ctos, readn) # l.debug("reading from Jython: %d %d %d %d" % (to_recv, readn, len(tstr), len(buf))) to_recv -= len(tstr) buf += tstr if to_recv == 0: if state == 1: to_recv = struct.unpack("<Q", buf[1:])[0] ttype = buf[0] buf = b"" state = 2 elif state == 2: return ttype, buf except KeyboardInterrupt: estr = "JYTHON CLIENT INTERRUPTED\n%s" % (self.get_client_std()) self.process.kill() self.process.poll() raise KeyboardInterrupt(estr) finally: # this seems to restore the shell to a usable state subprocess.Popen(["stty", "sane"]).communicate()
def __init__(self): self._fd_to_pconn = dict() self._pollset = select.poll()
def __init__(self, mastersock): self.p = select.poll() self.mastersock = mastersock self.watchread(mastersock) self.buffers = {} self.sockets = {mastersock.fileno(): mastersock}
def startShell(self, mnopts=None): """Start a shell process for running commands.""" if self.shell: error('shell is already running') return assert mnopts is None, 'mnopts not supported for DockerHost' self.container = '%s-%s' % (self.prefix, self.name) debug('Starting container %s with image "%s".' % (self.container, self.image)) self.kill(purge=True) container_tmp_dir = os.path.join(os.path.abspath(self.tmpdir), 'tmp') tmp_volume = container_tmp_dir + ':/tmp' base_cmd = [ "docker", "run", "-ti", "--privileged", "--entrypoint", "env", "-h", self.name, "--name", self.container ] opt_args = ['--net=%s' % self.network] env_args = reduce(operator.add, (['--env', var] for var in self.env_vars), []) vol_args = reduce(operator.add, (['-v', var] for var in self.vol_maps), ['-v', tmp_volume]) image_args = [ self.image, "TERM=dumb", "PS1=" + chr(127), "bash", "--norc", "-is", "mininet:" + self.name ] cmd = base_cmd + opt_args + env_args + vol_args + image_args self.master, self.slave = pty.openpty() debug('docker command "%s", fd %d, fd %d' % (' '.join(cmd), self.master, self.slave)) try: self.shell = self._popen(cmd, stdin=self.slave, stdout=self.slave, stderr=self.slave) self.stdin = os.fdopen(self.master, 'r') self.stdout = self.stdin self.pollOut = select.poll() # pylint: disable=invalid-name self.pollOut.register(self.stdout) # pylint: disable=no-member self.outToNode[self.stdout.fileno()] = self # pylint: disable=no-member self.pollIn = select.poll() # pylint: disable=invalid-name self.pollIn.register(self.stdout, select.POLLIN) # pylint: disable=no-member self.inToNode[self.stdin.fileno()] = self # pylint: disable=no-member self.execed = False self.lastCmd = None # pylint: disable=invalid-name self.lastPid = None # pylint: disable=invalid-name self.readbuf = '' self.waiting = True data = '' while True: data = self.read() if data[-1] == chr(127): break self.readbuf = '' self.waiting = False except: if self.shell: self.shell.poll() raise self.pid = self.inspect_pid() debug("Container %s created pid %s/%s." % (self.container, self.pid, self.shell.pid)) self.cmd('unset HISTFILE; stty -echo; set +m') # pylint: disable=no-member
def vm_start(self, arch='amd64', distro='buster'): build_dir = self.params.get('build_dir', default='.') time_to_wait = self.params.get('time_to_wait', default=60) self.log.info('===================================================') self.log.info('Running Isar VM boot test for (' + distro + '-' + arch + ')') self.log.info('Isar build folder is: ' + build_dir) self.log.info('===================================================') self.check_bitbake(build_dir) fd, output_file = tempfile.mkstemp(suffix='_log.txt', prefix='vm_start_' + distro + '_' + arch + '_', dir=build_dir, text=True) os.chmod(output_file, 0o644) cmdline = start_vm.format_qemu_cmdline(arch, build_dir, distro, output_file, None) cmdline.insert(1, '-nographic') self.log.info('QEMU boot line: ' + str(cmdline)) login_prompt = b'isar login:'******'Just an example' timeout = time.time() + int(time_to_wait) p1 = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.PIPE) try: poller = select.poll() poller.register(p1.stdout, select.POLLIN) poller.register(p1.stderr, select.POLLIN) while time.time() < timeout and p1.poll() is None: events = poller.poll(1000 * (timeout - time.time())) for fd, event in events: if fd == p1.stdout.fileno(): # Wait for the complete string if it is read in chunks # like "i", "sar", " login:"******"rb") as f1: data = f1.read() if service_prompt in data and login_prompt in data: return else: self.log.error(data) self.fail('Log ' + output_file)
def poll_cmd_execute(client, timeout=8): if has_poll(): p = select.poll() event_in_mask = select.POLLIN | select.POLLPRI event_err_mask = select.POLLERR event_closed_mask = select.POLLHUP | select.POLLNVAL event_mask = event_in_mask | event_err_mask | event_closed_mask p.register(client.conn, event_mask) count = 0 ret = '' while True: events = p.poll(timeout) if events: event = events[0][1] if event & select.POLLERR: ret = "Client Hung up\n" break ready = event & select.POLLPRI or event & select.POLLIN if not ready: ret = "execute command timeout\n" break else: ret += get_unicode(client.conn.recv(0x10000)) # ret += str(client.conn.recv(0x10000), "utf-8") else: if ret: break elif count > timeout: ret = "execute command timeout\n" break else: data_to_stdout(".") time.sleep(1) count += 1 p.unregister(client.conn) else: count = 0 ret = '' while True: ready = select.select([client.conn], [], [], 0.1) if ready[0]: ret += get_unicode(client.conn.recv(0x10000)) # ret += str(client.conn.recv(0x10000), "utf-8") else: if ret: break elif count > timeout: ret = "execute command timeout\n" else: data_to_stdout('.') time.sleep(1) count += 1 if ret and not ret.startswith('\r'): ret = "\r{0}".format(ret) if ret and not ret.endswith('\n'): ret = "{0}\n".format(ret) return ret
def __init__(self, python_path, max_pickle_version, config_dir): # Make sure to init these variables (used in __del__) early on in case we # have an exception self._poller = None self._server_process = None self._socket_path = None data_transferer.defaultProtocol = max_pickle_version self._config_dir = config_dir # The client launches the server when created; we use # Unix sockets for now server_module = ".".join([__package__, "server"]) self._socket_path = "/tmp/%s_%d" % (os.path.basename(config_dir), os.getpid()) if os.path.exists(self._socket_path): raise RuntimeError("Existing socket: %s" % self._socket_path) env = os.environ.copy() # env["PYTHONPATH"] = ":".join(sys.path) self._server_process = Popen( [ python_path, "-u", "-m", server_module, str(max_pickle_version), config_dir, self._socket_path, ], env=env, stdout=PIPE, stderr=PIPE, bufsize=1, universal_newlines=True, # Forces text as well ) # Read override configuration sys.path.insert(0, config_dir) override_module = importlib.import_module("overrides") sys.path = sys.path[1:] # Determine all overrides self._overrides = {} self._getattr_overrides = {} self._setattr_overrides = {} for override in override_module.__dict__.values(): if isinstance(override, (LocalOverride, LocalAttrOverride)): for obj_name, obj_funcs in override.obj_mapping.items(): if isinstance(override, LocalOverride): override_dict = self._overrides.setdefault( obj_name, {}) elif override.is_setattr: override_dict = self._setattr_overrides.setdefault( obj_name, {}) else: override_dict = self._getattr_overrides.setdefault( obj_name, {}) if isinstance(obj_funcs, str): obj_funcs = (obj_funcs, ) for name in obj_funcs: if name in override_dict: raise ValueError( "%s was already overridden for %s" % (name, obj_name)) override_dict[name] = override.func self._proxied_objects = {} # Wait for the socket to be up on the other side; we also check if the # server had issues starting up in which case we report that and crash out while not os.path.exists(self._socket_path): returncode = self._server_process.poll() if returncode is not None: raise RuntimeError( "Server did not properly start: %s" % self._server_process.stderr.read(), ) time.sleep(1) # Open up the channel and setup the datastransfer pipeline self._channel = Channel(SocketByteStream.unixconnect( self._socket_path)) self._datatransferer = DataTransferer(self) # Make PIPEs non-blocking; this is helpful to be able to # order the messages properly for f in (self._server_process.stdout, self._server_process.stderr): fl = fcntl.fcntl(f, fcntl.F_GETFL) fcntl.fcntl(f, fcntl.F_SETFL, fl | os.O_NONBLOCK) # Set up poller self._poller = select.poll() self._poller.register(self._server_process.stdout, select.POLLIN) self._poller.register(self._server_process.stderr, select.POLLIN) self._poller.register(self._channel, select.POLLIN | select.POLLHUP) # Get all exports that we are proxying response = self._communicate({ FIELD_MSGTYPE: MSG_CONTROL, FIELD_OPTYPE: CONTROL_GETEXPORTS }) self._proxied_classes = { k: None for k in itertools.chain(response[FIELD_CONTENT]["classes"], response[FIELD_CONTENT]["proxied"]) } # Proxied standalone functions are functions that are proxied # as part of other objects like defaultdict for which we create a # on-the-fly simple class that is just a callable. This is therefore # a special type of self._proxied_classes self._proxied_standalone_functions = {} self._export_info = { "classes": response[FIELD_CONTENT]["classes"], "functions": response[FIELD_CONTENT]["functions"], "values": response[FIELD_CONTENT]["values"], "exceptions": response[FIELD_CONTENT]["exceptions"], "aliases": response[FIELD_CONTENT]["aliases"], } self._aliases = response[FIELD_CONTENT]["aliases"]
def receive(p, timeout=None, bufsize=io.DEFAULT_BUFFER_SIZE): """ Receive data from a process, yielding data read from stdout and stderr until proccess terminates or timeout expires. Unlike Popen.communicate(), this supports a timeout, and allows reading both stdout and stderr with a single thread. Example usage:: # Reading data from both stdout and stderr until process # terminates: for src, data in cmdutils.receive(p): if src == cmdutils.OUT: # handle output elif src == cmdutils.ERR: # handler errors # Receiving data with a timeout: try: received = list(cmdutils.receive(p, timeout=10)) except cmdutils.TimeoutExpired: # handle timeout Arguments: p (`subprocess.Popen`): A subprocess created with subprocess.Popen or subprocess32.Popen or cpopen.CPopen. timeout (float): Number of seconds to wait for process. Timeout resolution is limited by the resolution of `common.time.monotonic_time`, typically 10 milliseconds. bufsize (int): Number of bytes to read from the process in each iteration. Returns: Generator of tuples (SRC, bytes). SRC may be either `cmdutils.OUT` or `cmdutils.ERR`, and bytes is a bytes object read from process stdout or stderr. Raises: `cmdutils.TimeoutExpired` if process did not terminate within the specified timeout. """ if timeout is not None: deadline = monotonic_time() + timeout remaining = timeout else: deadline = None remaining = None fds = {} if p.stdout: fds[p.stdout.fileno()] = OUT if p.stderr: fds[p.stderr.fileno()] = ERR if fds: poller = select.poll() for fd in fds: poller.register(fd, select.POLLIN) def discard(fd): if fd in fds: del fds[fd] poller.unregister(fd) while fds: log.debug("Waiting for process (pid=%d, remaining=%s)", p.pid, remaining) # Unlike all other time apis, poll is using milliseconds remaining_msec = remaining * 1000 if deadline else None try: ready = poller.poll(remaining_msec) except select.error as e: if e[0] != errno.EINTR: raise log.debug("Polling process (pid=%d) interrupted", p.pid) else: for fd, mode in ready: if mode & select.POLLIN: data = osutils.uninterruptible(os.read, fd, bufsize) if not data: log.debug("Fd %d closed, unregistering", fd) discard(fd) continue yield fds[fd], data else: log.debug("Fd %d hangup/error, unregistering", fd) discard(fd) if deadline: remaining = deadline - monotonic_time() if remaining <= 0: raise TimeoutExpired(p.pid) _wait(p, deadline)
def __init__(self): super(PollSelector, self).__init__() self._poll = select.poll()
matchObj = re.search(r'mouse0 (event\d+)', text, flags=0) if matchObj: newInput = '/dev/input/'+matchObj.group(1) return newInput mouse_device = '/dev/input/event3' dev = open(mouse_device,'rb') running = True print(getMouseDevice()) while running == True: mouseEventFormat = 'llHHI' keyEventSize = struct.calcsize(mouseEventFormat) #wait for new input on keyboard device pollobject = select.poll() pollobject.register(dev) print('kes: %s'%keyEventSize) listening = True while listening == True: result = pollobject.poll(1) if result[0][1] != 5: continue event = dev.read(keyEventSize) (time1, time2, eType, kCode, value) = struct.unpack(mouseEventFormat, event) if eType == 0: continue if kCode in [8,272,273,274] or value < 0:print('pressed') if value > 1: print('moved') print(str(struct.unpack(mouseEventFormat, event)))
import struct from subprocess import Popen, PIPE, call from select import poll, POLLIN, POLLERR, POLLHUP # Set-up student_id = "r0634191" printing = False magic_string = "anet_dispatcher started successfully." sys.stdout.write(magic_string) sys.stdout.flush() processes = { } # A dictionary mapping process.stdout.name to [process, request_id]-dictionaries unbuffered_stdin = fdopen( sys.stdin.fileno(), "rb", 0 ) # We create a separate reader which doesn't buffer, so epoll always knows if there is actually data left to be read poll_object = poll() poll_object.register(unbuffered_stdin, POLLIN | POLLERR | POLLHUP) should_run = True while should_run: # Wait for events events = poll_object.poll() # Handle events for fd, event in events: if fd == unbuffered_stdin.fileno(): # Event comes from standard input stream