def __init__(self, args, debugger, time, memory): self._args = args self._child = _find_exe(self._args[0]) self._debugger = debugger self._time = time self._memory = memory self._returncode = None self._tle = False self._pid = None self._rusage = None self._duration = None self._r_duration = None self._stdin_, self._stdin = os.pipe() self._stdout, self._stdout_ = os.pipe() self._stderr, self._stderr_ = os.pipe() self.stdin = os.fdopen(self._stdin, 'w') self.stdout = os.fdopen(self._stdout, 'r') self.stderr = os.fdopen(self._stderr, 'r') self._started = threading.Event() self._died = threading.Event() self._worker = threading.Thread(target=self.__spawn_execute) self._worker.start() if 0 and time: # Spawn thread to kill process after it times out self._shocker = threading.Thread(target=self.__shocker) self._shocker.start()
def connect_to_new_process(self, fds): '''Request forkserver to create a child process. Returns a pair of fds (status_r, data_w). The calling process can read the child process's pid and (eventually) its returncode from status_r. The calling process should write to data_w the pickled preparation and process data. ''' self.ensure_running() if len(fds) + 4 >= MAXFDS_TO_SEND: raise ValueError('too many fds') with socket.socket(socket.AF_UNIX) as client: client.connect(self._forkserver_address) parent_r, child_w = os.pipe() child_r, parent_w = os.pipe() allfds = [child_r, child_w, self._forkserver_alive_fd, semaphore_tracker.getfd()] allfds += fds try: reduction.sendfds(client, allfds) return parent_r, parent_w except: os.close(parent_r) os.close(parent_w) raise finally: os.close(child_r) os.close(child_w)
def _get_handles(self, stdin, stdout, stderr): """Construct and return tuple with IO objects: p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ p2cread, p2cwrite = None, None c2pread, c2pwrite = None, None errread, errwrite = None, None if stdin is None: pass elif stdin == PIPE: p2cread, p2cwrite = os.pipe() elif isinstance(stdin, int): p2cread = stdin else: # Assuming file-like object p2cread = stdin.fileno() if stdout is None: pass elif stdout == PIPE: try: c2pread, c2pwrite = os.pipe() except: # Cleanup of previous pipe() descriptors if stdin == PIPE: os.close(p2cread) os.close(p2cwrite) raise elif isinstance(stdout, int): c2pwrite = stdout else: # Assuming file-like object c2pwrite = stdout.fileno() if stderr is None: pass elif stderr == PIPE: try: errread, errwrite = os.pipe() except: # Cleanup of previous pipe() descriptors if stdin == PIPE: os.close(p2cread) os.close(p2cwrite) if stdout == PIPE: os.close(c2pread) os.close(c2pwrite) raise elif stderr == STDOUT: errwrite = c2pwrite elif isinstance(stderr, int): errwrite = stderr else: # Assuming file-like object errwrite = stderr.fileno() return (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite)
def __init__(self, rrdtool='rrdtool'): """Get an rrdtool pipe.""" r1=None r2=None w1=None w2=None self.pid=0 self.pfile=None try: r1, w1=os.pipe() r2, w2=os.pipe() self.pfile=PipeFile(r1, w2) # Fork off rrdtool self.pid = os.fork() if self.pid == 0: self.pfile.close() os.dup2(r2, 0) os.dup2(w1, 1) os.dup2(w1, 2) os.execv(rrdtool, [rrdtool, '-']) os._exit(1) except: if r1: os.close(r1) if r2: os.close(r2) if w1: os.close(w1) if w2: os.close(w2) # Rethrow raise sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]
def exec_piped_fork(l, env, stdout, stderr): # spawn using fork / exec and providing a pipe for the command's # stdout / stderr stream if stdout != stderr: (rFdOut, wFdOut) = os.pipe() (rFdErr, wFdErr) = os.pipe() else: (rFdOut, wFdOut) = os.pipe() rFdErr = rFdOut wFdErr = wFdOut # do the fork pid = os.fork() if not pid: # Child process os.close( rFdOut ) if rFdOut != rFdErr: os.close( rFdErr ) os.dup2( wFdOut, 1 ) # is there some symbolic way to do that ? os.dup2( wFdErr, 2 ) os.close( wFdOut ) if stdout != stderr: os.close( wFdErr ) exitval = 127 try: os.execvpe(l[0], l, env) except OSError, e: exitval = exitvalmap.get(e[0], e[0]) stderr.write("scons: %s: %s\n" % (l[0], e[1])) os._exit(exitval)
def _launch(self, process_obj): os.environ["MULTIPROCESSING_FORKING_DISABLE"] = "1" spawn._Django_old_layout_hack__save() from . import semaphore_tracker tracker_fd = semaphore_tracker.getfd() self._fds.append(tracker_fd) prep_data = spawn.get_preparation_data(process_obj._name) fp = io.BytesIO() context.set_spawning_popen(self) try: reduction.dump(prep_data, fp) reduction.dump(process_obj, fp) finally: context.set_spawning_popen(None) parent_r = child_w = child_r = parent_w = None try: parent_r, child_w = os.pipe() child_r, parent_w = os.pipe() cmd = spawn.get_command_line(tracker_fd=tracker_fd, pipe_handle=child_r) self._fds.extend([child_r, child_w]) self.pid = spawnv_passfds( spawn.get_executable(), cmd, self._fds, ) self.sentinel = parent_r with open(parent_w, 'wb', closefd=False) as f: f.write(fp.getbuffer()) finally: if parent_r is not None: util.Finalize(self, os.close, (parent_r,)) for fd in (child_r, child_w, parent_w): if fd is not None: os.close(fd)
def start_subprocess(self): """Start octave using a subprocess (no tty support)""" errmsg = ('\n\nPlease install GNU Octave and put it in your path\n') ON_POSIX = 'posix' in sys.builtin_module_names if self.use_pty: master, slave = pty.openpty() self.wfid, self.rfid = master, master rpipe, wpipe = slave, slave else: self.rfid, wpipe = os.pipe() rpipe, self.wfid = os.pipe() kwargs = dict(close_fds=ON_POSIX, bufsize=0, stdin=rpipe, stderr=wpipe, stdout=wpipe) if os.name == 'nt': startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW kwargs['startupinfo'] = startupinfo try: proc = subprocess.Popen(['octave', '-q', '--braindead'], **kwargs) except OSError: # pragma: no cover raise Oct2PyError(errmsg) else: self.reader = _Reader(self.rfid, self.read_queue) return proc
def __init__(self, cmd, capturestderr=False, bufsize=-1): """The parameter 'cmd' is the shell command to execute in a sub-process. On UNIX, 'cmd' may be a sequence, in which case arguments will be passed directly to the program without shell intervention (as with os.spawnv()). If 'cmd' is a string it will be passed to the shell (as with os.system()). The 'capturestderr' flag, if true, specifies that the object should capture standard error output of the child process. The default is false. If the 'bufsize' parameter is specified, it specifies the size of the I/O buffers to/from the child process.""" _cleanup() self.cmd = cmd p2cread, p2cwrite = os.pipe() c2pread, c2pwrite = os.pipe() if capturestderr: errout, errin = os.pipe() self.pid = os.fork() if self.pid == 0: # Child os.dup2(p2cread, 0) os.dup2(c2pwrite, 1) if capturestderr: os.dup2(errin, 2) self._run_child(cmd) os.close(p2cread) self.tochild = os.fdopen(p2cwrite, 'w', bufsize) os.close(c2pwrite) self.fromchild = os.fdopen(c2pread, 'r', bufsize) if capturestderr: os.close(errin) self.childerr = os.fdopen(errout, 'r', bufsize) else: self.childerr = None
def _run(self): self._log.debug("Starting IOProcess...") myRead, hisWrite = os.pipe() hisRead, myWrite = os.pipe() self._partialLogs = "" cmd = [self.IOPROCESS_EXE, "--read-pipe-fd", str(hisRead), "--write-pipe-fd", str(hisWrite), "--max-threads", str(self._max_threads)] if self._DEBUG_VALGRIND: cmd = ["valgrind", "--log-file=ioprocess.valgrind.log", "--leak-check=full", "--tool=memcheck"] + cmd + \ ["--keep-fds"] p = CPopen(cmd) os.close(hisRead) os.close(hisWrite) setNonBlocking(myRead) setNonBlocking(myWrite) self._startCommunication(p, myRead, myWrite)
def start(self): assert not self._started self._started = True up_read, up_write = os.pipe() down_read, down_write = os.pipe() args, sock = self.args, self.sock pid = os.fork() if pid: # parent os.close(up_read) os.close(down_write) tulip.async(self.connect(pid, up_write, down_read)) else: # child os.close(up_write) os.close(down_read) # cleanup after fork tulip.set_event_loop(None) # setup process process = ChildProcess(up_read, down_write, args, sock) process.start()
def __init__(self, cmd, capturestderr=False, bufsize=-1): """The parameter 'cmd' is the shell command to execute in a sub-process. The 'capturestderr' flag, if true, specifies that the object should capture standard error output of the child process. The default is false. If the 'bufsize' parameter is specified, it specifies the size of the I/O buffers to/from the child process.""" _cleanup() p2cread, p2cwrite = os.pipe() c2pread, c2pwrite = os.pipe() if capturestderr: errout, errin = os.pipe() self.pid = os.fork() if self.pid == 0: # Child os.dup2(p2cread, 0) os.dup2(c2pwrite, 1) if capturestderr: os.dup2(errin, 2) self._run_child(cmd) os.close(p2cread) self.tochild = os.fdopen(p2cwrite, 'w', bufsize) os.close(c2pwrite) self.fromchild = os.fdopen(c2pread, 'r', bufsize) if capturestderr: os.close(errin) self.childerr = os.fdopen(errout, 'r', bufsize) else: self.childerr = None _active.append(self)
def __init__(self, *args, **kwargs): self.io_loop = kwargs.pop('io_loop', None) to_close = [] if kwargs.get('stdin') is Subprocess.STREAM: in_r, in_w = os.pipe() kwargs['stdin'] = in_r to_close.append(in_r) self.stdin = PipeIOStream(in_w, io_loop=self.io_loop) if kwargs.get('stdout') is Subprocess.STREAM: out_r, out_w = os.pipe() kwargs['stdout'] = out_w to_close.append(out_w) self.stdout = PipeIOStream(out_r, io_loop=self.io_loop) if kwargs.get('stderr') is Subprocess.STREAM: err_r, err_w = os.pipe() kwargs['stderr'] = err_w to_close.append(err_w) self.stdout = PipeIOStream(err_r, io_loop=self.io_loop) self.proc = subprocess.Popen(*args, **kwargs) for fd in to_close: os.close(fd) for attr in ['stdin', 'stdout', 'stderr', 'pid']: if not hasattr(self, attr): # don't clobber streams set above setattr(self, attr, getattr(self.proc, attr)) self._exit_callback = None self.returncode = None
def _processRequest(conn, sessionDict): DEBUG(EXTCGI, 'extcgi Processing Request') #dict of environ, headers, stdin kid_stdin, parent_to_kid_stdin = os.pipe() parent_to_kid_stdout, kid_stdout = os.pipe() parent_to_kid_stderr, kid_stderr = os.pipe() pid = os.fork() try: if pid: #ok I'm the parent DEBUG(EXTCGI, 'child pid is %d' % pid) #close kid sides os.close(kid_stdin) os.close(kid_stdout) os.close(kid_stderr) stdin = conn._stdin return _doCGI(conn, pid, parent_to_kid_stdin, parent_to_kid_stdout, parent_to_kid_stderr, stdin) else: #I'm the kid #close parent side of pipes os.close(parent_to_kid_stdin) os.close(parent_to_kid_stdout) os.close(parent_to_kid_stderr) #dup kid sides to my stdin/out/err os.dup2(kid_stdin, 0) os.dup2(kid_stdout, 1) os.dup2(kid_stderr, 2) env = _fix(conn.env) if DEBUGIT(EXTCGI): DEBUG(EXTCGI, "environment is %s" % _dispenv(env)) prog = Configuration.CGIProgram args = ( (prog,) + (prog,) + Configuration.CGIProgramArgs +(env,)) DEBUG(EXTCGI, 'args is %s' % repr(args)) oldpwd = os.getcwd() try: os.chdir(os.path.split(env["PATH_TRANSLATED"])[0]) os.execle(*args) finally: os.chdir(oldpwd) except: if pid == 0: #I'm the kid logException() #give the parent some info as to why I died os.write(kid_stderr, "exception executing CGI : %s %s" % ( sys.exc_info()[0], sys.exc_info()[1])) DEBUG(EXTCGI, "I'm still here! killing self"); os.kill(os.getpid(), 9) else: #I'm the parent e, t, tb = sys.exc_info() try: os.kill(pid, 9) # we screwed up, kill kid except: #in event that it's already dead, that's ok too pass raise e, t, tb
def spawn_test(test, prefix, passthrough, run_skipped, show_cmd): """Spawn one child, return a task struct.""" if not test.enable and not run_skipped: return None cmd = test.get_command(prefix) if show_cmd: print(escape_cmdline(cmd)) if not passthrough: (rout, wout) = os.pipe() (rerr, werr) = os.pipe() rv = os.fork() # Parent. if rv: os.close(wout) os.close(werr) return Task(test, prefix, rv, rout, rerr) # Child. os.close(rout) os.close(rerr) os.dup2(wout, 1) os.dup2(werr, 2) os.execvp(cmd[0], cmd)
def __init__(self, tests, num_workers, total_cpus, total_mem, bindir): """ Initialize the class. @param tests: A list of test dictionaries. @param num_workers: The number of workers (pipelines). @param total_cpus: The total number of CPUs to dedicate to tests. @param total_mem: The total amount of memory to dedicate to tests. @param bindir: The directory where environment files reside. """ self.tests = tests self.num_workers = num_workers self.total_cpus = total_cpus self.total_mem = total_mem self.bindir = bindir # Pipes -- s stands for scheduler, w stands for worker self.s2w = [os.pipe() for i in range(num_workers)] self.w2s = [os.pipe() for i in range(num_workers)] self.s2w_r = [os.fdopen(r, "r", 0) for r, w in self.s2w] self.s2w_w = [os.fdopen(w, "w", 0) for r, w in self.s2w] self.w2s_r = [os.fdopen(r, "r", 0) for r, w in self.w2s] self.w2s_w = [os.fdopen(w, "w", 0) for r, w in self.w2s] # "Personal" worker dicts contain modifications that are applied # specifically to each worker. For example, each worker must use a # different environment file and a different MAC address pool. self.worker_dicts = [{"env": "env%d" % i} for i in range(num_workers)]
def test_gevent_friendly(self): # Used to verify that file descriptors aren't consumed r,w = os.pipe() os.close(r) os.close(w) # Get a good benchmark without any concurrent actions t1 = time.time() with AsyncDispatcher(self.block_stuff) as dispatcher: v = dispatcher.wait(10) dt = time.time() - t1 # Check that it takes less than 5 seconds and that it's the right value self.assertTrue(dt < 10) self.assertTrue(np.array_equal(v,np.arange(20))) # Try it again but this time with a gevent sleep that should run # Concurrently with the dispatcher thread t1 = time.time() with AsyncDispatcher(self.block_stuff) as dispatcher: gevent.sleep(5) v = dispatcher.wait(10) ndt = time.time() - t1 # There is ususally some difference but should definitely be less than # one second self.assertTrue( abs(dt - ndt) < 5) try: # Make sure we're not losing file descriptors to maintain thread synchronization self.assertEquals((r,w), os.pipe()) finally: os.close(r) os.close(w)
def spawn(prog, args): p2cread, p2cwrite = os.pipe() c2pread, c2pwrite = os.pipe() pid = os.fork() if pid == 0: # Child for i in 0, 1, 2: try: os.close(i) except os.error: pass if os.dup(p2cread) != 0: sys.stderr.write('popen2: bad read dup\n') if os.dup(c2pwrite) != 1: sys.stderr.write('popen2: bad write dup\n') if os.dup(c2pwrite) != 2: sys.stderr.write('popen2: bad write dup\n') os.closerange(3, MAXFD) try: os.execvp(prog, args) finally: sys.stderr.write('execvp failed\n') os._exit(1) os.close(p2cread) os.close(c2pwrite) return pid, c2pread, p2cwrite
def decrypt_content(content): """Decrypt content and verify a valid signature exists.""" r_status, w_status = os.pipe() r_log, w_log = os.pipe() r_att, w_att = os.pipe() command = get_gpg_command() command.extend(['--decrypt', '--status-fd', str(w_status), '--logger-fd', str(w_log), '--attribute-fd', str(w_att)]) p = subprocess.Popen(command, stdout=subprocess.PIPE, stdin=subprocess.PIPE, pass_fds=[w_status, w_log, w_att]) output = p.communicate(input=content.encode('utf-8'))[0] if p.returncode != 0: raise Exception('Error decrypting content') os.close(w_status) os.close(w_log) os.close(w_att) config = get_config_dict() messages = os.fdopen(r_status).read() if not (' VALIDSIG ' + config['default-key']) in messages: raise Exception('No valid signature from default key') return output.decode()
def does_stuff(): a, b = os.pipe() c = os.dup(a) d = os.dup(b) assert a != b assert a != c assert a != d assert b != c assert b != d assert c != d os.close(c) os.dup2(d, c) e, f = os.pipe() assert e != a assert e != b assert e != c assert e != d assert f != a assert f != b assert f != c assert f != d assert f != e os.close(a) os.close(b) os.close(c) os.close(d) os.close(e) os.close(f) return 42
def spawn(self, children): if self.children: raise RuntimeError, "children are already running" self.childinfo = [] self.fdchild = {} self.childitem = [None] * children self.readpipes = [] self.reporter.init(children) self.children = children for i in range(children): childread, parentwrite = os.pipe() parentread, childwrite = os.pipe() pid = os.fork() self.fdchild[parentread] = i if pid > 0: os.close(childread) os.close(childwrite) self.childinfo.append((pid, parentread, parentwrite)) self.reporter.spawn(pid) else: os.close(parentread) os.close(parentwrite) self.child(childread, childwrite)
def __init_streams(self, stdin, stdout, stderr, unbuffered): self.stdin = self.stdout = self.stderr = None if unbuffered: master, slave = pty.openpty() if stdin is PIPE: self._child_stdin, self._stdin = (slave, master) if unbuffered else os.pipe() self.stdin = os.fdopen(self._stdin, 'w') elif isinstance(stdin, int): self._child_stdin, self._stdin = stdin, -1 elif stdin is not None: self._child_stdin, self._stdin = stdin.fileno(), -1 else: self._child_stdin = self._stdin = -1 if stdout is PIPE: self._stdout, self._child_stdout = (master, slave) if unbuffered else os.pipe() self.stdout = os.fdopen(self._stdout, 'r') elif isinstance(stdout, int): self._stdout, self._child_stdout = -1, stdout elif stdout is not None: self._stdout, self._child_stdout = -1, stdout.fileno() else: self._stdout = self._child_stdout = -1 if stderr is PIPE: self._stderr, self._child_stderr = os.pipe() self.stderr = os.fdopen(self._stderr, 'r') elif isinstance(stderr, int): self._stderr, self._child_stderr = -1, stderr elif stderr is not None: self._stderr, self._child_stderr = -1, stderr.fileno() else: self._stderr = self._child_stderr = -1
def _OpenPipes (self, cmd, args): # PRE: cmd is assigned the path to a prolog interpreter # && args is assigned a tuple containing any arguments to pass to cmd # POST: cmd has been spawned with args as arguments # && FCTVAL = (file descriptor to incoming pipe, # file descriptor to outgoing pipe, # file descriptor to incoming error pipe, # child process's pid) # "A better popen2 than popen2.popen2" pipe1 = os.pipe () pipe2 = os.pipe () pipe3 = os.pipe () pid = os.fork () #pid = os.spawnv(os.P_NOWAIT, cmd, args) if pid: # I'm the parent os.close (pipe1[1]) os.close (pipe2[0]) os.close (pipe3[1]) return (pipe1[0], pipe2[1], pipe3[0], pid) else: # I'm the child os.close (pipe1[0]) os.close (pipe2[1]) os.close (pipe3[0]) os.dup2 (pipe1[1], 1) os.dup2 (pipe2[0], 0) os.dup2 (pipe3[1], 2) os.execvp (cmd, args)
def _bash2(command, mode='o'): stdout_r, stdout_w = None, None (stdin_r, stdin_w) = os.pipe() if 'o' in mode: (stdout_r, stdout_w) = os.pipe() p = Popen('/bin/bash', stdin=stdin_r, stdout=stdout_w) os.write(stdin_w, command + '\nexit $?\n') os.close(stdin_w) output = [] ret = p.wait() os.close(stdin_r) if stdout_r: os.close(stdout_w) out = os.read(stdout_r, 32768) while out != '': output.append(out) out = os.read(stdout_r, 32768) output = (''.join(output)).strip() os.close(stdout_r) if 'ov' == mode: return (output, ret) elif 'o' in mode: return output elif 'v' in mode: return ret return None
def __init__(self, prog = None, r = None, w = None): self.prog = prog self.objects = {} self.warn = 0 self._erase = [] self.wantarray = 1 if prog is None: return self.minor() if prog == -1: self.r = r self.w = w self.role = "TEST" return ch_reader, writer = os.pipe() reader, ch_writer = os.pipe() pid = os.fork() if pid==0: prog = PROG.get(prog, prog) prog = prog % (os.path.dirname(RPC.__module__.__file__)) if ch_reader != 4: os.dup2(ch_reader, 4) if ch_writer != 5: os.dup2(ch_writer, 5) args = shlex.split(prog) os.execvp(args[0], args[1:]) self.r = os.fdopen(reader, "rb") self.w = os.fdopen(writer, "wb") self.role = "MAJOR"
def __init__(self, cwd=None, env=None): BaseProxy.__init__(self, set()) if env is None: env = os.environ.copy() self._servers = [ None ] * 5 p0, p1 = os.pipe(), os.pipe() self._pid = os.fork() if self._pid == 0: os.close(p1[1]) os.close(p0[0]) if not cwd: if 'conary_test.serverCacheProxy' in sys.modules: fpath = sys.modules['conary_test.serverCacheProxy'].__file__ else: fpath = sys.argv[0] cwd = os.path.dirname(fpath) cmd = ["/usr/bin/python", "-c", "from conary_test import serverCacheProxy; " "serverCacheProxy.Child(%d, %d)" % (p1[0], p0[1])] os.chdir(cwd) os.execve(cmd[0], cmd, env) os._exit(0) try: os.close(p0[1]) os.close(p1[0]) self._pipe = Pipe(p0[0], p1[1]) sdict = loadData(self._pipe) self._methods = sdict['methods'] #print "Child started", methodname, params except: os.waitpid(self._pid, 0) raise
def startQuickAsy(): global quickAsy global quickAsyFailed global AsyTempDir global fout,fin if quickAsyRunning(): return try: fout.close() quickAsy.wait() except: pass try: quickAsyFailed = False if os.name == "nt": AsyTempDir=mkdtemp(prefix="asy_", dir="./") else: AsyTempDir=mkdtemp(prefix="asy_")+os.sep if sys.platform[:3] == 'win': quickAsy=Popen([xasyOptions.options['asyPath'],"-noV","-multiline","-q", "-o"+AsyTempDir,"-inpipe=0","-outpipe=2"],stdin=PIPE,stderr=PIPE) fout=quickAsy.stdin fin=quickAsy.stderr else: (rx,wx) = os.pipe() (ra,wa) = os.pipe() quickAsy=Popen([xasyOptions.options['asyPath'],"-noV","-multiline","-q", "-o"+AsyTempDir,"-inpipe="+str(rx),"-outpipe="+str(wa)]) fout=os.fdopen(wx,'w') fin=os.fdopen(ra,'r') if quickAsy.returncode != None: quickAsyFailed = True except: quickAsyFailed = True
def timed_command(cmd, timeout=None): '''timed_command(cmd, timeout=None) Run a shell command, with an optional timeout If command takes longer than specified time it will be killed, first with SIGTERM then 2 seconds later with SIGKILL if needed. (The 2 second value can be adjusted by changing GRACE_PERIOD). Return (exit_status, time_used, command_output, error_output) If the program was terminated due to timeout, the exit status will be artificially set as though the program returned ETIMEDOUT.''' if timeout==0: timeout=None stdin_r, stdin_w = os.pipe() stdout_r, stdout_w = os.pipe() stderr_r, stderr_w = os.pipe() prev_sighandler = signal.getsignal(signal.SIGCHLD) signal.signal(signal.SIGCHLD, _sighandler) pid = os.fork() if pid == 0: # Child _child(stdin_r, stdout_w, stderr_w, cmd) else: # Parent ret = _parent(stdout_r, stderr_r, pid, timeout) # Close all file descriptors we opened for fd in stdin_r, stdin_w, stdout_r, stdout_w, stderr_r, stderr_w: try: os.close(fd) except: pass # Restore default behavior on SIGCHLD signal.signal(signal.SIGCHLD, prev_sighandler) return ret
def spawn(prog, args): p2cread, p2cwrite = os.pipe() c2pread, c2pwrite = os.pipe() pid = os.fork() if pid == 0: # Child os.close(0) os.close(1) os.close(2) if os.dup(p2cread) <> 0: sys.stderr.write('popen2: bad read dup\n') if os.dup(c2pwrite) <> 1: sys.stderr.write('popen2: bad write dup\n') if os.dup(c2pwrite) <> 2: sys.stderr.write('popen2: bad write dup\n') for i in range(3, MAXFD): try: os.close(i) except: pass try: os.execvp(prog, args) finally: sys.stderr.write('execvp failed\n') os._exit(1) os.close(p2cread) os.close(c2pwrite) return pid, c2pread, p2cwrite
def __init__(self): (client_reader, server_writer) = os.pipe() (server_reader, client_writer) = os.pipe() if not os.fork(): # the child process starts a server and exits when done RMI.DEBUG_MSG_PREFIX = ' SERVER' #RMI.DEBUG_FLAG = 1 server_reader = os.fdopen(server_reader, 'r') server_writer = os.fdopen(server_writer, 'w') s = RMI.Node(reader = server_reader, writer = server_writer) # the server should return fals whenever a client disconnects # somehow it just hangs :( # we need to fix this response = s.receive_request_and_send_response() while (response): if response[3] == 'exitnow': response = None else: response = s.receive_request_and_send_response() print("SERVER DONE") exit() else: # the parent process initializes as the client and continues RMI.DEBUG_MSG_PREFIX = 'CLIENT' #RMI.DEBUG_FLAG = 1 client_reader = os.fdopen(client_reader, 'r') client_writer = os.fdopen(client_writer, 'w') RMI.Client.__init__(self,client_reader,client_writer)
def kinit(username, password, cfile=None): kc = cfile if kc is None: kc = mkstemp(); os.close(kc[0]); kc = kc[1] p2fr, p2fw = os.pipe() f2pr, f2pw = os.pipe() pid = os.fork() if pid == 0: os.dup2(p2fr, 0) os.dup2(f2pw, 1) os.dup2(f2pw, 2) os.execv('/usr/bin/kinit', ['kinit', '-l', '3d', '-c', kc, username]) else: os.close(p2fr) os.close(f2pw) os.fdopen(p2fw,'w').write('%s\n' % password) pres = os.waitpid(pid, 0) if pres[1] == 0: # return ticket kc return kc else: # execv failure if cfile is None: # remove ticket kc os.unlink(kc)
def run(self): self.logger.debug("Starting logging thread") self.readpipe, self.writepipe = os.pipe() threading.Thread.run(self)
def setUp(self): self.read_fd, self.write_fd = os.pipe()
def test_os_pipe(): import os r, w = os.pipe() written = os.write(w, b"hello") assert os.read(r, written) == b"hello"
def execWithRedirect(command, argv, stdin = None, stdout = None, stderr = None, root = '/'): def chroot (): os.chroot(root) stdinclose = stdoutclose = stderrclose = lambda : None argv = list(argv) if isinstance(stdin, str): if os.access(stdin, os.R_OK): stdin = os.open(stdin, os.O_RDONLY) stdinclose = lambda : os.close(stdin) else: stdin = sys.stdin.fileno() elif isinstance(stdin, int): pass elif stdin is None or not isinstance(stdin, file): stdin = sys.stdin.fileno() if isinstance(stdout, str): stdout = os.open(stdout, os.O_RDWR|os.O_CREAT) stdoutclose = lambda : os.close(stdout) elif isinstance(stdout, int): pass elif stdout is None or not isinstance(stdout, file): stdout = sys.stdout.fileno() if isinstance(stderr, str): stderr = os.open(stderr, os.O_RDWR|os.O_CREAT) stderrclose = lambda : os.close(stderr) elif isinstance(stderr, int): pass elif stderr is None or not isinstance(stderr, file): stderr = sys.stderr.fileno() program_log.info("Running... %s" % (" ".join([command] + argv),)) #prepare os pipes for feeding tee proceses pstdout, pstdin = os.pipe() perrout, perrin = os.pipe() env = os.environ.copy() env.update({"LC_ALL": "C", "LANGUAGE": "C", "LANG": "C"}) try: #prepare tee proceses proc_std = tee(pstdout, stdout, program_log.info) proc_err = tee(perrout, stderr, program_log.error) #start monitoring the outputs proc_std.start() proc_err.start() proc = subprocess.Popen([command] + argv, stdin=stdin, stdout=pstdin, stderr=perrin, preexec_fn=chroot, cwd=root, env=env) proc.wait() ret = proc.returncode #close the input ends of pipes so we get EOF in the tee processes os.close(pstdin) os.close(perrin) #wait for the output to be written and destroy them proc_std.join() del proc_std proc_err.join() del proc_err stdinclose() stdoutclose() stderrclose() except OSError as e: errstr = "Error running %s: %s" % (command, e.strerror) log.error(errstr) program_log.error(errstr) #close the input ends of pipes so we get EOF in the tee processes os.close(pstdin) os.close(perrin) proc_std.join() proc_err.join() stdinclose() stdoutclose() stderrclose() raise RuntimeError, errstr return ret
def execWithCallback(command, argv, stdin = None, stdout = None, stderr = None, echo = True, callback = None, callback_data = None, root = '/'): def chroot(): os.chroot(root) def closefds (): stdinclose() stdoutclose() stderrclose() stdinclose = stdoutclose = stderrclose = lambda : None argv = list(argv) if isinstance(stdin, str): if os.access(stdin, os.R_OK): stdin = os.open(stdin, os.O_RDONLY) stdinclose = lambda : os.close(stdin) else: stdin = sys.stdin.fileno() elif isinstance(stdin, int): pass elif stdin is None or not isinstance(stdin, file): stdin = sys.stdin.fileno() if isinstance(stdout, str): stdout = os.open(stdout, os.O_RDWR|os.O_CREAT) stdoutclose = lambda : os.close(stdout) elif isinstance(stdout, int): pass elif stdout is None or not isinstance(stdout, file): stdout = sys.stdout.fileno() if isinstance(stderr, str): stderr = os.open(stderr, os.O_RDWR|os.O_CREAT) stderrclose = lambda : os.close(stderr) elif isinstance(stderr, int): pass elif stderr is None or not isinstance(stderr, file): stderr = sys.stderr.fileno() program_log.info("Running... %s" % (" ".join([command] + argv),)) p = os.pipe() p_stderr = os.pipe() childpid = os.fork() if not childpid: os.close(p[0]) os.close(p_stderr[0]) os.dup2(p[1], 1) os.dup2(p_stderr[1], 2) os.dup2(stdin, 0) os.close(stdin) os.close(p[1]) os.close(p_stderr[1]) os.execvp(command, [command] + argv) os._exit(1) os.close(p[1]) os.close(p_stderr[1]) logline = '' while 1: try: s = os.read(p[0], 1) except OSError as e: if e.errno != 4: raise IOError, e.args if echo: os.write(stdout, s) if s == '\n': program_log.info(logline) logline = '' else: logline += s if callback: callback(s, callback_data=callback_data) # break out early if the sub-process changes status. # no need to flush the stream if the process has exited try: (pid, status) = os.waitpid(childpid,os.WNOHANG) if pid != 0: break except OSError as e: log.critical("exception from waitpid: %s %s" %(e.errno, e.strerror)) if len(s) < 1: break if len(logline) > 0: program_log.info(logline) log_errors = '' while 1: try: err = os.read(p_stderr[0], 128) except OSError as e: if e.errno != 4: raise IOError, e.args break log_errors += err if len(err) < 1: break map(program_log.error, log_errors.splitlines()) os.close(p[0]) os.close(p_stderr[0]) try: #if we didn't already get our child's exit status above, do so now. if not pid: (pid, status) = os.waitpid(childpid, 0) except OSError as e: log.critical("exception from waitpid: %s %s" %(e.errno, e.strerror)) closefds() # *shrug* no clue why this would happen, but hope that things are fine if status is None: return 0 if os.WIFEXITED(status): return os.WEXITSTATUS(status) return 1
import os, sys, time rd, wd = os.pipe() r, w = os.fdopen(rd, 'rb', 0), os.fdopen(wd, 'wb', 0) pid = os.fork() if (pid == 0): print('if') r.close() #time.sleep(5) for i in range(10): print('bucle for') mensaje = 'linea %d \n' % i w.write(mensaje.encode('utf8')) w.flush() #Podemos comentar la siguiente linea y observamos el comportamiento, ayuda a entender el ejercicio time.sleep(1) print('Soy el hijo \n') if (pid == -1): print('Error') else: print('else') w.close() while True: print('while True') data = r.readline() print('data') if not data:
def make_iostream_pair(self, **kwargs): r, w = os.pipe() return PipeIOStream(r, **kwargs), PipeIOStream(w, **kwargs)
def __init__(self): self.queue = Queue.Queue() self.pipe = os.pipe() self._closed = False
def sendzfs(fromsnap, tosnap, dataset, localfs, remotefs, followdelete, throttle, compression, replication, reached_last): global results global templog progressfile = '/tmp/.repl_progress_%d' % replication.id cmd = ['/sbin/zfs', 'send', '-V'] # -p switch will send properties for whole dataset, including snapshots # which will result in stale snapshots being delete as well if followdelete: cmd.append('-p') if fromsnap is None: cmd.append("%s@%s" % (dataset, tosnap)) else: cmd.extend(['-i', "%s@%s" % (dataset, fromsnap), "%s@%s" % (dataset, tosnap)]) # subprocess.Popen does not handle large stream of data between # processes very well, do it on our own readfd, writefd = os.pipe() zproc_pid = os.fork() if zproc_pid == 0: os.close(readfd) os.dup2(writefd, 1) os.close(writefd) os.execv('/sbin/zfs', cmd) # NOTREACHED else: with open(progressfile, 'w') as f2: f2.write(str(zproc_pid)) os.close(writefd) compress, decompress = compress_pipecmds(compression) replcmd = '%s%s/bin/dd obs=1m 2> /dev/null | /bin/dd obs=1m 2> /dev/null | /usr/local/bin/pipewatcher $$ | %s "%s/sbin/zfs receive -F -d \'%s\' && echo Succeeded"' % (compress, throttle, sshcmd, decompress, remotefs) log.debug('Sending zfs snapshot: %s | %s', ' '.join(cmd), replcmd) with open(templog, 'w+') as f: readobj = os.fdopen(readfd, 'rb', 0) proc = subprocess.Popen( replcmd, shell=True, stdin=readobj, stdout=f, stderr=subprocess.STDOUT, ) proc.wait() os.waitpid(zproc_pid, os.WNOHANG) readobj.close() os.remove(progressfile) f.seek(0) msg = f.read().strip('\n').strip('\r') os.remove(templog) msg = msg.replace('WARNING: ENABLED NONE CIPHER', '') msg = msg.strip('\r').strip('\n') log.debug("Replication result: %s" % (msg)) results[replication.id]['msg'] = msg # When replicating to a target "container" dataset that doesn't exist on the sending # side the target dataset will have to be readonly, however that will preclude # creating mountpoints for the datasets that are sent. # In that case you'll get back a failed to create mountpoint message, which # we'll go ahead and consider a success. if reached_last and ("Succeeded" in msg or "failed to create mountpoint" in msg): results[replication.id]['last_snapshot'] = tosnap return ("Succeeded" in msg or "failed to create mountpoint" in msg)
def launch(self, launch_cmd, get_ip=True, qemuparams=None, extra_bootparams=None, env=None): try: threadsock, threadport = self.create_socket() self.server_socket, self.serverport = self.create_socket() except socket.error as msg: self.logger.error("Failed to create listening socket: %s" % msg[1]) return False bootparams = 'console=tty1 console=ttyS0,115200n8 printk.time=1' if extra_bootparams: bootparams = bootparams + ' ' + extra_bootparams # Ask QEMU to store the QEMU process PID in file, this way we don't have to parse running processes # and analyze descendents in order to determine it. if os.path.exists(self.qemu_pidfile): os.remove(self.qemu_pidfile) self.qemuparams = 'bootparams="{0}" qemuparams="-serial tcp:127.0.0.1:{1} -pidfile {2}"'.format( bootparams, threadport, self.qemu_pidfile) if qemuparams: self.qemuparams = self.qemuparams[:-1] + " " + qemuparams + " " + '\"' launch_cmd += ' tcpserial=%s %s' % (self.serverport, self.qemuparams) self.origchldhandler = signal.getsignal(signal.SIGCHLD) signal.signal(signal.SIGCHLD, self.handleSIGCHLD) self.logger.debug('launchcmd=%s' % (launch_cmd)) # FIXME: We pass in stdin=subprocess.PIPE here to work around stty # blocking at the end of the runqemu script when using this within # oe-selftest (this makes stty error out immediately). There ought # to be a proper fix but this will suffice for now. self.runqemu = subprocess.Popen(launch_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, preexec_fn=os.setpgrp, env=env) output = self.runqemu.stdout # # We need the preexec_fn above so that all runqemu processes can easily be killed # (by killing their process group). This presents a problem if this controlling # process itself is killed however since those processes don't notice the death # of the parent and merrily continue on. # # Rather than hack runqemu to deal with this, we add something here instead. # Basically we fork off another process which holds an open pipe to the parent # and also is setpgrp. If/when the pipe sees EOF from the parent dieing, it kills # the process group. This is like pctrl's PDEATHSIG but for a process group # rather than a single process. # r, w = os.pipe() self.monitorpid = os.fork() if self.monitorpid: os.close(r) self.monitorpipe = os.fdopen(w, "w") else: # child process os.setpgrp() os.close(w) r = os.fdopen(r) x = r.read() os.killpg(os.getpgid(self.runqemu.pid), signal.SIGTERM) sys.exit(0) self.logger.debug("runqemu started, pid is %s" % self.runqemu.pid) self.logger.debug("waiting at most %s seconds for qemu pid (%s)" % (self.runqemutime, time.strftime("%D %H:%M:%S"))) endtime = time.time() + self.runqemutime while not self.is_alive() and time.time() < endtime: if self.runqemu.poll(): if self.runqemu.returncode: # No point waiting any longer self.logger.debug('runqemu exited with code %d' % self.runqemu.returncode) self._dump_host() self.stop() self.logger.debug("Output from runqemu:\n%s" % self.getOutput(output)) return False time.sleep(0.5) if not self.is_alive(): self.logger.error("Qemu pid didn't appear in %s seconds (%s)" % (self.runqemutime, time.strftime("%D %H:%M:%S"))) # Dump all processes to help us to figure out what is going on... ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command '], stdout=subprocess.PIPE).communicate()[0] processes = ps.decode("utf-8") self.logger.debug("Running processes:\n%s" % processes) self._dump_host() self.stop() op = self.getOutput(output) if op: self.logger.error("Output from runqemu:\n%s" % op) else: self.logger.error("No output from runqemu.\n") return False # We are alive: qemu is running out = self.getOutput(output) netconf = False # network configuration is not required by default self.logger.debug( "qemu started in %s seconds - qemu procces pid is %s (%s)" % (time.time() - (endtime - self.runqemutime), self.qemupid, time.strftime("%D %H:%M:%S"))) if get_ip: cmdline = '' with open('/proc/%s/cmdline' % self.qemupid) as p: cmdline = p.read() # It is needed to sanitize the data received # because is possible to have control characters cmdline = re_control_char.sub(' ', cmdline) try: ips = re.findall("((?:[0-9]{1,3}\.){3}[0-9]{1,3})", cmdline.split("ip=")[1]) self.ip = ips[0] self.server_ip = ips[1] self.logger.debug("qemu cmdline used:\n{}".format(cmdline)) except (IndexError, ValueError): # Try to get network configuration from runqemu output match = re.match( '.*Network configuration: ([0-9.]+)::([0-9.]+):([0-9.]+)$.*', out, re.MULTILINE | re.DOTALL) if match: self.ip, self.server_ip, self.netmask = match.groups() # network configuration is required as we couldn't get it # from the runqemu command line, so qemu doesn't run kernel # and guest networking is not configured netconf = True else: self.logger.error( "Couldn't get ip from qemu command line and runqemu output! " "Here is the qemu command line used:\n%s\n" "and output from runqemu:\n%s" % (cmdline, out)) self._dump_host() self.stop() return False self.logger.debug("Target IP: %s" % self.ip) self.logger.debug("Server IP: %s" % self.server_ip) self.thread = LoggingThread(self.log, threadsock, self.logger) self.thread.start() if not self.thread.connection_established.wait(self.boottime): self.logger.error("Didn't receive a console connection from qemu. " "Here is the qemu command line used:\n%s\nand " "output from runqemu:\n%s" % (cmdline, out)) self.stop_thread() return False self.logger.debug("Output from runqemu:\n%s", out) self.logger.debug("Waiting at most %d seconds for login banner (%s)" % (self.boottime, time.strftime("%D %H:%M:%S"))) endtime = time.time() + self.boottime socklist = [self.server_socket] reachedlogin = False stopread = False qemusock = None bootlog = b'' data = b'' while time.time() < endtime and not stopread: try: sread, swrite, serror = select.select(socklist, [], [], 5) except InterruptedError: continue for sock in sread: if sock is self.server_socket: qemusock, addr = self.server_socket.accept() qemusock.setblocking(0) socklist.append(qemusock) socklist.remove(self.server_socket) self.logger.debug("Connection from %s:%s" % addr) else: data = data + sock.recv(1024) if data: bootlog += data data = b'' if b' login:'******'t reach login banner in %d seconds (%s)" % (self.boottime, time.strftime("%D %H:%M:%S"))) tail = lambda l: "\n".join(l.splitlines()[-25:]) # in case bootlog is empty, use tail qemu log store at self.msg lines = tail(bootlog if bootlog else self.msg) self.logger.debug("Last 25 lines of text:\n%s" % lines) self.logger.debug("Check full boot log: %s" % self.logfile) self._dump_host() self.stop() return False # If we are not able to login the tests can continue try: (status, output) = self.run_serial("root\n", raw=True) if re.search("root@[a-zA-Z0-9\-]+:~#", output): self.logged = True self.logger.debug("Logged as root in serial console") if netconf: # configure guest networking cmd = "ifconfig eth0 %s netmask %s up\n" % (self.ip, self.netmask) output = self.run_serial(cmd, raw=True)[1] if re.search("root@[a-zA-Z0-9\-]+:~#", output): self.logger.debug("configured ip address %s", self.ip) else: self.logger.debug( "Couldn't configure guest networking") else: self.logger.debug("Couldn't login into serial console" " as root using blank password") except: self.logger.debug("Serial console failed while trying to login") return True
def pipe(self): return pipe()
if replication.repl_limit != 0: limit = '/usr/local/bin/throttle -K %d | ' % replication.repl_limit else: limit = '' cmd = ['/sbin/zfs', 'send', '-V'] if replication.repl_userepl: cmd.append('-R') if last_snapshot == '': cmd.append(snapname) else: cmd.extend(['-I', last_snapshot, snapname]) progressfile = '/tmp/.repl_progress_%d' % replication.id # subprocess.Popen does not handle large stream of data between # processes very well, do it on our own readfd, writefd = os.pipe() zproc_pid = os.fork() if zproc_pid == 0: os.close(readfd) os.dup2(writefd, 1) os.close(writefd) os.execv('/sbin/zfs', cmd) # NOTREACHED else: with open(progressfile, 'w') as f2: f2.write(str(zproc_pid)) os.close(writefd) if compression == 'pigz': compress = '/usr/local/bin/pigz | ' decompress = '/usr/local/bin/pigz -d | '
# so need link it to current position # DEBUG = False DEBUG = True # stage 1: argv if not DEBUG: args = ['/home/input2/input'] + ['A']*99 else: args = ['./input'] + ['A']*99 args[ord('A')] = "" args[ord('B')] = "\x20\x0a\x0d" args[ord('C')] = "12345" # stage 2: stdio stderr, stderw = os.pipe() # stage 3: env env = os.environ.copy() env["\xde\xad\xbe\xef"] = "\xca\xfe\xba\xbe" # print env['PATH'] # stage 4: file f = open("\x0a", "wb") f.write("\x00\x00\x00\x00") f.close() # stage 5: network s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) proc = subprocess.Popen(args, stdin=subprocess.PIPE, stderr=stderr, env=env)
def pipe(self): r, w = pipe() os.make_nonblocking(r) os.make_nonblocking(w) return r, w
def handle1(fd): while True: buf = os.read(fd, 1024) if buf: print 'GET:%s' % buf def handle2(fd): while True: buf = os.read(0, 1024) if buf: print 'SEND:%s' % buf os.write(fd, buf) p2cr, p2cw = os.pipe() c2pr, c2pw = os.pipe() pid = os.fork() if pid == 0: # child os.close(p2cw) os.close(c2pr) os.dup2(p2cr, 0) os.dup2(c2pw, 1) os.execvp('python', ['python']) os._exit(0) else: os.close(p2cr) os.close(c2pw) t = Thread(target=handle1, args=(c2pr, )) t.setDaemon(True) t.start()
def __enter__(self): import os self.read_fd, self.write_fd = os.pipe() self.copy_fd = os.dup(self.capture_fd) os.dup2(self.write_fd, self.capture_fd) return self
def sndrcv(pks, pkt, timeout=None, inter=0, verbose=None, chainCC=0, retry=0, multi=0): if not isinstance(pkt, Gen): pkt = SetGen(pkt) if verbose is None: verbose = conf.verb debug.recv = plist.PacketList([], "Unanswered") debug.sent = plist.PacketList([], "Sent") debug.match = plist.SndRcvList([]) nbrecv = 0 ans = [] # do it here to fix random fields, so that parent and child have the same all_stimuli = tobesent = [p for p in pkt] notans = len(tobesent) hsent = {} for i in tobesent: h = i.hashret() if h in hsent: hsent[h].append(i) else: hsent[h] = [i] if retry < 0: retry = -retry autostop = retry else: autostop = 0 while retry >= 0: found = 0 if timeout < 0: timeout = None rdpipe, wrpipe = os.pipe() rdpipe = os.fdopen(rdpipe) wrpipe = os.fdopen(wrpipe, "w") pid = 1 try: pid = os.fork() if pid == 0: try: sys.stdin.close() rdpipe.close() try: i = 0 if verbose: print "Begin emission:" for p in tobesent: pks.send(p) i += 1 time.sleep(inter) if verbose: print "Finished to send %i packets." % i except SystemExit: pass except KeyboardInterrupt: pass except: log_runtime.exception("--- Error in child %i" % os.getpid()) log_runtime.info("--- Error in child %i" % os.getpid()) finally: try: os.setpgrp() # Chance process group to avoid ctrl-C sent_times = [ p.sent_time for p in all_stimuli if p.sent_time ] cPickle.dump((conf.netcache, sent_times), wrpipe) wrpipe.close() except: pass elif pid < 0: log_runtime.error("fork error") else: wrpipe.close() stoptime = 0 remaintime = None inmask = [rdpipe, pks] try: try: while 1: if stoptime: remaintime = stoptime - time.time() if remaintime <= 0: break r = None if not isinstance(pks, StreamSocket) and ( arch.FREEBSD or arch.DARWIN): inp, out, err = select(inmask, [], [], 0.05) if len(inp) == 0 or pks in inp: r = pks.nonblock_recv() else: inp = [] try: inp, out, err = select( inmask, [], [], remaintime) except IOError, exc: if exc.errno != errno.EINTR: raise if len(inp) == 0: break if pks in inp: r = pks.recv(MTU) if rdpipe in inp: if timeout: stoptime = time.time() + timeout del (inmask[inmask.index(rdpipe)]) if r is None: continue ok = 0 h = r.hashret() if h in hsent: hlst = hsent[h] for i, sentpkt in enumerate(hlst): if r.answers(sentpkt): ans.append((sentpkt, r)) if verbose > 1: os.write(1, "*") ok = 1 if not multi: del hlst[i] notans -= 1 else: if not hasattr( sentpkt, '_answered'): notans -= 1 sentpkt._answered = 1 break if notans == 0 and not multi: break if not ok: if verbose > 1: os.write(1, ".") nbrecv += 1 if conf.debug_match: debug.recv.append(r) except KeyboardInterrupt: if chainCC: raise finally: try: nc, sent_times = cPickle.load(rdpipe) except EOFError: warning( "Child died unexpectedly. Packets may have not been sent %i" % os.getpid()) else: conf.netcache.update(nc) for p, t in zip(all_stimuli, sent_times): p.sent_time = t os.waitpid(pid, 0) finally: if pid == 0: os._exit(0) remain = list(itertools.chain(*hsent.itervalues())) if multi: remain = [p for p in remain if not hasattr(p, '_answered')] if autostop and len(remain) > 0 and len(remain) != len(tobesent): retry = autostop tobesent = remain if len(tobesent) == 0: break retry -= 1 if conf.debug_match: debug.sent = plist.PacketList(remain[:], "Sent") debug.match = plist.SndRcvList(ans[:]) #clean the ans list to delete the field _answered if (multi): for s, r in ans: if hasattr(s, '_answered'): del (s._answered) if verbose: print "\nReceived %i packets, got %i answers, remaining %i packets" % ( nbrecv + len(ans), len(ans), notans) return plist.SndRcvList(ans), plist.PacketList(remain, "Unanswered")
def webm2mp4_worker(message, url): """Generic process spawned every time user sends a link or a file""" global telegram_token filename = "".join([TEMP_FOLDER, random_string(), ".mp4"]) # Tell user that we are working status_message = bot.reply_to(message, message_starting, parse_mode="HTML") # Try to download URL try: r = requests.get(url, stream=True, headers=HEADERS) except: update_status_message(status_message, error_downloading) return # Something went wrong on the server side if r.status_code != 200: update_status_message(status_message, error_wrong_code.format(r.status_code)) return # Is it a webm file? if r.headers["Content-Type"] not in ALLOWED_MIME_TYPES and message.document.mime_type not in ALLOWED_MIME_TYPES: update_status_message(status_message, error_file_not_webm) return # Can't determine file size if not "Content-Length" in r.headers or not "Content-Type" in r.headers: update_status_message(status_message, error_no_header) return # Check file size webm_size = int(r.headers["Content-Length"]) if webm_size >= MAXIMUM_FILESIZE_ALLOWED: update_status_message(status_message, error_huge_file) return # Create a pipe to pass downloading file to ffmpeg without delays pipe_read, pipe_write = os.pipe() # Start ffmpeg ffmpeg_process = subprocess.Popen(["ffmpeg", "-v", "error", "-threads", str(FFMPEG_THREADS), "-i", "pipe:0", # read input from stdin "-map", "V:0?", # select video stream "-map", "0:a?", # ignore audio if doesn't exist "-c:v", "libx264", # specify video encoder "-max_muxing_queue_size", "9999", # https://trac.ffmpeg.org/ticket/6375 "-movflags", "+faststart", # optimize for streaming "-preset", "slow", # https://trac.ffmpeg.org/wiki/Encode/H.264#a2.Chooseapresetandtune "-timelimit", "900", # prevent DoS (exit after 15 min) "-vf", "pad=ceil(iw/2)*2:ceil(ih/2)*2", # https://stackoverflow.com/questions/20847674/ffmpeg-libx264-height-not-divisible-by-2#20848224 filename ], stdin=pipe_read) # Download file in and pass it to ffmpeg with pipe try: threading.Thread( target=download_file, kwargs={ "request": r, "pipe_write": pipe_write } ).start() # Initial delay to start downloading time.sleep(1) except: update_status_message(status_message, error_downloading) # Close pipe explicitly os.close(pipe_read) return # While ffmpeg process is alive (i.e. is working) old_progress = "" while ffmpeg_process.poll() == None: try: output_file_size = os.stat(filename).st_size except FileNotFoundError: output_file_size = 0 mp4_size = size(output_file_size, system=alternative) webm_size = size(int(r.headers["Content-Length"]), system=alternative) human_readable_progress = " ".join([mp4_size, "/", webm_size]) if human_readable_progress != old_progress: update_status_message(status_message, message_converting.format(human_readable_progress)) old_prpgress = human_readable_progress time.sleep(3) # Exit in case of error with ffmpeg if ffmpeg_process.returncode != 0: update_status_message(status_message, error_converting) # Clean up and close pipe explicitly rm(filename) os.close(pipe_read) return # Check output file size mp4_size = os.path.getsize(filename) if mp4_size >= MAXIMUM_FILESIZE_ALLOWED: update_status_message(status_message, error_huge_file) # Clean up and close pipe explicitly rm(filename) os.close(pipe_read) return # Close pipe after using os.close(pipe_read) # 1. Get video duration in seconds video_duration = subprocess.run(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", "format=duration", "-of", "default=noprint_wrappers=1:nokey=1", filename ], stdout=subprocess.PIPE).stdout.decode("utf-8").strip() video_duration = round(float(video_duration)) # 2. Get video height and width video_props = subprocess.run(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries","stream=width,height", "-of", "csv=s=x:p=0", filename ], stdout=subprocess.PIPE).stdout.decode("utf-8").strip() video_width, video_height = video_props.split("x") # 3. Take one frame from the middle of the video update_status_message(status_message, message_generating_thumbnail) thumbnail = "".join([TEMP_FOLDER, random_string(), ".jpg"]) generate_thumbnail_process = subprocess.Popen(["ffmpeg", "-v", "error", "-i", filename, "-vcodec", "mjpeg", "-vframes", "1", "-an", "-f", "rawvideo", "-ss", str(int(video_duration/2)), # keep the limit of 90px height/width (Telegram API) while preserving the aspect ratio "-vf", "scale='if(gt(iw,ih),90,trunc(oh*a/2)*2)':'if(gt(iw,ih),trunc(ow/a/2)*2,90)'", thumbnail ]) # While process is alive (i.e. is working) while generate_thumbnail_process.poll() == None: time.sleep(1) # Exit in case of error with ffmpeg if generate_thumbnail_process.returncode != 0: update_status_message(status_message, error_generating_thumbnail) # Clean up rm(filename) rm(thumbnail) return # Upload to Telegram update_status_message(status_message, message_uploading) mp4 = open(filename, "rb") thumb = open(thumbnail, "rb") requests.post( f"https://api.telegram.org/bot{telegram_token}/sendVideo", data={ "chat_id": message.chat.id, "duration": video_duration, "width": video_width, "height": video_height, "reply_to_message_id": message.message_id, "supports_streaming": True }, files=[ ("video", (random_string()+".mp4", mp4, "video/mp4")), ("thumb", (random_string()+".jpg", thumb, "image/jpeg")) ] ) bot.delete_message(message.chat.id, status_message.message_id) # Clean up mp4.close() thumb.close() rm(filename) rm(thumbnail)
from tempfile import gettempdir from threading import Thread from lib.common.constants import PATHS from lib.common.exceptions import CuckooError from lib.core.config import Config from lib.core.startup import create_folders, init_logging from lib.api.tracer import SyscallTracer, FilesystemTracer from time import sleep log = logging.getLogger() FILES_LIST = [] DUMPED_LIST = [] PIPE = os.pipe() def dump_file(): """Create a copy of the given file path.""" log.info("PLS IMPLEMENT DUMP, want to dump %s", file_path) pass def dump_files(): """Dump all the dropped files.""" for file_path in FILES_LIST: dump_file(file_path) class PipeServer(Thread):
def main(cscreen = None): ###################################################### ## STARTUP ## ###################################################### #default values arg_filter = None arg_address = None input_path = "NREL_sequence_canadian_1.csv" file_mode = False output_path = "SSTDR_waveforms.csv" yaml_path = 'default.yaml' #read cmd line arguments valid_args = ['-yaml', 'y', '-filter', '-f', '-address', '-a', '-file', '-out', '-o', '-curses', '-c', '-no-curses', '-nc'] args = {} skip = False for i,arg in enumerate(sys.argv): if skip: skip = False continue #only look at args in loop if arg in valid_args: skip = True #skip next word; we use it as a value here if (i+1 < len(sys.argv)): value = sys.argv[i+1] if arg in ['-yaml', '-y']: yaml_path = value elif arg in ['-filter', '-f']: arg_filter = int(value) elif arg in ['-address', '-a']: arg_address = int(value) elif arg in ['-file']: file_mode = True input_path = value elif arg in ['-out', '-o']: output_path = value #elif arg in ['-curses', '-c']: # USE_CURSES = True # skip = False #elif arg in ['-no-curses', '-nc']: # USE_CURSES = False # skip = False #prepare usb sniffing if (arg_filter is None or arg_address is None): sstdr_device = usb.core.find(idVendor=0x067b, idProduct=0x2303) #constants for our SSTDR device if sstdr_device == None: print("Error: Could not automatically find SSTDR device. Either restart it or provide filter/address manually.") return arg_filter = sstdr_device.bus arg_address = sstdr_device.address usb_path = "C:\\Program Files\\USBPcap\\USBPcapCMD.exe" usb_args = [usb_path, "-d", "\\\\.\\USBPcap" + str(arg_filter), "--devices", str(arg_address), "-o", "-"] #prepare output file for logging with open(output_path, "a+") as out_f: out_f.seek(0,0) first_char = out_f.read(1) if (first_char == ''): #file did not exist or is empty. write header row; set session/log index to 0 #write header row out_f.write("session_number,log_number,timestamp,waveform\n") session_number = 0 log_number = 0 else: #file was not empty. jump almost to end, read last line, extract session index #"read up until start of last line" code from S.O. user Trasp: https://stackoverflow.com/questions/3346430/what-is-the-most-efficient-way-to-get-first-and-last-line-of-a-text-file/3346788 with open(output_path, "rb") as f: f.seek(-2, os.SEEK_END) # Jump to the second last byte. while f.read(1) != b"\n": # Until EOL is found... f.seek(-2, os.SEEK_CUR) # ...jump back the read byte plus one more. last = f.readline() # Read last line as bytes. session_number = 1+int(chr(int.from_bytes(last.split(b',')[0],'little'))) #assumes little endian, and that session index is present in column 0 (as will be standard in the future) log_number = 0 #set up scanning interface in curses (cscreen = curses screen) print("Opening scanner interface...") if not(cscreen is None): cscreen.clear() cscreen.nodelay(True) if (file_mode): cscreen.addstr(0,0,"Playing back input file: '" + input_path +"'...") else: cscreen.addstr(0,0,"Scanning on filter " + str(arg_filter) + ", address " + str(arg_address) + "...") cscreen.addstr(1,0,"Press 'q' to stop.") cscreen.addstr(3,0,"System OK.") cscreen.refresh() else: print("Scanning on filter " + str(arg_filter) + ", address " + str(arg_address) + "...") if (not file_mode): #open USBPcap, throwing all output onto a pipe usb_fd_r, usb_fd_w = os.pipe() usbpcap_process = subprocess.Popen(usb_args, stdout=usb_fd_w) #start receiving usbpcap output and organizing it into packets usb_stream = os.fdopen(usb_fd_r, "rb") #set up receiver to process raw USB bytestream halt_threads = threading.Event() receiver = PcapPacketReceiver(usb_stream, loop=True, halt_event=halt_threads) #prepare deque for waveform visualization; only stores a few of the most recently received waveforms. appended entries cycle out old ones #larger deque -> more maximum latency between visualization and actual system state #smaller deque -> not sure why this would be a problem (something about losing information if packets aren't received constantly) wf_deque = deque(maxlen=1) #prepare to visualize waveforms fig = plt.figure() plot_window = pf.screen(title='SSTDR Correlation Waveform') ###################################################### ## PYGAME SETUP ## ###################################################### #initializing pygame pygame.init() pscreen = pygame.display.set_mode(SCREEN_SIZE) pygame.display.set_caption("PV Fault Scanner") #loading assets, preparing pre-baked surfaces TERMINAL_FONT = pygame.font.Font(None, 40) STATUS_FONT = pygame.font.Font(None, 20) panel_surf = pygame.image.load(os.path.join("Assets", "PV_panel_CharlesMJames_CC.jpg")) panel_surf = pygame.transform.scale(panel_surf, (int(panel_surf.get_width()*PANEL_SCALE), int(panel_surf.get_height()*PANEL_SCALE))) panel_rect = panel_surf.get_rect() grass_surf = pygame.image.load(os.path.join("Assets", "grass.png")) grass_rect = grass_surf.get_rect() hazard_surf = pygame.image.load(os.path.join("Assets", "hazard.png")) hazard_rect = hazard_surf.get_rect() bg_surf = pygame.Surface(pscreen.get_size()) bg_surf.convert() bg_rect = bg_surf.get_rect() bg_surf.fill(BG_COLOR) """ for r in range(int(SCREEN_Y / grass_rect.h+1)): for c in range(int(SCREEN_X / grass_rect.w+1)): bg_surf.blit(grass_surf, grass_rect) grass_rect.move_ip(grass_rect.w,0) grass_rect.x = 0 grass_rect.move_ip(0, grass_rect.h) """ line_surf = pygame.Surface((SCREEN_X, BORDER_WIDTH)) line_surf.fill(COLOR_ORANGE) line_rect = line_surf.get_rect() line_rect.y = VISUAL_Y - BORDER_WIDTH - int(BORDER_PADDING/2) bg_surf.blit(line_surf, line_rect) line_surf.fill(COLOR_BLUE) line_rect.move_ip(0, BORDER_WIDTH + BORDER_PADDING) bg_surf.blit(line_surf, line_rect) text_surf = STATUS_FONT.render("Scanning at 24MHz...", True, COLOR_WHITE) text_rect = text_surf.get_rect() text_rect.move_ip(3,3) bg_surf.blit(text_surf, text_rect) text_surf = STATUS_FONT.render("Selected Array Layout: " + yaml_path, True, COLOR_WHITE) text_rect = text_surf.get_rect() text_rect.x = 3 text_rect.bottom = VISUAL_Y - BORDER_WIDTH - int(0.5*BORDER_PADDING) - 3 bg_surf.blit(text_surf, text_rect) #load panel layout panel_layout, panel_ds, panel_length = load_panel_layout(yaml_path) panel_cols = panel_rows = 0 try: N = len(panel_ds) H = int(N/2)-1 if (panel_layout['layout'] == 'loop'): panel_rows = 2 panel_cols = int(N/2+0.5) r = 0 PANEL_COORDS = [(c*(PANEL_PADDING[0] + panel_rect.w), r*(PANEL_PADDING[1] + panel_rect.h)) for c in range(0,H+1)] if (len(panel_ds)%2): r = 0.5 PANEL_COORDS.append(((H+1)*(PANEL_PADDING[0] + panel_rect.w), r*(PANEL_PADDING[1] + panel_rect.h))) r = 1 PANEL_COORDS = PANEL_COORDS + [(c*(PANEL_PADDING[0] + panel_rect.w), r*(PANEL_PADDING[1] + panel_rect.h)) for c in range(H,-1,-1)] elif(panel_layout['layout'] == 'home-run'): panel_rows = 1 panel_cols = N r = 0 PANEL_COORDS = [(c*(PANEL_PADDING[0] + panel_rect.w), r*(PANEL_PADDING[1] + panel_rect.h)) for c in range(0,N)] else: raise Exception("Error: unknown layout field in layout yaml file.") except: print("Error: invalid layout yaml file.") return ARRAY_SIZE = (panel_cols*(panel_rect.w + PANEL_PADDING[0]), panel_rows*(panel_rect.h + PANEL_PADDING[1])) array_surf = pygame.Surface(ARRAY_SIZE, pygame.SRCALPHA) array_surf.convert() for p in PANEL_COORDS: panel_rect.topleft = p array_surf.blit(panel_surf, panel_rect) array_rect = array_surf.get_rect() array_rect.center = (int(SCREEN_X*PANEL_SCREEN_X_RATIO), int(VISUAL_Y/2)) WIRE_COORDS = [] for p in PANEL_COORDS: panel_rect.topleft = p WIRE_COORDS.append((panel_rect.center[0] + array_rect.topleft[0], panel_rect.center[1] + array_rect.topleft[1])) WIRE_COORDS.insert(0,(0,WIRE_COORDS[0][1])) WIRE_COORDS.append((0,WIRE_COORDS[-1][1])) pygame.draw.lines(bg_surf, WIRE_COLOR, False, WIRE_COORDS, WIRE_WIDTH) term_surf = pygame.Surface((SCREEN_X, TERMINAL_Y - int(BORDER_PADDING/2) - BORDER_WIDTH)) term_surf.fill(TERMINAL_COLOR) term_rect = term_surf.get_rect() term_rect.bottom = SCREEN_Y button_text_p = 5 #padding button_outer_p = 5 button_size = list(STATUS_FONT.size("Measure")) button_size[0] += 2*button_text_p button_size[1] += 2*button_text_p button_rect = pygame.Rect(SCREEN_X-button_size[0]-button_outer_p, button_outer_p, button_size[0], button_size[1]) button_surf = pygame.Surface(button_size) button_text_surf = STATUS_FONT.render("Measure", True, COLOR_WHITE) ###################################################### ## FAULT DETECTION SETUP ## ###################################################### detector = fault_detection.Detector(fault_detection.METHOD_NONE) fault = (fault_detection.FAULT_NONE, 0) terminal_waveform = None logging = False #if this is true, measured waveforms will be written to a file measurement_counter = 0 first_timestamp = None first_time_played = None input_row_index = 0 if file_mode: input_data = fault_detection.read_csv_ungrouped(input_path) ###################################################### ## LOOP ## ###################################################### #set up threads: #first child thread: receives and interprets packets using receiver.run() with ThreadPoolExecutor(max_workers=3) as executor: if not file_mode: rec_thread = executor.submit(receiver.run) valid_waveform_prefix = b'\xaa\xaa\xaa\xad\x00\xbf' #valid waveform regions start with this pattern valid_waveform_suffix = 253 #valid waveform regions end with this pattern payloadString = b'' byteCount = 0 WAVEFORM_BYTE_COUNT = 199 #every waveform region contains 199 payload bytes try: while(True): #take packet from Q, process in some way if file_mode: #TODO change this to whatever the baseline index ought to be if input_row_index == 12: detector.set_baseline(input_data[input_row_index][3:]) if input_row_index == 0: first_time_played = time.time() first_timestamp = input_data[0][2] if True:#time.time() - first_time_played >= input_data[input_row_index+1][2] - first_timestamp: input_row_index = input_row_index + 1 wf_deque.append(np.array(input_data[input_row_index][3:])) time.sleep(0.25) """ goal is to identify shape of data in intermittent test, and have this code recognize when a sequence of packet blocks represents a correlation waveform. This waveform should be calculated from payload bytes, and either shown for visualization (pyplot?) or fed to matlab for processing (which is the ultimate goal). """ if not file_mode and receiver.q.empty() == False: pBlock = receiver.q.get() #commented out because this printing was very very slow, and ruined realtime #if not(cscreen is None): #cscreen.addstr(5,0,"Received packet at timestamp: " + str(pBlock.ts_sec + 0.000001*pBlock.ts_usec)) #show some packet data so it's clear the scanner is working #cscreen.refresh() #if received packet may be in a waveform region of the stream: #criteria: input (to host) from endpoint 3 and function == URB_FUNCTION_BULK_OR_INTERRUPT_TRANSFER if (pBlock.packet.endpoint == 0x83 and pBlock.packet.function == 0x09): #if block has a payload: p = pBlock.packet.payload l = len(p) if (l > 0): payloadString = payloadString + p byteCount = byteCount + l if ((l==1 and p==0xaa) or (l>1 and p[0] == 0xaa) or (b'\xaa' in p)): if not cscreen is None and DEBUG_VERIFICATION: cscreen.addstr(13,0,"Received start of prefix at timestamp: " + str(pBlock.ts_sec + 0.000001*pBlock.ts_usec)) cscreen.addstr(14,4,"Starting payload: " + str(p)) cscreen.addstr(15,4,"New payload string: " + str(payloadString)) cscreen.refresh() #for the first few bytes, compare it to a prefix pattern that all valid waveforms start with. if (VERIFY_WAVEFORMS and byteCount <= len(valid_waveform_prefix)): #TODO: currently assumes prefix bytes arrive one at a time (ie not in the same payload). this is OK the vast majority of the time. if (p[0] != valid_waveform_prefix[byteCount-1]): #the current waveform is not valid. throw it out. if not(cscreen is None): cscreen.addstr(10,0,"Invalid waveform at timestamp: " + str(pBlock.ts_sec + 0.000001*pBlock.ts_usec)) cscreen.addstr(11,4,"Prefix string: " + str(payloadString) + "; Byte count: " + str(byteCount)) if (byteCount > 1): cscreen.addstr(12,4,"Last long invalid prefix: "+ str(payloadString) + "; Byte count: " + str(byteCount)) cscreen.refresh() else: print("Invalid waveform at timestamp: " + str(pBlock.ts_sec + 0.000001*pBlock.ts_usec)) print("Prefix string: " + str(payloadString)+'\n') if ((l==1 and p==0xaa) or (l>1 and p[0] == 0xaa) or (b'\xaa' in p)): #if the byte that ruined everything may be the start of a valid region, keep it payloadString = b'' + p byteCount = l else: payloadString = b'' byteCount = 0 elif (byteCount >= WAVEFORM_BYTE_COUNT): #perform processing on raw waveform wf = process_waveform_region(payloadString,cscreen) wf_deque.append(wf) if not(cscreen is None): #show that we've received a waveform cscreen.addstr(7,0,"Received waveform at timestamp: " + str(pBlock.ts_sec + 0.000001*pBlock.ts_usec)) cscreen.refresh() #prepare to receive next waveform region payloadString = b'' byteCount = 0 elif (byteCount > 0): payloadString = b'' byteCount = 0 elif len(wf_deque) > 0: #q was empty, we have some extra time to visualize things wf = np.array(wf_deque.popleft()) if (logging): #write row with session index, log index, timestamp, and measured waveform. with open("SSTDR_waveforms.csv", "a") as f: f.write(str(session_number)+","+str(log_number)+","+str(pBlock.ts_sec + 0.000001*pBlock.ts_usec)+","+str(list(wf))[1:-1]+'\n') if measurement_counter > 0: measurement_counter -= 1 if measurement_counter == 0: logging = False log_number += 1 ################################################################################################################################### # PYFORMULAS: visualize waveform ################################################################################################################################### #some code from https://stackoverflow.com/questions/40126176/fast-live-plotting-in-matplotlib-pyplot plt.clf() plt.xlabel("Distance (feet)") plt.ylabel("Correlation With Reflection") plt.gcf().subplots_adjust(left=0.15) if detector.raw_baseline is None: #plt.plot(fault_detection.FEET_VECTOR, wf_i/max(abs(wf_i))) plt.plot(fault_detection.SPLINE_FEET_VECTOR-detector.spline_feet_offset, detector.last_processed_waveform) #plt.ylim((-1,1)) plt.ylim((-(2**15), 2**15)) plt.xlim((fault_detection.SPLINE_FEET_VECTOR[0]-detector.spline_feet_offset, fault_detection.SPLINE_FEET_VECTOR[-1]-detector.spline_feet_offset)) else: #plot BLS bls = detector.last_processed_waveform - detector.processed_baseline max_f = fault_detection.SPLINE_FEET_VECTOR[np.argmax(bls)]-detector.spline_feet_offset plt.plot(fault_detection.SPLINE_FEET_VECTOR-detector.spline_feet_offset, bls) plt.plot([max_f, max_f], [-750, 750]) #plt.ylim((-1,1)) plt.ylim((-(750), 750)) plt.xlim((fault_detection.SPLINE_FEET_VECTOR[0]-detector.spline_feet_offset, fault_detection.SPLINE_FEET_VECTOR[-1]-detector.spline_feet_offset)) fig.canvas.draw() image = np.frombuffer(fig.canvas.tostring_rgb(), dtype=np.uint8) image = image.reshape(fig.canvas.get_width_height()[::-1] + (3,)) plot_window.update(image) ################################################################################################################################### # PYGAME: fault visualization & event queue ################################################################################################################################### fault = detector.detect_faults(wf) for event in pygame.event.get(): if event.type == pygame.QUIT or (event.type == pygame.KEYDOWN and event.key == pygame.K_ESCAPE): pygame.display.quit() pygame.quit() if event.type == pygame.MOUSEBUTTONUP: if (button_rect.collidepoint(pygame.mouse.get_pos())): #log for 10 samples. "window capture" logging = True measurement_counter = 10 #counts down to zero if event.type == pygame.KEYDOWN: if event.key == pygame.K_b: detector.set_baseline(wf)#set baseline elif event.key == pygame.K_l: #START/STOP LOGGING. if logging: #stop logging; increment log index. log_number = log_number+1 measurement_counter = 0 logging = False else: #start logging logging = True elif event.key == pygame.K_a: terminal_waveform = wf #record waveform representing a disconnect at the panel terminal elif event.key == pygame.K_t: if (terminal_waveform is None): terminal_waveform = wf detector.set_terminal(terminal_waveform)#set terminal points based on recorded terminal waveform and current BLSDT elif event.key == pygame.K_LEFT: detector.bls_deviation_thresh = detector.bls_deviation_thresh - 0.01 #adjust deviation threshold for peak location elif event.key == pygame.K_RIGHT: detector.bls_deviation_thresh = detector.bls_deviation_thresh + 0.01 elif event.key == pygame.K_w: #log for 10 samples. "window capture" logging = True measurement_counter = 10 #counts down to zero #per-frame logic here is_fault = (fault[0] != fault_detection.FAULT_NONE) fault_d_f = fault[1] if (is_fault): d = 0 px = WIRE_COORDS[0][0] py = WIRE_COORDS[0][1] hazard_point = WIRE_COORDS[-1] #determine which panel the fault is AFTER for i in range(len(panel_ds)): if (panel_ds[i] > fault_d_f): break #get the distance from the SSTDR positive lead to the pre-fault point ('point' being a point in WIRE_COORDS) if i == 0: pre_d = 0 else: pre_d = panel_ds[i-1] #get the distance from the SSTDR positive lead to the post-fault point if i == len(panel_ds): post_d = panel_ds[-1] + panel_layout['home_cable_length'] #point in feet at final SSTDR terminal else: post_d = panel_ds[i] #get PIXEL locations of pre-fault and post-fault points, then calculate PIXEL location of fault point pre_x, pre_y = WIRE_COORDS[i] #WIRE COORDS has an extra point at i=0 (where x=0), so this chooses the point of the panel/terminal BEFORE the fault post_x, post_y = WIRE_COORDS[i+1] #certainly safe; WIRE_COORDS has two more points than PANEL_COORDS. chooses the point AFTER the fault hsr = (fault_d_f - pre_d)/(post_d - pre_d) #hazard step ratio: ratio at which the fault lies in between post point and pre point, s.t. fault_d = pre_d + hsr*(post_d - pre_d) step = ((post_x-pre_x)**2 + (post_y-pre_y)**2)**0.5 #distance IN PIXELS between post and pre points hazard_rect.center = (pre_x + hsr*(post_x-pre_x), pre_y + hsr*(post_y-pre_y)) #subtract from distance to account for panel length; only want to report cable length fault_cable_location = fault_d_f - panel_length*i fault_name = fault_detection.get_fault_name(fault[0]) fault_text_surf = TERMINAL_FONT.render(fault_name + " located at " + str(round(fault_cable_location,3)) + " feet", True, TEXT_COLOR) else: fault_text_surf = TERMINAL_FONT.render("System OK", True, TEXT_COLOR) fault_text_rect = fault_text_surf.get_rect() fault_text_rect.center = term_rect.center #param_text_surf = STATUS_FONT.render("BLS deviation threshold:" + str(detector.bls_deviation_thresh), True, COLOR_WHITE) #param_text_surf = STATUS_FONT.render("LPF Cutoff Frequency: 6 MHz", True, COLOR_WHITE) #TODO don't hard code this, allow for live control of cutoff frequency param_text_surf = STATUS_FONT.render("Current Log Number: "+str(log_number), True, COLOR_WHITE) param_text_rect = param_text_surf.get_rect() param_text_rect.bottomright = (SCREEN_X-3, VISUAL_Y - BORDER_WIDTH - int(0.5*BORDER_PADDING) - 3) logging_string = "Logging to 'SSTDR_waveforms.csv'..." if logging else "Not logging." logging_text_surf = STATUS_FONT.render(logging_string, True, COLOR_WHITE) logging_text_rect = logging_text_surf.get_rect() logging_text_rect.bottomright = param_text_rect.topright #buttons: fill with color depending on context #for now just fill orange if (button_rect.collidepoint(pygame.mouse.get_pos())): button_surf.fill(COLOR_ORANGE) else: button_surf.fill(COLOR_BLUE) button_surf.blit(button_text_surf,(button_text_p,button_text_p)) #drawing pscreen.blit(bg_surf, bg_rect) pscreen.blit(term_surf, term_rect) pscreen.blit(fault_text_surf, fault_text_rect) pscreen.blit(param_text_surf, param_text_rect) pscreen.blit(logging_text_surf, logging_text_rect) pscreen.blit(array_surf, array_rect) pscreen.blit(button_surf, button_rect) if (is_fault): pscreen.blit(hazard_surf, hazard_rect) pygame.display.flip() ################################################################################################################################### # CURSES: Check for quit ################################################################################################################################### if not(cscreen is None): c = cscreen.getch() if (c == ord('q')): cscreen.addstr(0,0,"Quitting: Terminating scanner...") cscreen.refresh() usbpcap_process.terminate() cscreen.addstr(0,0, "Stopped scanner. Waiting for threads...") cscreen.refresh() receiver.halt() #while(rec_thread.running()): # pass usb_stream.close() #executor.shutdown() #performed implicitly by "with" statement cscreen.addstr(0,0, "Finished. Exiting...") break except: print("Exception Occurred:") print('='*40) traceback.print_exc(file=sys.stdout) print('='*40) print("All done. :)")
def DebugPipe(): fd1, fd2 = os.pipe() c1 = DebugConnection(fd1, writable=False) c2 = DebugConnection(fd2, readable=False) return c1, c2
def __init__(self): self._rfd, self._wfd = os.pipe() self._set = False self._forever = False self._closed = False
def _pipe_cloexec() -> Tuple[int, int]: r, w = os.pipe() set_close_exec(r) set_close_exec(w) return r, w
def cleanup(fcn=None, *args, **kwargs): # We setup a safe procedure here to ensure that the # child will receive a SIGTERM when the parent exits. # This is used to cleanup all sorts of things once the # VM finishes automatically (like removing file trees, # removing tap devices, kill dnsmasq, etc.). libc = ctypes.CDLL("libc.so.6") if fcn is None: # This will only exit when the parent dies, # we will not run any function. This will be # used generally as the subprocess preexec_fn. child_pid = 0 parent_pid = os.getppid() else: # Open a pipe to notify the parent when we're # ready to handle parent death and run the fcn. rpipe, wpipe = os.pipe() # Fork a child process. # This child process will execute the given code # when its parent dies. It's normally used inline. child_pid = os.fork() parent_pid = os.getppid() if child_pid == 0: os.close(rpipe) else: # Wait for the child to finish setup. # When it writes to its end of the pipe, # we know that it is prepared to handle # parent death and run the function. os.close(wpipe) os.read(rpipe, 1) os.close(rpipe) if child_pid == 0: # Set P_SETSIGDEATH to SIGTERM. libc.prctl(1, signal.SIGTERM) # Did we catch a race above, where we've # missed the re-parenting to init? if os.getppid() != parent_pid: os.kill(os.getpid(), signal.SIGTERM) # Are we finished? # In the case of not having a function to # execute, we simply return control. This is # a pre-exec hook for subprocess, for eaxample. if fcn is None: return # Close descriptors. # (Make sure we don't close our pipe). for fd in range(3, os.sysconf("SC_OPEN_MAX")): try: if fd != wpipe: os.close(fd) except OSError: pass # Eat a signal. def squash(*args): pass # Suppress SIGINT. We'll get it when the user # hits Ctrl-C, which may be before our parent dies. signal.signal(signal.SIGINT, squash) def interrupt(*args): # Temporarily suppress SIGTERM. We'll enable it # once we are ready to wait (and recheck races). signal.signal(signal.SIGTERM, squash) thread.exit() # Temporarily supress SIGTERM, we do this until # it's re-enabled in the main wait loop below. signal.signal(signal.SIGTERM, squash) # Notify that we are ready to go. os.write(wpipe, 'o') os.close(wpipe) while True: try: # Get ready to receive our SIGTERM. # NOTE: When we receive the exception, # it will automatically be suppressed. signal.signal(signal.SIGTERM, interrupt) # Catch our race condition. if os.getppid() != parent_pid: break # Wait for a signal. signal.pause() except (SystemExit, KeyboardInterrupt): continue try: fcn(*args, **kwargs) except: # We eat all exceptions from the # cleanup function. If the user wants # to generate any output, they may -- # however by default we silence it. pass os._exit(0)
def TouchFileAsUser(self,user,file_dir,file_dir_orig): #file_dir_orig is needed to access the "Warn" or "Offline" information for the file/directory in question from the config file when variable substitution has taken place # Define the child var child = 0 user_data = None pbs.logmsg(pbs.EVENT_DEBUG3,"User name: %s\tFile dir: %s"%(user,file_dir)) try: #user_data = getpwnam(self.user) user_data = getpwnam(user) if file_dir.find('<user_home>') != -1: file_dir = file_dir.replace('<user_home>',user_data[5]) pbs.logmsg(pbs.EVENT_DEBUG3,"User name: %s\tDir to write to: %s"% \ (user_data[4],file_dir)) #This is a special case where the user account does not exist on the node. Offlining here is a good alternative to the job failing to run 20 times and being held, but it can be changed if desired except KeyError: pbs.logmsg(pbs.EVENT_DEBUG,"Unable to find user: %s"%user) # return ['Offline','unable to find user: %s'%user] # Fork the process for touching a file as the user r,w = os.pipe() pid = os.fork() pbs.logmsg(pbs.EVENT_DEBUG3,"pid: %d"%pid) if pid: # We are the parent os.close(w) r = os.fdopen(r) # turn r into a file object child = pid pbs.logmsg(pbs.EVENT_DEBUG3,"Ready to read from the child process: %d"%pid) lines = r.read() pbs.logmsg(pbs.EVENT_DEBUG3,lines) # Wait for the child process to complete os.waitpid(child,0) # Close the pipes r.close() # Check to see if the file was successfully touched if lines.find('Successfully touched file') == -1 or lines.find('Failed to remove file') != -1: pbs.logmsg(pbs.EVENT_DEBUG3,"Failed to touch/remove file in %s as %s"%(file_dir,user)) return [self.nhc_cfg["as_user_operations"]["touch_files"][file_dir_orig][1],'Failed to touch/remove file for %s in %s'%(user,file_dir)] else: pbs.logmsg(pbs.EVENT_DEBUG3,"Successfully touched and removed file for %s in %s"%(user,file_dir)) else: try: # Close the reading pipe os.close(r) # Turn w into a file object w = os.fdopen(w,'w') # Switch to the user w.write("Ready to switch to user: %s\tuid: %s\n"%(user,user_data[2])) os.setuid(user_data[2]) # Change to the user home dir w.write("Changing dir to: %s\n"%(file_dir)) if os.path.isdir(file_dir): os.chdir(file_dir) # Touch a file in the user's home directory touch_file_name = "__user_%s_jobid_%s_host_%s_pbs_test.txt"%(user,self.job_id,self.host) w.write("Ready to touch file: %s\n"%(touch_file_name)) touchFileSuccess = self.TouchFile(touch_file_name) if touchFileSuccess: w.write("Successfully touched file\n") try: os.remove(touch_file_name) except OSError: w.write("Failed to remove file: %s"%touch_file_name) except Exception, e: w.write("Remove file exception: %s\n"%(e)) else:
def __enter__(self): if self._active: raise RuntimeError("Can't re-enter the same log_output!") if self.file_like is None: raise RuntimeError( "file argument must be set by either __init__ or __call__") # set up a stream for the daemon to write to self.log_file = FileWrapper(self.file_like) # record parent color settings before redirecting. We do this # because color output depends on whether the *original* stdout # is a TTY. New stdout won't be a TTY so we force colorization. self._saved_color = tty.color._force_color forced_color = tty.color.get_color_when() # also record parent debug settings -- in case the logger is # forcing debug output. self._saved_debug = tty._debug # OS-level pipe for redirecting output to logger read_fd, write_fd = os.pipe() read_multiprocess_fd = MultiProcessFd(read_fd) # Multiprocessing pipe for communication back from the daemon # Currently only used to save echo value between uses self.parent_pipe, child_pipe = multiprocessing.Pipe() # Sets a daemon that writes to file what it reads from a pipe try: # need to pass this b/c multiprocessing closes stdin in child. input_multiprocess_fd = None try: if sys.stdin.isatty(): input_multiprocess_fd = MultiProcessFd( os.dup(sys.stdin.fileno())) except BaseException: # just don't forward input if this fails pass with replace_environment(self.env): self.process = multiprocessing.Process( target=_writer_daemon, args=(input_multiprocess_fd, read_multiprocess_fd, write_fd, self.echo, self.log_file, child_pipe, self.filter_fn)) self.process.daemon = True # must set before start() self.process.start() finally: if input_multiprocess_fd: input_multiprocess_fd.close() read_multiprocess_fd.close() # Flush immediately before redirecting so that anything buffered # goes to the original stream sys.stdout.flush() sys.stderr.flush() # Now do the actual output redirection. self.use_fds = _file_descriptors_work(sys.stdout, sys.stderr) if self.use_fds: # We try first to use OS-level file descriptors, as this # redirects output for subprocesses and system calls. # Save old stdout and stderr file descriptors self._saved_stdout = os.dup(sys.stdout.fileno()) self._saved_stderr = os.dup(sys.stderr.fileno()) # redirect to the pipe we created above os.dup2(write_fd, sys.stdout.fileno()) os.dup2(write_fd, sys.stderr.fileno()) os.close(write_fd) else: # Handle I/O the Python way. This won't redirect lower-level # output, but it's the best we can do, and the caller # shouldn't expect any better, since *they* have apparently # redirected I/O the Python way. # Save old stdout and stderr file objects self._saved_stdout = sys.stdout self._saved_stderr = sys.stderr # create a file object for the pipe; redirect to it. pipe_fd_out = os.fdopen(write_fd, 'w') sys.stdout = pipe_fd_out sys.stderr = pipe_fd_out # Unbuffer stdout and stderr at the Python level if not self.buffer: sys.stdout = Unbuffered(sys.stdout) sys.stderr = Unbuffered(sys.stderr) # Force color and debug settings now that we have redirected. tty.color.set_color_when(forced_color) tty._debug = self.debug # track whether we're currently inside this log_output self._active = True # return this log_output object so that the user can do things # like temporarily echo some output. return self
def setUpClass(cls): """ Perform class setup before running the testcase Remove shared memory files, start vpp and connect the vpp-api """ gc.collect() # run garbage collection first random.seed() if not hasattr(cls, 'logger'): cls.logger = getLogger(cls.__name__) else: cls.logger.name = cls.__name__ cls.tempdir = tempfile.mkdtemp( prefix='vpp-unittest-%s-' % cls.__name__) cls.stats_sock = "%s/stats.sock" % cls.tempdir cls.file_handler = FileHandler("%s/log.txt" % cls.tempdir) cls.file_handler.setFormatter( Formatter(fmt='%(asctime)s,%(msecs)03d %(message)s', datefmt="%H:%M:%S")) cls.file_handler.setLevel(DEBUG) cls.logger.addHandler(cls.file_handler) cls.shm_prefix = os.path.basename(cls.tempdir) os.chdir(cls.tempdir) cls.logger.info("Temporary dir is %s, shm prefix is %s", cls.tempdir, cls.shm_prefix) cls.setUpConstants() cls.reset_packet_infos() cls._captures = [] cls._zombie_captures = [] cls.verbose = 0 cls.vpp_dead = False cls.registry = VppObjectRegistry() cls.vpp_startup_failed = False cls.reporter = KeepAliveReporter() # need to catch exceptions here because if we raise, then the cleanup # doesn't get called and we might end with a zombie vpp try: cls.run_vpp() cls.reporter.send_keep_alive(cls) cls.vpp_stdout_deque = deque() cls.vpp_stderr_deque = deque() cls.pump_thread_stop_flag = Event() cls.pump_thread_wakeup_pipe = os.pipe() cls.pump_thread = Thread(target=pump_output, args=(cls,)) cls.pump_thread.daemon = True cls.pump_thread.start() if cls.debug_gdb or cls.debug_gdbserver: read_timeout = 0 else: read_timeout = 5 cls.vapi = VppPapiProvider(cls.shm_prefix, cls.shm_prefix, cls, read_timeout) if cls.step: hook = StepHook(cls) else: hook = PollHook(cls) cls.vapi.register_hook(hook) cls.wait_for_stats_socket() cls.statistics = VPPStats(socketname=cls.stats_sock) try: hook.poll_vpp() except VppDiedError: cls.vpp_startup_failed = True cls.logger.critical( "VPP died shortly after startup, check the" " output to standard error for possible cause") raise try: cls.vapi.connect() except Exception: try: cls.vapi.disconnect() except Exception: pass if cls.debug_gdbserver: print(colorize("You're running VPP inside gdbserver but " "VPP-API connection failed, did you forget " "to 'continue' VPP from within gdb?", RED)) raise except Exception: try: cls.quit() except Exception: pass raise
def image_search(settings, **spiderargs): from .spiders import crawler_setup image_url = spiderargs.get('image_url') image_data = spiderargs.get('image_data') if image_url: logger.info('Image-searching for %s' % image_url) elif image_data: logger.info('Image-searching for (image data)') else: return pipein, pipeout = os.pipe() pid = os.fork() if pid < 0: raise OSError('Forking child process failed.') if pid == 0: # child process os.close(pipein) writer = os.fdopen(pipeout, 'wb') statuscode = crawler_setup(settings, writer=writer, **spiderargs) writer.flush() writer.close() if not statuscode: statuscode = 0 os._exit(int(statuscode)) return # finis # parent process os.close(pipeout) reader = os.fdopen(pipein, 'rb') results = {} while True: # simple line-based protocol data = reader.readline()[:-1] if not data or len(data) == 0: break result = None try: result = json.loads(data) except ValueError: logger.error('Error decoding Spider data: %s' % (data,)) if not result: continue # result must have a search provider if not 'provider' in result: continue # should not happen result = SearchResultItem(result) provider = result['provider'] if not provider in results: results[provider] = [] results[provider].append(result) pid, status = os.waitpid(pid, 0) reader.close() # do not forget about empty results providers = ['KarmaDecay', 'Yandex', 'Bing', 'Tineye', 'Google'] for provider in providers: if not provider in results: results[provider] = [] # sort (by provider) for constant key order results = OrderedDict(sorted(results.items())) return results
def test_openFileDescriptors(self): """ Processes spawned with spawnProcess() close all extraneous file descriptors in the parent. They do have a stdin, stdout, and stderr open. """ # To test this, we are going to open a file descriptor in the parent # that is unlikely to be opened in the child, then verify that it's not # open in the child. source = networkString(""" import sys sys.path.insert(0, '{0}') from twisted.internet import process sys.stdout.write(repr(process._listOpenFDs())) sys.stdout.flush()""".format(twistedRoot.path)) r, w = os.pipe() self.addCleanup(os.close, r) self.addCleanup(os.close, w) # The call to "os.listdir()" (in _listOpenFDs's implementation) opens a # file descriptor (with "opendir"), which shows up in _listOpenFDs's # result. And speaking of "random" file descriptors, the code required # for _listOpenFDs itself imports logger, which imports random, which # (depending on your Python version) might leave /dev/urandom open. # More generally though, even if we were to use an extremely minimal C # program, the operating system would be within its rights to open file # descriptors we might not know about in the C library's # initialization; things like debuggers, profilers, or nsswitch plugins # might open some and this test should pass in those environments. # Although some of these file descriptors aren't predictable, we should # at least be able to select a very large file descriptor which is very # unlikely to be opened automatically in the subprocess. (Apply a # fudge factor to avoid hard-coding something too near a limit # condition like the maximum possible file descriptor, which a library # might at least hypothetically select.) fudgeFactor = 17 unlikelyFD = (resource.getrlimit(resource.RLIMIT_NOFILE)[0] - fudgeFactor) os.dup2(w, unlikelyFD) self.addCleanup(os.close, unlikelyFD) output = io.BytesIO() class GatheringProtocol(ProcessProtocol): outReceived = output.write def processEnded(self, reason): reactor.stop() reactor = self.buildReactor() reactor.callWhenRunning(reactor.spawnProcess, GatheringProtocol(), pyExe, [pyExe, b"-Wignore", b"-c", source], usePTY=self.usePTY) self.runReactor(reactor) reportedChildFDs = set(eval(output.getvalue())) stdFDs = [0, 1, 2] # Unfortunately this assertion is still not *entirely* deterministic, # since hypothetically, any library could open any file descriptor at # any time. See comment above. self.assertEqual( reportedChildFDs.intersection(set(stdFDs + [unlikelyFD])), set(stdFDs))
def __init__(self): super(_LinuxFifo, self).__init__() self.rfd, self.wfd = os.pipe() # must do before forking, since path contains pid! self.rpath = _LinuxFifo._mkFdPath(self.rfd) self.wpath = _LinuxFifo._mkFdPath(self.wfd)