Beispiel #1
0
    def __init__(self, args, debugger, time, memory):
        self._args = args
        self._child = _find_exe(self._args[0])
        self._debugger = debugger
        self._time = time
        self._memory = memory
        self._returncode = None
        self._tle = False
        self._pid = None
        self._rusage = None
        self._duration = None
        self._r_duration = None

        self._stdin_, self._stdin = os.pipe()
        self._stdout, self._stdout_ = os.pipe()
        self._stderr, self._stderr_ = os.pipe()
        self.stdin = os.fdopen(self._stdin, 'w')
        self.stdout = os.fdopen(self._stdout, 'r')
        self.stderr = os.fdopen(self._stderr, 'r')

        self._started = threading.Event()
        self._died = threading.Event()
        self._worker = threading.Thread(target=self.__spawn_execute)
        self._worker.start()
        if 0 and time:
            # Spawn thread to kill process after it times out
            self._shocker = threading.Thread(target=self.__shocker)
            self._shocker.start()
    def connect_to_new_process(self, fds):
        '''Request forkserver to create a child process.

        Returns a pair of fds (status_r, data_w).  The calling process can read
        the child process's pid and (eventually) its returncode from status_r.
        The calling process should write to data_w the pickled preparation and
        process data.
        '''
        self.ensure_running()
        if len(fds) + 4 >= MAXFDS_TO_SEND:
            raise ValueError('too many fds')
        with socket.socket(socket.AF_UNIX) as client:
            client.connect(self._forkserver_address)
            parent_r, child_w = os.pipe()
            child_r, parent_w = os.pipe()
            allfds = [child_r, child_w, self._forkserver_alive_fd,
                      semaphore_tracker.getfd()]
            allfds += fds
            try:
                reduction.sendfds(client, allfds)
                return parent_r, parent_w
            except:
                os.close(parent_r)
                os.close(parent_w)
                raise
            finally:
                os.close(child_r)
                os.close(child_w)
Beispiel #3
0
    def _get_handles(self, stdin, stdout, stderr):
        """Construct and return tuple with IO objects:
        p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
        """
        p2cread, p2cwrite = None, None
        c2pread, c2pwrite = None, None
        errread, errwrite = None, None

        if stdin is None:
            pass
        elif stdin == PIPE:
            p2cread, p2cwrite = os.pipe()
        elif isinstance(stdin, int):
            p2cread = stdin
        else:
            # Assuming file-like object
            p2cread = stdin.fileno()

        if stdout is None:
            pass
        elif stdout == PIPE:
            try:
                c2pread, c2pwrite = os.pipe()
            except:
                # Cleanup of previous pipe() descriptors
                if stdin == PIPE:
                    os.close(p2cread)
                    os.close(p2cwrite)
                raise
        elif isinstance(stdout, int):
            c2pwrite = stdout
        else:
            # Assuming file-like object
            c2pwrite = stdout.fileno()

        if stderr is None:
            pass
        elif stderr == PIPE:
            try:
                errread, errwrite = os.pipe()
            except:
                # Cleanup of previous pipe() descriptors
                if stdin == PIPE:
                    os.close(p2cread)
                    os.close(p2cwrite)
                if stdout == PIPE:
                    os.close(c2pread)
                    os.close(c2pwrite)
                raise
        elif stderr == STDOUT:
            errwrite = c2pwrite
        elif isinstance(stderr, int):
            errwrite = stderr
        else:
            # Assuming file-like object
            errwrite = stderr.fileno()

        return (p2cread, p2cwrite,
                c2pread, c2pwrite,
                errread, errwrite)
Beispiel #4
0
    def __init__(self, rrdtool='rrdtool'):
        """Get an rrdtool pipe."""

        r1=None
        r2=None
        w1=None
        w2=None
        self.pid=0
        self.pfile=None

        try:
            r1, w1=os.pipe()
            r2, w2=os.pipe()
            self.pfile=PipeFile(r1, w2)
            # Fork off rrdtool
            self.pid = os.fork()
            if self.pid == 0:
                self.pfile.close()
                os.dup2(r2, 0)
                os.dup2(w1, 1)
                os.dup2(w1, 2)
                os.execv(rrdtool, [rrdtool, '-'])
                os._exit(1)
        except:
            if r1: os.close(r1)
            if r2: os.close(r2)
            if w1: os.close(w1)
            if w2: os.close(w2)
            # Rethrow
            raise sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]
Beispiel #5
0
def exec_piped_fork(l, env, stdout, stderr):
    # spawn using fork / exec and providing a pipe for the command's
    # stdout / stderr stream
    if stdout != stderr:
        (rFdOut, wFdOut) = os.pipe()
        (rFdErr, wFdErr) = os.pipe()
    else:
        (rFdOut, wFdOut) = os.pipe()
        rFdErr = rFdOut
        wFdErr = wFdOut
    # do the fork
    pid = os.fork()
    if not pid:
        # Child process
        os.close( rFdOut )
        if rFdOut != rFdErr:
            os.close( rFdErr )
        os.dup2( wFdOut, 1 ) # is there some symbolic way to do that ?
        os.dup2( wFdErr, 2 )
        os.close( wFdOut )
        if stdout != stderr:
            os.close( wFdErr )
        exitval = 127
        try:
            os.execvpe(l[0], l, env)
        except OSError, e:
            exitval = exitvalmap.get(e[0], e[0])
            stderr.write("scons: %s: %s\n" % (l[0], e[1]))
        os._exit(exitval)
    def _launch(self, process_obj):
        os.environ["MULTIPROCESSING_FORKING_DISABLE"] = "1"
        spawn._Django_old_layout_hack__save()
        from . import semaphore_tracker
        tracker_fd = semaphore_tracker.getfd()
        self._fds.append(tracker_fd)
        prep_data = spawn.get_preparation_data(process_obj._name)
        fp = io.BytesIO()
        context.set_spawning_popen(self)
        try:
            reduction.dump(prep_data, fp)
            reduction.dump(process_obj, fp)
        finally:
            context.set_spawning_popen(None)

        parent_r = child_w = child_r = parent_w = None
        try:
            parent_r, child_w = os.pipe()
            child_r, parent_w = os.pipe()
            cmd = spawn.get_command_line(tracker_fd=tracker_fd,
                                         pipe_handle=child_r)
            self._fds.extend([child_r, child_w])
            self.pid = spawnv_passfds(
                spawn.get_executable(), cmd, self._fds,
            )
            self.sentinel = parent_r
            with open(parent_w, 'wb', closefd=False) as f:
                f.write(fp.getbuffer())
        finally:
            if parent_r is not None:
                util.Finalize(self, os.close, (parent_r,))
            for fd in (child_r, child_w, parent_w):
                if fd is not None:
                    os.close(fd)
Beispiel #7
0
 def start_subprocess(self):
     """Start octave using a subprocess (no tty support)"""
     errmsg = ('\n\nPlease install GNU Octave and put it in your path\n')
     ON_POSIX = 'posix' in sys.builtin_module_names
     if self.use_pty:
         master, slave = pty.openpty()
         self.wfid, self.rfid = master, master
         rpipe, wpipe = slave, slave
     else:
         self.rfid, wpipe = os.pipe()
         rpipe, self.wfid = os.pipe()
     kwargs = dict(close_fds=ON_POSIX, bufsize=0, stdin=rpipe,
                   stderr=wpipe, stdout=wpipe)
     if os.name == 'nt':
         startupinfo = subprocess.STARTUPINFO()
         startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
         kwargs['startupinfo'] = startupinfo
     try:
         proc = subprocess.Popen(['octave', '-q', '--braindead'],
                                 **kwargs)
     except OSError:  # pragma: no cover
         raise Oct2PyError(errmsg)
     else:
         self.reader = _Reader(self.rfid, self.read_queue)
         return proc
Beispiel #8
0
 def __init__(self, cmd, capturestderr=False, bufsize=-1):
     """The parameter 'cmd' is the shell command to execute in a
     sub-process.  On UNIX, 'cmd' may be a sequence, in which case arguments
     will be passed directly to the program without shell intervention (as
     with os.spawnv()).  If 'cmd' is a string it will be passed to the shell
     (as with os.system()).   The 'capturestderr' flag, if true, specifies
     that the object should capture standard error output of the child
     process.  The default is false.  If the 'bufsize' parameter is
     specified, it specifies the size of the I/O buffers to/from the child
     process."""
     _cleanup()
     self.cmd = cmd
     p2cread, p2cwrite = os.pipe()
     c2pread, c2pwrite = os.pipe()
     if capturestderr:
         errout, errin = os.pipe()
     self.pid = os.fork()
     if self.pid == 0:
         # Child
         os.dup2(p2cread, 0)
         os.dup2(c2pwrite, 1)
         if capturestderr:
             os.dup2(errin, 2)
         self._run_child(cmd)
     os.close(p2cread)
     self.tochild = os.fdopen(p2cwrite, 'w', bufsize)
     os.close(c2pwrite)
     self.fromchild = os.fdopen(c2pread, 'r', bufsize)
     if capturestderr:
         os.close(errin)
         self.childerr = os.fdopen(errout, 'r', bufsize)
     else:
         self.childerr = None
Beispiel #9
0
    def _run(self):
        self._log.debug("Starting IOProcess...")
        myRead, hisWrite = os.pipe()
        hisRead, myWrite = os.pipe()

        self._partialLogs = ""
        cmd = [self.IOPROCESS_EXE,
               "--read-pipe-fd", str(hisRead),
               "--write-pipe-fd", str(hisWrite),
               "--max-threads", str(self._max_threads)]

        if self._DEBUG_VALGRIND:
            cmd = ["valgrind", "--log-file=ioprocess.valgrind.log",
                   "--leak-check=full", "--tool=memcheck"] + cmd + \
                  ["--keep-fds"]

        p = CPopen(cmd)

        os.close(hisRead)
        os.close(hisWrite)

        setNonBlocking(myRead)
        setNonBlocking(myWrite)

        self._startCommunication(p, myRead, myWrite)
Beispiel #10
0
    def start(self):
        assert not self._started
        self._started = True

        up_read, up_write = os.pipe()
        down_read, down_write = os.pipe()
        args, sock = self.args, self.sock

        pid = os.fork()
        if pid:
            # parent
            os.close(up_read)
            os.close(down_write)
            tulip.async(self.connect(pid, up_write, down_read))
        else:
            # child
            os.close(up_write)
            os.close(down_read)

            # cleanup after fork
            tulip.set_event_loop(None)

            # setup process
            process = ChildProcess(up_read, down_write, args, sock)
            process.start()
Beispiel #11
0
 def __init__(self, cmd, capturestderr=False, bufsize=-1):
     """The parameter 'cmd' is the shell command to execute in a
     sub-process.  The 'capturestderr' flag, if true, specifies that
     the object should capture standard error output of the child process.
     The default is false.  If the 'bufsize' parameter is specified, it
     specifies the size of the I/O buffers to/from the child process."""
     _cleanup()
     p2cread, p2cwrite = os.pipe()
     c2pread, c2pwrite = os.pipe()
     if capturestderr:
         errout, errin = os.pipe()
     self.pid = os.fork()
     if self.pid == 0:
         # Child
         os.dup2(p2cread, 0)
         os.dup2(c2pwrite, 1)
         if capturestderr:
             os.dup2(errin, 2)
         self._run_child(cmd)
     os.close(p2cread)
     self.tochild = os.fdopen(p2cwrite, 'w', bufsize)
     os.close(c2pwrite)
     self.fromchild = os.fdopen(c2pread, 'r', bufsize)
     if capturestderr:
         os.close(errin)
         self.childerr = os.fdopen(errout, 'r', bufsize)
     else:
         self.childerr = None
     _active.append(self)
Beispiel #12
0
 def __init__(self, *args, **kwargs):
     self.io_loop = kwargs.pop('io_loop', None)
     to_close = []
     if kwargs.get('stdin') is Subprocess.STREAM:
         in_r, in_w = os.pipe()
         kwargs['stdin'] = in_r
         to_close.append(in_r)
         self.stdin = PipeIOStream(in_w, io_loop=self.io_loop)
     if kwargs.get('stdout') is Subprocess.STREAM:
         out_r, out_w = os.pipe()
         kwargs['stdout'] = out_w
         to_close.append(out_w)
         self.stdout = PipeIOStream(out_r, io_loop=self.io_loop)
     if kwargs.get('stderr') is Subprocess.STREAM:
         err_r, err_w = os.pipe()
         kwargs['stderr'] = err_w
         to_close.append(err_w)
         self.stdout = PipeIOStream(err_r, io_loop=self.io_loop)
     self.proc = subprocess.Popen(*args, **kwargs)
     for fd in to_close:
         os.close(fd)
     for attr in ['stdin', 'stdout', 'stderr', 'pid']:
         if not hasattr(self, attr):  # don't clobber streams set above
             setattr(self, attr, getattr(self.proc, attr))
     self._exit_callback = None
     self.returncode = None
Beispiel #13
0
def _processRequest(conn, sessionDict):
    DEBUG(EXTCGI, 'extcgi Processing Request')
    #dict of environ, headers, stdin
    kid_stdin, parent_to_kid_stdin = os.pipe()
    parent_to_kid_stdout, kid_stdout = os.pipe()
    parent_to_kid_stderr, kid_stderr = os.pipe()
    pid = os.fork()
    try:
        if pid: #ok I'm the parent
            DEBUG(EXTCGI, 'child pid is %d' % pid)
            #close kid sides
            os.close(kid_stdin)
            os.close(kid_stdout)
            os.close(kid_stderr)

            stdin = conn._stdin

            return _doCGI(conn, pid, parent_to_kid_stdin,
                          parent_to_kid_stdout,
                          parent_to_kid_stderr, stdin)

        else: #I'm the kid
            #close parent side of pipes
            os.close(parent_to_kid_stdin)
            os.close(parent_to_kid_stdout)
            os.close(parent_to_kid_stderr)
            #dup kid sides to my stdin/out/err
            os.dup2(kid_stdin, 0)
            os.dup2(kid_stdout, 1)
            os.dup2(kid_stderr, 2)
            env = _fix(conn.env)
            if DEBUGIT(EXTCGI):
                DEBUG(EXTCGI, "environment is %s" % _dispenv(env))
            prog = Configuration.CGIProgram
            args = ( (prog,) + (prog,) +
                     Configuration.CGIProgramArgs +(env,))
            DEBUG(EXTCGI, 'args is %s' % repr(args))
            oldpwd = os.getcwd()
            try:
                os.chdir(os.path.split(env["PATH_TRANSLATED"])[0])
                os.execle(*args)

            finally:
                os.chdir(oldpwd)
    except:
        if pid == 0: #I'm the kid
            logException()
            #give the parent some info as to why I died
            os.write(kid_stderr, "exception executing CGI : %s %s" % (
                sys.exc_info()[0], sys.exc_info()[1]))
            DEBUG(EXTCGI, "I'm still here! killing self");
            os.kill(os.getpid(), 9)

        else: #I'm the parent
            e, t, tb = sys.exc_info()
            try:
                os.kill(pid, 9) # we screwed up, kill kid
            except: #in event that it's already dead, that's ok too
                pass
            raise e, t, tb
Beispiel #14
0
def spawn_test(test, prefix, passthrough, run_skipped, show_cmd):
    """Spawn one child, return a task struct."""
    if not test.enable and not run_skipped:
        return None

    cmd = test.get_command(prefix)
    if show_cmd:
        print(escape_cmdline(cmd))

    if not passthrough:
        (rout, wout) = os.pipe()
        (rerr, werr) = os.pipe()

        rv = os.fork()

        # Parent.
        if rv:
            os.close(wout)
            os.close(werr)
            return Task(test, prefix, rv, rout, rerr)

        # Child.
        os.close(rout)
        os.close(rerr)

        os.dup2(wout, 1)
        os.dup2(werr, 2)

    os.execvp(cmd[0], cmd)
Beispiel #15
0
    def __init__(self, tests, num_workers, total_cpus, total_mem, bindir):
        """
        Initialize the class.

        @param tests: A list of test dictionaries.
        @param num_workers: The number of workers (pipelines).
        @param total_cpus: The total number of CPUs to dedicate to tests.
        @param total_mem: The total amount of memory to dedicate to tests.
        @param bindir: The directory where environment files reside.
        """
        self.tests = tests
        self.num_workers = num_workers
        self.total_cpus = total_cpus
        self.total_mem = total_mem
        self.bindir = bindir
        # Pipes -- s stands for scheduler, w stands for worker
        self.s2w = [os.pipe() for i in range(num_workers)]
        self.w2s = [os.pipe() for i in range(num_workers)]
        self.s2w_r = [os.fdopen(r, "r", 0) for r, w in self.s2w]
        self.s2w_w = [os.fdopen(w, "w", 0) for r, w in self.s2w]
        self.w2s_r = [os.fdopen(r, "r", 0) for r, w in self.w2s]
        self.w2s_w = [os.fdopen(w, "w", 0) for r, w in self.w2s]
        # "Personal" worker dicts contain modifications that are applied
        # specifically to each worker.  For example, each worker must use a
        # different environment file and a different MAC address pool.
        self.worker_dicts = [{"env": "env%d" % i} for i in range(num_workers)]
Beispiel #16
0
    def test_gevent_friendly(self):

        # Used to verify that file descriptors aren't consumed
        r,w = os.pipe()
        os.close(r)
        os.close(w)

        # Get a good benchmark without any concurrent actions
        t1 = time.time()
        with AsyncDispatcher(self.block_stuff) as dispatcher:
            v = dispatcher.wait(10)
        dt = time.time() - t1

        # Check that it takes less than 5 seconds and that it's the right value
        self.assertTrue(dt < 10)
        self.assertTrue(np.array_equal(v,np.arange(20)))

        # Try it again but this time with a gevent sleep that should run
        # Concurrently with the dispatcher thread
        t1 = time.time()
        with AsyncDispatcher(self.block_stuff) as dispatcher:
            gevent.sleep(5)
            v = dispatcher.wait(10)
        ndt = time.time() - t1

        # There is ususally some difference but should definitely be less than
        # one second
        self.assertTrue( abs(dt - ndt) < 5)

        try:
            # Make sure we're not losing file descriptors to maintain thread synchronization
            self.assertEquals((r,w), os.pipe())
        finally:
            os.close(r)
            os.close(w)
def spawn(prog, args):
    p2cread, p2cwrite = os.pipe()
    c2pread, c2pwrite = os.pipe()
    pid = os.fork()
    if pid == 0:
        # Child
        for i in 0, 1, 2:
            try:
                os.close(i)
            except os.error:
                pass
        if os.dup(p2cread) != 0:
            sys.stderr.write('popen2: bad read dup\n')
        if os.dup(c2pwrite) != 1:
            sys.stderr.write('popen2: bad write dup\n')
        if os.dup(c2pwrite) != 2:
            sys.stderr.write('popen2: bad write dup\n')
        os.closerange(3, MAXFD)
        try:
            os.execvp(prog, args)
        finally:
            sys.stderr.write('execvp failed\n')
            os._exit(1)
    os.close(p2cread)
    os.close(c2pwrite)
    return pid, c2pread, p2cwrite
Beispiel #18
0
def decrypt_content(content):
    """Decrypt content and verify a valid signature exists."""
    r_status, w_status = os.pipe()
    r_log, w_log = os.pipe()
    r_att, w_att = os.pipe()

    command = get_gpg_command()
    command.extend(['--decrypt', '--status-fd', str(w_status),
                    '--logger-fd', str(w_log), '--attribute-fd', str(w_att)])
    p = subprocess.Popen(command, stdout=subprocess.PIPE, stdin=subprocess.PIPE,
                        pass_fds=[w_status, w_log, w_att])
    output = p.communicate(input=content.encode('utf-8'))[0]
    if p.returncode != 0:
        raise Exception('Error decrypting content')

    os.close(w_status)
    os.close(w_log)
    os.close(w_att)

    config = get_config_dict()
    messages = os.fdopen(r_status).read()
    if not (' VALIDSIG ' + config['default-key']) in messages:
        raise Exception('No valid signature from default key')

    return output.decode()
Beispiel #19
0
 def does_stuff():
     a, b = os.pipe()
     c = os.dup(a)
     d = os.dup(b)
     assert a != b
     assert a != c
     assert a != d
     assert b != c
     assert b != d
     assert c != d
     os.close(c)
     os.dup2(d, c)
     e, f = os.pipe()
     assert e != a
     assert e != b
     assert e != c
     assert e != d
     assert f != a
     assert f != b
     assert f != c
     assert f != d
     assert f != e
     os.close(a)
     os.close(b)
     os.close(c)
     os.close(d)
     os.close(e)
     os.close(f)
     return 42
    def spawn(self, children):
        if self.children:
            raise RuntimeError, "children are already running"

        self.childinfo = []
        self.fdchild = {}
        self.childitem = [None] * children
        self.readpipes = []
        self.reporter.init(children)
        self.children = children

        for i in range(children):
            childread, parentwrite = os.pipe()
            parentread, childwrite = os.pipe()
            pid = os.fork()
            self.fdchild[parentread] = i
            if pid > 0:
                os.close(childread)
                os.close(childwrite)
                self.childinfo.append((pid, parentread, parentwrite))
                self.reporter.spawn(pid)
            else:
                os.close(parentread)
                os.close(parentwrite)
                self.child(childread, childwrite)
Beispiel #21
0
    def __init_streams(self, stdin, stdout, stderr, unbuffered):
        self.stdin = self.stdout = self.stderr = None
        
        if unbuffered:
            master, slave = pty.openpty()

        if stdin is PIPE:
            self._child_stdin, self._stdin = (slave, master) if unbuffered else os.pipe()
            self.stdin = os.fdopen(self._stdin, 'w')
        elif isinstance(stdin, int):
            self._child_stdin, self._stdin = stdin, -1
        elif stdin is not None:
            self._child_stdin, self._stdin = stdin.fileno(), -1
        else:
            self._child_stdin = self._stdin = -1

        if stdout is PIPE:
            self._stdout, self._child_stdout = (master, slave) if unbuffered else os.pipe()
            self.stdout = os.fdopen(self._stdout, 'r')
        elif isinstance(stdout, int):
            self._stdout, self._child_stdout = -1, stdout
        elif stdout is not None:
            self._stdout, self._child_stdout = -1, stdout.fileno()
        else:
            self._stdout = self._child_stdout = -1

        if stderr is PIPE:
            self._stderr, self._child_stderr = os.pipe()
            self.stderr = os.fdopen(self._stderr, 'r')
        elif isinstance(stderr, int):
            self._stderr, self._child_stderr = -1, stderr
        elif stderr is not None:
            self._stderr, self._child_stderr = -1, stderr.fileno()
        else:
            self._stderr = self._child_stderr = -1
Beispiel #22
0
    def _OpenPipes (self, cmd, args):
    #  PRE: cmd is assigned the path to a prolog interpreter
    #   &&  args is assigned a tuple containing any arguments to pass to cmd
    # POST: cmd has been spawned with args as arguments
    #   &&  FCTVAL = (file descriptor to incoming pipe,
    #                 file descriptor to outgoing pipe,
    #                 file descriptor to incoming error pipe,
    #                 child process's pid)
    #       "A better popen2 than popen2.popen2"

        pipe1 = os.pipe ()
        pipe2 = os.pipe ()
        pipe3 = os.pipe ()
        pid = os.fork ()
        #pid = os.spawnv(os.P_NOWAIT, cmd, args)

        if pid:
            # I'm the parent
            os.close (pipe1[1])
            os.close (pipe2[0])
            os.close (pipe3[1])

            return (pipe1[0], pipe2[1], pipe3[0], pid)
        else:
            # I'm the child
            os.close (pipe1[0])
            os.close (pipe2[1])
            os.close (pipe3[0])

            os.dup2 (pipe1[1], 1)
            os.dup2 (pipe2[0], 0)
            os.dup2 (pipe3[1], 2)

            os.execvp (cmd, args)
Beispiel #23
0
def _bash2(command, mode='o'):
	stdout_r, stdout_w = None, None
	(stdin_r, stdin_w) = os.pipe()
	if 'o' in mode:
		(stdout_r, stdout_w) = os.pipe()
	p = Popen('/bin/bash', stdin=stdin_r, stdout=stdout_w)
	os.write(stdin_w, command + '\nexit $?\n')
	os.close(stdin_w)
	output = []
	ret = p.wait()
	os.close(stdin_r)
	if stdout_r:
		os.close(stdout_w)
		out = os.read(stdout_r, 32768)
		while out != '':
			output.append(out)
			out = os.read(stdout_r, 32768)
		output = (''.join(output)).strip()
		os.close(stdout_r)
	if 'ov' == mode:
		return (output, ret)
	elif 'o' in mode:
		return output
	elif 'v' in mode:
		return ret
	return None
Beispiel #24
0
	def __init__(self, prog = None, r = None, w = None):
	
		self.prog = prog
		self.objects = {}
		self.warn = 0
		self._erase = []
		self.wantarray = 1
	
		if prog is None: return self.minor()
		
		if prog == -1:
			self.r = r
			self.w = w
			self.role = "TEST"
			return
	
		ch_reader, writer = os.pipe()
		reader, ch_writer = os.pipe()
		
		pid = os.fork()
		
		if pid==0:
			prog = PROG.get(prog, prog)
			prog = prog % (os.path.dirname(RPC.__module__.__file__))
			if ch_reader != 4: os.dup2(ch_reader, 4)
			if ch_writer != 5: os.dup2(ch_writer, 5)
			args = shlex.split(prog)
			os.execvp(args[0], args[1:])
		
		self.r = os.fdopen(reader, "rb")
		self.w = os.fdopen(writer, "wb")
		self.role = "MAJOR"
Beispiel #25
0
    def __init__(self, cwd=None, env=None):
        BaseProxy.__init__(self, set())
        if env is None:
            env = os.environ.copy()
        self._servers = [ None ] * 5
        p0, p1 = os.pipe(), os.pipe()

        self._pid = os.fork()
        if self._pid == 0:
            os.close(p1[1])
            os.close(p0[0])
            if not cwd:
                if 'conary_test.serverCacheProxy' in sys.modules:
                    fpath = sys.modules['conary_test.serverCacheProxy'].__file__
                else:
                    fpath = sys.argv[0]
                cwd = os.path.dirname(fpath)
            cmd = ["/usr/bin/python", "-c",
                "from conary_test import serverCacheProxy; "
                "serverCacheProxy.Child(%d, %d)" %
                    (p1[0], p0[1])]
            os.chdir(cwd)
            os.execve(cmd[0], cmd, env)
            os._exit(0)

        try:
            os.close(p0[1])
            os.close(p1[0])
            self._pipe = Pipe(p0[0], p1[1])
            sdict = loadData(self._pipe)
            self._methods = sdict['methods']
            #print "Child started", methodname, params
        except:
            os.waitpid(self._pid, 0)
            raise
Beispiel #26
0
def startQuickAsy():
  global quickAsy
  global quickAsyFailed
  global AsyTempDir
  global fout,fin
  if quickAsyRunning():
    return
  try:
    fout.close()
    quickAsy.wait()
  except:
    pass
  try:
    quickAsyFailed = False
    if os.name == "nt":
      AsyTempDir=mkdtemp(prefix="asy_", dir="./")
    else:
      AsyTempDir=mkdtemp(prefix="asy_")+os.sep
    if sys.platform[:3] == 'win':
      quickAsy=Popen([xasyOptions.options['asyPath'],"-noV","-multiline","-q",
               "-o"+AsyTempDir,"-inpipe=0","-outpipe=2"],stdin=PIPE,stderr=PIPE)
      fout=quickAsy.stdin
      fin=quickAsy.stderr
    else:
      (rx,wx) = os.pipe()
      (ra,wa) = os.pipe()
      quickAsy=Popen([xasyOptions.options['asyPath'],"-noV","-multiline","-q",
               "-o"+AsyTempDir,"-inpipe="+str(rx),"-outpipe="+str(wa)])
      fout=os.fdopen(wx,'w')
      fin=os.fdopen(ra,'r')
    if quickAsy.returncode != None:
      quickAsyFailed = True
  except:
    quickAsyFailed = True
Beispiel #27
0
def timed_command(cmd, timeout=None):
    '''timed_command(cmd, timeout=None)
    Run a shell command, with an optional timeout
    If command takes longer than specified time it will be killed,
        first with SIGTERM then 2 seconds later with SIGKILL if needed.
        (The 2 second value can be adjusted by changing GRACE_PERIOD).
    Return (exit_status, time_used, command_output, error_output)
    If the program was terminated due to timeout, the exit status will be
        artificially set as though the program returned ETIMEDOUT.'''

    if timeout==0:
        timeout=None
        
    stdin_r, stdin_w = os.pipe()
    stdout_r, stdout_w = os.pipe()
    stderr_r, stderr_w = os.pipe()

    prev_sighandler = signal.getsignal(signal.SIGCHLD)
    signal.signal(signal.SIGCHLD, _sighandler)
    pid = os.fork()
    if pid == 0: # Child
        _child(stdin_r, stdout_w, stderr_w, cmd)
    else: # Parent
        ret = _parent(stdout_r, stderr_r, pid, timeout)
        # Close all file descriptors we opened
        for fd in stdin_r, stdin_w, stdout_r, stdout_w, stderr_r, stderr_w:
            try:
                os.close(fd)
            except:
                pass
        # Restore default behavior on SIGCHLD
        signal.signal(signal.SIGCHLD, prev_sighandler)
        return ret
Beispiel #28
0
def spawn(prog, args):
	p2cread, p2cwrite = os.pipe()
	c2pread, c2pwrite = os.pipe()
	pid = os.fork()
	if pid == 0:
		# Child
		os.close(0)
		os.close(1)
		os.close(2)
		if os.dup(p2cread) <> 0:
			sys.stderr.write('popen2: bad read dup\n')
		if os.dup(c2pwrite) <> 1:
			sys.stderr.write('popen2: bad write dup\n')
		if os.dup(c2pwrite) <> 2:
			sys.stderr.write('popen2: bad write dup\n')
		for i in range(3, MAXFD):
			try:
				os.close(i)
			except:
				pass
		try:
			os.execvp(prog, args)
		finally:
			sys.stderr.write('execvp failed\n')
			os._exit(1)
	os.close(p2cread)
	os.close(c2pwrite)
	return pid, c2pread, p2cwrite
Beispiel #29
0
    def __init__(self):
        (client_reader, server_writer) = os.pipe()
        (server_reader, client_writer) = os.pipe()

        if not os.fork():
            # the child process starts a server and exits when done
            RMI.DEBUG_MSG_PREFIX = '    SERVER'
            #RMI.DEBUG_FLAG = 1
            server_reader = os.fdopen(server_reader, 'r')
            server_writer = os.fdopen(server_writer, 'w')
            s = RMI.Node(reader = server_reader, writer = server_writer)

            # the server should return fals whenever a client disconnects
            # somehow it just hangs :( 
            # we need to fix this
            response = s.receive_request_and_send_response()
            while (response):
                if response[3] == 'exitnow':
                    response = None
                else:
                    response = s.receive_request_and_send_response()
            print("SERVER DONE")
            exit()

        else:
            # the parent process initializes as the client and continues
            RMI.DEBUG_MSG_PREFIX = 'CLIENT'
            #RMI.DEBUG_FLAG = 1 
            client_reader = os.fdopen(client_reader, 'r')
            client_writer = os.fdopen(client_writer, 'w')
            RMI.Client.__init__(self,client_reader,client_writer)
Beispiel #30
0
def kinit(username, password, cfile=None):
    kc = cfile
    if kc is None:
        kc = mkstemp();
        os.close(kc[0]);
        kc = kc[1]
    p2fr, p2fw = os.pipe()
    f2pr, f2pw = os.pipe()
    pid = os.fork()
    if pid == 0:
        os.dup2(p2fr, 0)
        os.dup2(f2pw, 1)
        os.dup2(f2pw, 2)
        os.execv('/usr/bin/kinit', ['kinit', '-l', '3d', '-c', kc, username])
    else:
        os.close(p2fr)
        os.close(f2pw)
        os.fdopen(p2fw,'w').write('%s\n' % password)
        pres = os.waitpid(pid, 0)
        if pres[1] == 0:
            # return ticket kc
            return kc
        else:
            # execv failure
            if cfile is None:
                # remove ticket kc
                os.unlink(kc)
 def run(self):
     self.logger.debug("Starting logging thread")
     self.readpipe, self.writepipe = os.pipe()
     threading.Thread.run(self)
 def setUp(self):
     self.read_fd, self.write_fd = os.pipe()
Beispiel #33
0
def test_os_pipe():
    import os
    r, w = os.pipe()
    written = os.write(w, b"hello")
    assert os.read(r, written) == b"hello"
Beispiel #34
0
def execWithRedirect(command, argv, stdin = None, stdout = None,
                     stderr = None, root = '/'):
    def chroot ():
        os.chroot(root)

    stdinclose = stdoutclose = stderrclose = lambda : None

    argv = list(argv)
    if isinstance(stdin, str):
        if os.access(stdin, os.R_OK):
            stdin = os.open(stdin, os.O_RDONLY)
            stdinclose = lambda : os.close(stdin)
        else:
            stdin = sys.stdin.fileno()
    elif isinstance(stdin, int):
        pass
    elif stdin is None or not isinstance(stdin, file):
        stdin = sys.stdin.fileno()

    if isinstance(stdout, str):
        stdout = os.open(stdout, os.O_RDWR|os.O_CREAT)
        stdoutclose = lambda : os.close(stdout)
    elif isinstance(stdout, int):
        pass
    elif stdout is None or not isinstance(stdout, file):
        stdout = sys.stdout.fileno()

    if isinstance(stderr, str):
        stderr = os.open(stderr, os.O_RDWR|os.O_CREAT)
        stderrclose = lambda : os.close(stderr)
    elif isinstance(stderr, int):
        pass
    elif stderr is None or not isinstance(stderr, file):
        stderr = sys.stderr.fileno()

    program_log.info("Running... %s" % (" ".join([command] + argv),))

    #prepare os pipes for feeding tee proceses
    pstdout, pstdin = os.pipe()
    perrout, perrin = os.pipe()
   
    env = os.environ.copy()
    env.update({"LC_ALL": "C", "LANGUAGE": "C", "LANG": "C"})

    try:
        #prepare tee proceses
        proc_std = tee(pstdout, stdout, program_log.info)
        proc_err = tee(perrout, stderr, program_log.error)

        #start monitoring the outputs
        proc_std.start()
        proc_err.start()

        proc = subprocess.Popen([command] + argv, stdin=stdin,
                                stdout=pstdin,
                                stderr=perrin,
                                preexec_fn=chroot, cwd=root,
                                env=env)

        proc.wait()
        ret = proc.returncode

        #close the input ends of pipes so we get EOF in the tee processes
        os.close(pstdin)
        os.close(perrin)

        #wait for the output to be written and destroy them
        proc_std.join()
        del proc_std

        proc_err.join()
        del proc_err

        stdinclose()
        stdoutclose()
        stderrclose()
    except OSError as e:
        errstr = "Error running %s: %s" % (command, e.strerror)
        log.error(errstr)
        program_log.error(errstr)
        #close the input ends of pipes so we get EOF in the tee processes
        os.close(pstdin)
        os.close(perrin)
        proc_std.join()
        proc_err.join()

        stdinclose()
        stdoutclose()
        stderrclose()
        raise RuntimeError, errstr

    return ret
Beispiel #35
0
def execWithCallback(command, argv, stdin = None, stdout = None,
                     stderr = None, echo = True, callback = None,
                     callback_data = None, root = '/'):
    def chroot():
        os.chroot(root)

    def closefds ():
        stdinclose()
        stdoutclose()
        stderrclose()

    stdinclose = stdoutclose = stderrclose = lambda : None

    argv = list(argv)
    if isinstance(stdin, str):
        if os.access(stdin, os.R_OK):
            stdin = os.open(stdin, os.O_RDONLY)
            stdinclose = lambda : os.close(stdin)
        else:
            stdin = sys.stdin.fileno()
    elif isinstance(stdin, int):
        pass
    elif stdin is None or not isinstance(stdin, file):
        stdin = sys.stdin.fileno()

    if isinstance(stdout, str):
        stdout = os.open(stdout, os.O_RDWR|os.O_CREAT)
        stdoutclose = lambda : os.close(stdout)
    elif isinstance(stdout, int):
        pass
    elif stdout is None or not isinstance(stdout, file):
        stdout = sys.stdout.fileno()

    if isinstance(stderr, str):
        stderr = os.open(stderr, os.O_RDWR|os.O_CREAT)
        stderrclose = lambda : os.close(stderr)
    elif isinstance(stderr, int):
        pass
    elif stderr is None or not isinstance(stderr, file):
        stderr = sys.stderr.fileno()

    program_log.info("Running... %s" % (" ".join([command] + argv),))

    p = os.pipe()
    p_stderr = os.pipe()
    childpid = os.fork()
    if not childpid:
        os.close(p[0])
        os.close(p_stderr[0])
        os.dup2(p[1], 1)
        os.dup2(p_stderr[1], 2)
        os.dup2(stdin, 0)
        os.close(stdin)
        os.close(p[1])
        os.close(p_stderr[1])

        os.execvp(command, [command] + argv)
        os._exit(1)

    os.close(p[1])
    os.close(p_stderr[1])

    logline = ''
    while 1:
        try:
            s = os.read(p[0], 1)
        except OSError as e:
            if e.errno != 4:
                raise IOError, e.args

        if echo:
            os.write(stdout, s)

        if s == '\n':
            program_log.info(logline)
            logline = ''
        else:
            logline += s

        if callback:
            callback(s, callback_data=callback_data)

        # break out early if the sub-process changes status.
        # no need to flush the stream if the process has exited
        try:
            (pid, status) = os.waitpid(childpid,os.WNOHANG)
            if pid != 0:
                break
        except OSError as e:
            log.critical("exception from waitpid: %s %s" %(e.errno, e.strerror))

        if len(s) < 1:
            break
    if len(logline) > 0:
        program_log.info(logline)

    log_errors = ''
    while 1:
        try:
            err = os.read(p_stderr[0], 128)
        except OSError as e:
            if e.errno != 4:
                raise IOError, e.args
            break
        log_errors += err
        if len(err) < 1:
            break
    map(program_log.error, log_errors.splitlines())
    os.close(p[0])
    os.close(p_stderr[0])

    try:
        #if we didn't already get our child's exit status above, do so now.
        if not pid:
            (pid, status) = os.waitpid(childpid, 0)
    except OSError as e:
        log.critical("exception from waitpid: %s %s" %(e.errno, e.strerror))

    closefds()
    # *shrug*  no clue why this would happen, but hope that things are fine
    if status is None:
        return 0

    if os.WIFEXITED(status):
        return os.WEXITSTATUS(status)

    return 1
Beispiel #36
0
import os, sys, time

rd, wd = os.pipe()
r, w = os.fdopen(rd, 'rb', 0), os.fdopen(wd, 'wb', 0)

pid = os.fork()

if (pid == 0):
    print('if')
    r.close()
    #time.sleep(5)
    for i in range(10):
        print('bucle for')
        mensaje = 'linea %d \n' % i
        w.write(mensaje.encode('utf8'))
        w.flush()
        #Podemos comentar la siguiente linea y observamos el comportamiento, ayuda a entender el ejercicio
        time.sleep(1)
    print('Soy el hijo \n')

if (pid == -1):
    print('Error')

else:
    print('else')
    w.close()
    while True:
        print('while True')
        data = r.readline()
        print('data')
        if not data:
Beispiel #37
0
    def make_iostream_pair(self, **kwargs):
        r, w = os.pipe()

        return PipeIOStream(r, **kwargs), PipeIOStream(w, **kwargs)
Beispiel #38
0
 def __init__(self):
     self.queue = Queue.Queue()
     self.pipe = os.pipe()
     self._closed = False
Beispiel #39
0
def sendzfs(fromsnap, tosnap, dataset, localfs, remotefs, followdelete, throttle, compression, replication, reached_last):
    global results
    global templog

    progressfile = '/tmp/.repl_progress_%d' % replication.id
    cmd = ['/sbin/zfs', 'send', '-V']

    # -p switch will send properties for whole dataset, including snapshots
    # which will result in stale snapshots being delete as well
    if followdelete:
        cmd.append('-p')

    if fromsnap is None:
        cmd.append("%s@%s" % (dataset, tosnap))
    else:
        cmd.extend(['-i', "%s@%s" % (dataset, fromsnap), "%s@%s" % (dataset, tosnap)])
    # subprocess.Popen does not handle large stream of data between
    # processes very well, do it on our own
    readfd, writefd = os.pipe()
    zproc_pid = os.fork()
    if zproc_pid == 0:
        os.close(readfd)
        os.dup2(writefd, 1)
        os.close(writefd)
        os.execv('/sbin/zfs', cmd)
        # NOTREACHED
    else:
        with open(progressfile, 'w') as f2:
            f2.write(str(zproc_pid))
        os.close(writefd)

    compress, decompress = compress_pipecmds(compression)
    replcmd = '%s%s/bin/dd obs=1m 2> /dev/null | /bin/dd obs=1m 2> /dev/null | /usr/local/bin/pipewatcher $$ | %s "%s/sbin/zfs receive -F -d \'%s\' && echo Succeeded"' % (compress, throttle, sshcmd, decompress, remotefs)
    log.debug('Sending zfs snapshot: %s | %s', ' '.join(cmd), replcmd)
    with open(templog, 'w+') as f:
        readobj = os.fdopen(readfd, 'rb', 0)
        proc = subprocess.Popen(
            replcmd,
            shell=True,
            stdin=readobj,
            stdout=f,
            stderr=subprocess.STDOUT,
        )
        proc.wait()
        os.waitpid(zproc_pid, os.WNOHANG)
        readobj.close()
        os.remove(progressfile)
        f.seek(0)
        msg = f.read().strip('\n').strip('\r')
    os.remove(templog)
    msg = msg.replace('WARNING: ENABLED NONE CIPHER', '')
    msg = msg.strip('\r').strip('\n')
    log.debug("Replication result: %s" % (msg))
    results[replication.id]['msg'] = msg
    # When replicating to a target "container" dataset that doesn't exist on the sending
    # side the target dataset will have to be readonly, however that will preclude
    # creating mountpoints for the datasets that are sent.
    # In that case you'll get back a failed to create mountpoint message, which
    # we'll go ahead and consider a success.
    if reached_last and ("Succeeded" in msg or "failed to create mountpoint" in msg):
        results[replication.id]['last_snapshot'] = tosnap
    return ("Succeeded" in msg or "failed to create mountpoint" in msg)
    def launch(self,
               launch_cmd,
               get_ip=True,
               qemuparams=None,
               extra_bootparams=None,
               env=None):
        try:
            threadsock, threadport = self.create_socket()
            self.server_socket, self.serverport = self.create_socket()
        except socket.error as msg:
            self.logger.error("Failed to create listening socket: %s" % msg[1])
            return False

        bootparams = 'console=tty1 console=ttyS0,115200n8 printk.time=1'
        if extra_bootparams:
            bootparams = bootparams + ' ' + extra_bootparams

        # Ask QEMU to store the QEMU process PID in file, this way we don't have to parse running processes
        # and analyze descendents in order to determine it.
        if os.path.exists(self.qemu_pidfile):
            os.remove(self.qemu_pidfile)
        self.qemuparams = 'bootparams="{0}" qemuparams="-serial tcp:127.0.0.1:{1} -pidfile {2}"'.format(
            bootparams, threadport, self.qemu_pidfile)
        if qemuparams:
            self.qemuparams = self.qemuparams[:-1] + " " + qemuparams + " " + '\"'

        launch_cmd += ' tcpserial=%s %s' % (self.serverport, self.qemuparams)

        self.origchldhandler = signal.getsignal(signal.SIGCHLD)
        signal.signal(signal.SIGCHLD, self.handleSIGCHLD)

        self.logger.debug('launchcmd=%s' % (launch_cmd))

        # FIXME: We pass in stdin=subprocess.PIPE here to work around stty
        # blocking at the end of the runqemu script when using this within
        # oe-selftest (this makes stty error out immediately). There ought
        # to be a proper fix but this will suffice for now.
        self.runqemu = subprocess.Popen(launch_cmd,
                                        shell=True,
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.STDOUT,
                                        stdin=subprocess.PIPE,
                                        preexec_fn=os.setpgrp,
                                        env=env)
        output = self.runqemu.stdout

        #
        # We need the preexec_fn above so that all runqemu processes can easily be killed
        # (by killing their process group). This presents a problem if this controlling
        # process itself is killed however since those processes don't notice the death
        # of the parent and merrily continue on.
        #
        # Rather than hack runqemu to deal with this, we add something here instead.
        # Basically we fork off another process which holds an open pipe to the parent
        # and also is setpgrp. If/when the pipe sees EOF from the parent dieing, it kills
        # the process group. This is like pctrl's PDEATHSIG but for a process group
        # rather than a single process.
        #
        r, w = os.pipe()
        self.monitorpid = os.fork()
        if self.monitorpid:
            os.close(r)
            self.monitorpipe = os.fdopen(w, "w")
        else:
            # child process
            os.setpgrp()
            os.close(w)
            r = os.fdopen(r)
            x = r.read()
            os.killpg(os.getpgid(self.runqemu.pid), signal.SIGTERM)
            sys.exit(0)

        self.logger.debug("runqemu started, pid is %s" % self.runqemu.pid)
        self.logger.debug("waiting at most %s seconds for qemu pid (%s)" %
                          (self.runqemutime, time.strftime("%D %H:%M:%S")))
        endtime = time.time() + self.runqemutime
        while not self.is_alive() and time.time() < endtime:
            if self.runqemu.poll():
                if self.runqemu.returncode:
                    # No point waiting any longer
                    self.logger.debug('runqemu exited with code %d' %
                                      self.runqemu.returncode)
                    self._dump_host()
                    self.stop()
                    self.logger.debug("Output from runqemu:\n%s" %
                                      self.getOutput(output))
                    return False
            time.sleep(0.5)

        if not self.is_alive():
            self.logger.error("Qemu pid didn't appear in %s seconds (%s)" %
                              (self.runqemutime, time.strftime("%D %H:%M:%S")))
            # Dump all processes to help us to figure out what is going on...
            ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command '],
                                  stdout=subprocess.PIPE).communicate()[0]
            processes = ps.decode("utf-8")
            self.logger.debug("Running processes:\n%s" % processes)
            self._dump_host()
            self.stop()
            op = self.getOutput(output)
            if op:
                self.logger.error("Output from runqemu:\n%s" % op)
            else:
                self.logger.error("No output from runqemu.\n")
            return False

        # We are alive: qemu is running
        out = self.getOutput(output)
        netconf = False  # network configuration is not required by default
        self.logger.debug(
            "qemu started in %s seconds - qemu procces pid is %s (%s)" %
            (time.time() - (endtime - self.runqemutime), self.qemupid,
             time.strftime("%D %H:%M:%S")))
        if get_ip:
            cmdline = ''
            with open('/proc/%s/cmdline' % self.qemupid) as p:
                cmdline = p.read()
                # It is needed to sanitize the data received
                # because is possible to have control characters
                cmdline = re_control_char.sub(' ', cmdline)
            try:
                ips = re.findall("((?:[0-9]{1,3}\.){3}[0-9]{1,3})",
                                 cmdline.split("ip=")[1])
                self.ip = ips[0]
                self.server_ip = ips[1]
                self.logger.debug("qemu cmdline used:\n{}".format(cmdline))
            except (IndexError, ValueError):
                # Try to get network configuration from runqemu output
                match = re.match(
                    '.*Network configuration: ([0-9.]+)::([0-9.]+):([0-9.]+)$.*',
                    out, re.MULTILINE | re.DOTALL)
                if match:
                    self.ip, self.server_ip, self.netmask = match.groups()
                    # network configuration is required as we couldn't get it
                    # from the runqemu command line, so qemu doesn't run kernel
                    # and guest networking is not configured
                    netconf = True
                else:
                    self.logger.error(
                        "Couldn't get ip from qemu command line and runqemu output! "
                        "Here is the qemu command line used:\n%s\n"
                        "and output from runqemu:\n%s" % (cmdline, out))
                    self._dump_host()
                    self.stop()
                    return False

        self.logger.debug("Target IP: %s" % self.ip)
        self.logger.debug("Server IP: %s" % self.server_ip)

        self.thread = LoggingThread(self.log, threadsock, self.logger)
        self.thread.start()
        if not self.thread.connection_established.wait(self.boottime):
            self.logger.error("Didn't receive a console connection from qemu. "
                              "Here is the qemu command line used:\n%s\nand "
                              "output from runqemu:\n%s" % (cmdline, out))
            self.stop_thread()
            return False

        self.logger.debug("Output from runqemu:\n%s", out)
        self.logger.debug("Waiting at most %d seconds for login banner (%s)" %
                          (self.boottime, time.strftime("%D %H:%M:%S")))
        endtime = time.time() + self.boottime
        socklist = [self.server_socket]
        reachedlogin = False
        stopread = False
        qemusock = None
        bootlog = b''
        data = b''
        while time.time() < endtime and not stopread:
            try:
                sread, swrite, serror = select.select(socklist, [], [], 5)
            except InterruptedError:
                continue
            for sock in sread:
                if sock is self.server_socket:
                    qemusock, addr = self.server_socket.accept()
                    qemusock.setblocking(0)
                    socklist.append(qemusock)
                    socklist.remove(self.server_socket)
                    self.logger.debug("Connection from %s:%s" % addr)
                else:
                    data = data + sock.recv(1024)
                    if data:
                        bootlog += data
                        data = b''
                        if b' login:'******'t reach login banner in %d seconds (%s)" %
                    (self.boottime, time.strftime("%D %H:%M:%S")))
            tail = lambda l: "\n".join(l.splitlines()[-25:])
            # in case bootlog is empty, use tail qemu log store at self.msg
            lines = tail(bootlog if bootlog else self.msg)
            self.logger.debug("Last 25 lines of text:\n%s" % lines)
            self.logger.debug("Check full boot log: %s" % self.logfile)
            self._dump_host()
            self.stop()
            return False

        # If we are not able to login the tests can continue
        try:
            (status, output) = self.run_serial("root\n", raw=True)
            if re.search("root@[a-zA-Z0-9\-]+:~#", output):
                self.logged = True
                self.logger.debug("Logged as root in serial console")
                if netconf:
                    # configure guest networking
                    cmd = "ifconfig eth0 %s netmask %s up\n" % (self.ip,
                                                                self.netmask)
                    output = self.run_serial(cmd, raw=True)[1]
                    if re.search("root@[a-zA-Z0-9\-]+:~#", output):
                        self.logger.debug("configured ip address %s", self.ip)
                    else:
                        self.logger.debug(
                            "Couldn't configure guest networking")
            else:
                self.logger.debug("Couldn't login into serial console"
                                  " as root using blank password")
        except:
            self.logger.debug("Serial console failed while trying to login")
        return True
Beispiel #41
0
 def pipe(self):
     return pipe()
Beispiel #42
0
        if replication.repl_limit != 0:
            limit = '/usr/local/bin/throttle -K %d | ' % replication.repl_limit
        else:
            limit = ''
        cmd = ['/sbin/zfs', 'send', '-V']
        if replication.repl_userepl:
            cmd.append('-R')
        if last_snapshot == '':
            cmd.append(snapname)
        else:
            cmd.extend(['-I', last_snapshot, snapname])

        progressfile = '/tmp/.repl_progress_%d' % replication.id
        # subprocess.Popen does not handle large stream of data between
        # processes very well, do it on our own
        readfd, writefd = os.pipe()
        zproc_pid = os.fork()
        if zproc_pid == 0:
            os.close(readfd)
            os.dup2(writefd, 1)
            os.close(writefd)
            os.execv('/sbin/zfs', cmd)
            # NOTREACHED
        else:
            with open(progressfile, 'w') as f2:
                f2.write(str(zproc_pid))
            os.close(writefd)

        if compression == 'pigz':
            compress = '/usr/local/bin/pigz | '
            decompress = '/usr/local/bin/pigz -d | '
Beispiel #43
0
# so need link it to current position

# DEBUG = False
DEBUG = True

# stage 1: argv
if not DEBUG:
    args = ['/home/input2/input'] + ['A']*99
else:
    args = ['./input'] + ['A']*99
args[ord('A')] = ""
args[ord('B')] = "\x20\x0a\x0d"
args[ord('C')] = "12345"

# stage 2: stdio
stderr, stderw = os.pipe()

# stage 3: env
env = os.environ.copy()
env["\xde\xad\xbe\xef"] = "\xca\xfe\xba\xbe"
# print env['PATH']

# stage 4: file
f = open("\x0a", "wb")
f.write("\x00\x00\x00\x00")
f.close()

# stage 5: network
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

proc = subprocess.Popen(args, stdin=subprocess.PIPE, stderr=stderr, env=env)
Beispiel #44
0
 def pipe(self):
     r, w = pipe()
     os.make_nonblocking(r)
     os.make_nonblocking(w)
     return r, w
Beispiel #45
0
def handle1(fd):
    while True:
        buf = os.read(fd, 1024)
        if buf:
            print 'GET:%s' % buf


def handle2(fd):
    while True:
        buf = os.read(0, 1024)
        if buf:
            print 'SEND:%s' % buf
            os.write(fd, buf)


p2cr, p2cw = os.pipe()
c2pr, c2pw = os.pipe()
pid = os.fork()
if pid == 0:  # child
    os.close(p2cw)
    os.close(c2pr)
    os.dup2(p2cr, 0)
    os.dup2(c2pw, 1)
    os.execvp('python', ['python'])
    os._exit(0)
else:
    os.close(p2cr)
    os.close(c2pw)
    t = Thread(target=handle1, args=(c2pr, ))
    t.setDaemon(True)
    t.start()
Beispiel #46
0
 def __enter__(self):
     import os
     self.read_fd, self.write_fd = os.pipe()
     self.copy_fd = os.dup(self.capture_fd)
     os.dup2(self.write_fd, self.capture_fd)
     return self
Beispiel #47
0
def sndrcv(pks,
           pkt,
           timeout=None,
           inter=0,
           verbose=None,
           chainCC=0,
           retry=0,
           multi=0):
    if not isinstance(pkt, Gen):
        pkt = SetGen(pkt)

    if verbose is None:
        verbose = conf.verb
    debug.recv = plist.PacketList([], "Unanswered")
    debug.sent = plist.PacketList([], "Sent")
    debug.match = plist.SndRcvList([])
    nbrecv = 0
    ans = []
    # do it here to fix random fields, so that parent and child have the same
    all_stimuli = tobesent = [p for p in pkt]
    notans = len(tobesent)

    hsent = {}
    for i in tobesent:
        h = i.hashret()
        if h in hsent:
            hsent[h].append(i)
        else:
            hsent[h] = [i]
    if retry < 0:
        retry = -retry
        autostop = retry
    else:
        autostop = 0

    while retry >= 0:
        found = 0

        if timeout < 0:
            timeout = None

        rdpipe, wrpipe = os.pipe()
        rdpipe = os.fdopen(rdpipe)
        wrpipe = os.fdopen(wrpipe, "w")

        pid = 1
        try:
            pid = os.fork()
            if pid == 0:
                try:
                    sys.stdin.close()
                    rdpipe.close()
                    try:
                        i = 0
                        if verbose:
                            print "Begin emission:"
                        for p in tobesent:
                            pks.send(p)
                            i += 1
                            time.sleep(inter)
                        if verbose:
                            print "Finished to send %i packets." % i
                    except SystemExit:
                        pass
                    except KeyboardInterrupt:
                        pass
                    except:
                        log_runtime.exception("--- Error in child %i" %
                                              os.getpid())
                        log_runtime.info("--- Error in child %i" % os.getpid())
                finally:
                    try:
                        os.setpgrp()  # Chance process group to avoid ctrl-C
                        sent_times = [
                            p.sent_time for p in all_stimuli if p.sent_time
                        ]
                        cPickle.dump((conf.netcache, sent_times), wrpipe)
                        wrpipe.close()
                    except:
                        pass
            elif pid < 0:
                log_runtime.error("fork error")
            else:
                wrpipe.close()
                stoptime = 0
                remaintime = None
                inmask = [rdpipe, pks]
                try:
                    try:
                        while 1:
                            if stoptime:
                                remaintime = stoptime - time.time()
                                if remaintime <= 0:
                                    break
                            r = None
                            if not isinstance(pks, StreamSocket) and (
                                    arch.FREEBSD or arch.DARWIN):
                                inp, out, err = select(inmask, [], [], 0.05)
                                if len(inp) == 0 or pks in inp:
                                    r = pks.nonblock_recv()
                            else:
                                inp = []
                                try:
                                    inp, out, err = select(
                                        inmask, [], [], remaintime)
                                except IOError, exc:
                                    if exc.errno != errno.EINTR:
                                        raise
                                if len(inp) == 0:
                                    break
                                if pks in inp:
                                    r = pks.recv(MTU)
                            if rdpipe in inp:
                                if timeout:
                                    stoptime = time.time() + timeout
                                del (inmask[inmask.index(rdpipe)])
                            if r is None:
                                continue
                            ok = 0
                            h = r.hashret()
                            if h in hsent:
                                hlst = hsent[h]
                                for i, sentpkt in enumerate(hlst):
                                    if r.answers(sentpkt):
                                        ans.append((sentpkt, r))
                                        if verbose > 1:
                                            os.write(1, "*")
                                        ok = 1
                                        if not multi:
                                            del hlst[i]
                                            notans -= 1
                                        else:
                                            if not hasattr(
                                                    sentpkt, '_answered'):
                                                notans -= 1
                                            sentpkt._answered = 1
                                        break
                            if notans == 0 and not multi:
                                break
                            if not ok:
                                if verbose > 1:
                                    os.write(1, ".")
                                nbrecv += 1
                                if conf.debug_match:
                                    debug.recv.append(r)
                    except KeyboardInterrupt:
                        if chainCC:
                            raise
                finally:
                    try:
                        nc, sent_times = cPickle.load(rdpipe)
                    except EOFError:
                        warning(
                            "Child died unexpectedly. Packets may have not been sent %i"
                            % os.getpid())
                    else:
                        conf.netcache.update(nc)
                        for p, t in zip(all_stimuli, sent_times):
                            p.sent_time = t
                    os.waitpid(pid, 0)
        finally:
            if pid == 0:
                os._exit(0)

        remain = list(itertools.chain(*hsent.itervalues()))
        if multi:
            remain = [p for p in remain if not hasattr(p, '_answered')]

        if autostop and len(remain) > 0 and len(remain) != len(tobesent):
            retry = autostop

        tobesent = remain
        if len(tobesent) == 0:
            break
        retry -= 1

    if conf.debug_match:
        debug.sent = plist.PacketList(remain[:], "Sent")
        debug.match = plist.SndRcvList(ans[:])

    #clean the ans list to delete the field _answered
    if (multi):
        for s, r in ans:
            if hasattr(s, '_answered'):
                del (s._answered)

    if verbose:
        print "\nReceived %i packets, got %i answers, remaining %i packets" % (
            nbrecv + len(ans), len(ans), notans)
    return plist.SndRcvList(ans), plist.PacketList(remain, "Unanswered")
Beispiel #48
0
def webm2mp4_worker(message, url):
    """Generic process spawned every time user sends a link or a file"""
    global telegram_token
    filename = "".join([TEMP_FOLDER, random_string(), ".mp4"])

    # Tell user that we are working
    status_message = bot.reply_to(message, message_starting, parse_mode="HTML")

    # Try to download URL
    try:
        r = requests.get(url, stream=True, headers=HEADERS)
    except:
        update_status_message(status_message, error_downloading)
        return

    # Something went wrong on the server side
    if r.status_code != 200:
        update_status_message(status_message, error_wrong_code.format(r.status_code))
        return

    # Is it a webm file?
    
    if r.headers["Content-Type"] not in ALLOWED_MIME_TYPES and message.document.mime_type not in ALLOWED_MIME_TYPES:
        update_status_message(status_message, error_file_not_webm)
        return
    # Can't determine file size
    if not "Content-Length" in r.headers or not "Content-Type" in r.headers:
        update_status_message(status_message, error_no_header)
        return

    # Check file size
    webm_size = int(r.headers["Content-Length"])
    if webm_size >= MAXIMUM_FILESIZE_ALLOWED:
        update_status_message(status_message, error_huge_file)
        return

    # Create a pipe to pass downloading file to ffmpeg without delays
    pipe_read, pipe_write = os.pipe()

    # Start ffmpeg
    ffmpeg_process = subprocess.Popen(["ffmpeg",
        "-v", "error",
        "-threads", str(FFMPEG_THREADS),
        "-i", "pipe:0", # read input from stdin
        "-map", "V:0?", # select video stream
        "-map", "0:a?", # ignore audio if doesn't exist
        "-c:v", "libx264", # specify video encoder
        "-max_muxing_queue_size", "9999", # https://trac.ffmpeg.org/ticket/6375
        "-movflags", "+faststart", # optimize for streaming
        "-preset", "slow", # https://trac.ffmpeg.org/wiki/Encode/H.264#a2.Chooseapresetandtune
        "-timelimit", "900", # prevent DoS (exit after 15 min)
        "-vf", "pad=ceil(iw/2)*2:ceil(ih/2)*2", # https://stackoverflow.com/questions/20847674/ffmpeg-libx264-height-not-divisible-by-2#20848224
        filename
    ], stdin=pipe_read)

    # Download file in and pass it to ffmpeg with pipe
    try:
        threading.Thread(
            target=download_file,
            kwargs={
                "request": r,
                "pipe_write": pipe_write
            }
        ).start()
        # Initial delay to start downloading 
        time.sleep(1)
    except:
        update_status_message(status_message, error_downloading)
        # Close pipe explicitly
        os.close(pipe_read)
        return

    # While ffmpeg process is alive (i.e. is working)
    old_progress = ""
    while ffmpeg_process.poll() == None:
        try:
            output_file_size = os.stat(filename).st_size
        except FileNotFoundError:
            output_file_size = 0
        mp4_size = size(output_file_size, system=alternative)
        webm_size = size(int(r.headers["Content-Length"]), system=alternative)
        human_readable_progress = " ".join([mp4_size, "/", webm_size])
        if human_readable_progress != old_progress:
            update_status_message(status_message, message_converting.format(human_readable_progress))
            old_prpgress = human_readable_progress
        time.sleep(3)

    # Exit in case of error with ffmpeg
    if ffmpeg_process.returncode != 0:
        update_status_message(status_message, error_converting)
        # Clean up and close pipe explicitly
        rm(filename)
        os.close(pipe_read)
        return

    # Check output file size
    mp4_size = os.path.getsize(filename)
    if mp4_size >= MAXIMUM_FILESIZE_ALLOWED:
        update_status_message(status_message, error_huge_file)
        # Clean up and close pipe explicitly
        rm(filename)
        os.close(pipe_read)
        return

    # Close pipe after using
    os.close(pipe_read)

    # 1. Get video duration in seconds
    video_duration = subprocess.run(["ffprobe",
        "-v", "error",
        "-select_streams", "v:0",
        "-show_entries", "format=duration",
        "-of", "default=noprint_wrappers=1:nokey=1",
        filename
    ], stdout=subprocess.PIPE).stdout.decode("utf-8").strip()
    video_duration = round(float(video_duration))

    # 2. Get video height and width
    video_props = subprocess.run(["ffprobe",
        "-v", "error",
        "-select_streams", "v:0",
        "-show_entries","stream=width,height",
        "-of", "csv=s=x:p=0",
        filename
    ], stdout=subprocess.PIPE).stdout.decode("utf-8").strip()
    video_width, video_height = video_props.split("x")

    # 3. Take one frame from the middle of the video
    update_status_message(status_message, message_generating_thumbnail)
    thumbnail = "".join([TEMP_FOLDER, random_string(), ".jpg"])
    generate_thumbnail_process = subprocess.Popen(["ffmpeg",
        "-v", "error",
        "-i", filename,
        "-vcodec", "mjpeg",
        "-vframes", "1",
        "-an", "-f", "rawvideo",
        "-ss", str(int(video_duration/2)),
        # keep the limit of 90px height/width (Telegram API) while preserving the aspect ratio
        "-vf", "scale='if(gt(iw,ih),90,trunc(oh*a/2)*2)':'if(gt(iw,ih),trunc(ow/a/2)*2,90)'",
        thumbnail
    ])

    # While process is alive (i.e. is working)
    while generate_thumbnail_process.poll() == None:
        time.sleep(1)

    # Exit in case of error with ffmpeg
    if generate_thumbnail_process.returncode != 0:
        update_status_message(status_message, error_generating_thumbnail)
        # Clean up
        rm(filename)
        rm(thumbnail)
        return

    # Upload to Telegram
    update_status_message(status_message, message_uploading)
    mp4 = open(filename, "rb")
    thumb = open(thumbnail, "rb")
    requests.post(
        f"https://api.telegram.org/bot{telegram_token}/sendVideo",
        data={
            "chat_id": message.chat.id,
            "duration": video_duration,
            "width": video_width,
            "height": video_height,
            "reply_to_message_id": message.message_id,
            "supports_streaming": True
        },
        files=[
            ("video", (random_string()+".mp4", mp4, "video/mp4")),
            ("thumb", (random_string()+".jpg", thumb, "image/jpeg"))
        ]
    )
    bot.delete_message(message.chat.id, status_message.message_id)

    # Clean up
    mp4.close()
    thumb.close()
    rm(filename)
    rm(thumbnail)
Beispiel #49
0
from tempfile import gettempdir
from threading import Thread

from lib.common.constants import PATHS
from lib.common.exceptions import CuckooError
from lib.core.config import Config
from lib.core.startup import create_folders, init_logging
from lib.api.tracer import SyscallTracer, FilesystemTracer
from time import sleep

log = logging.getLogger()

FILES_LIST = []
DUMPED_LIST = []

PIPE = os.pipe()


def dump_file():
    """Create a copy of the given file path."""
    log.info("PLS IMPLEMENT DUMP, want to dump %s", file_path)
    pass


def dump_files():
    """Dump all the dropped files."""
    for file_path in FILES_LIST:
        dump_file(file_path)


class PipeServer(Thread):
Beispiel #50
0
def main(cscreen = None):
    ######################################################
    ##                    STARTUP                       ##
    ######################################################
    
    #default values
    arg_filter = None
    arg_address = None
    input_path = "NREL_sequence_canadian_1.csv"
    file_mode = False
    output_path = "SSTDR_waveforms.csv"
    yaml_path = 'default.yaml'
    
    #read cmd line arguments
    valid_args = ['-yaml', 'y', '-filter', '-f', '-address', '-a', '-file', '-out', '-o', '-curses', '-c', '-no-curses', '-nc']
    args = {}
    skip = False
    for i,arg in enumerate(sys.argv):
        if skip:
            skip = False
            continue #only look at args in loop
        if arg in valid_args:
            skip = True #skip next word; we use it as a value here
            if (i+1 < len(sys.argv)):
                value = sys.argv[i+1]
        if arg in ['-yaml', '-y']:
            yaml_path = value
        elif arg in ['-filter', '-f']:
            arg_filter = int(value)
        elif arg in ['-address', '-a']:
            arg_address = int(value)
        elif arg in ['-file']:
            file_mode = True
            input_path = value
        elif arg in ['-out', '-o']:
            output_path = value
        #elif arg in ['-curses', '-c']:
        #    USE_CURSES = True
        #    skip = False
        #elif arg in ['-no-curses', '-nc']:
        #    USE_CURSES = False
        #    skip = False
        
    #prepare usb sniffing
    if (arg_filter is None or arg_address is None):
        sstdr_device = usb.core.find(idVendor=0x067b, idProduct=0x2303) #constants for our SSTDR device
        if sstdr_device == None:
            print("Error: Could not automatically find SSTDR device. Either restart it or provide filter/address manually.")
            return
        arg_filter  = sstdr_device.bus
        arg_address = sstdr_device.address
    
    usb_path = "C:\\Program Files\\USBPcap\\USBPcapCMD.exe"
    usb_args = [usb_path, "-d", "\\\\.\\USBPcap" + str(arg_filter), "--devices", str(arg_address), "-o", "-"]
    
    #prepare output file for logging
    with open(output_path, "a+") as out_f:
        out_f.seek(0,0)
        first_char = out_f.read(1)
        if (first_char == ''):
            #file did not exist or is empty. write header row; set session/log index to 0
            #write header row
            out_f.write("session_number,log_number,timestamp,waveform\n")
            session_number = 0
            log_number = 0
        else:
            #file was not empty. jump almost to end, read last line, extract session index
            #"read up until start of last line" code from S.O. user Trasp: https://stackoverflow.com/questions/3346430/what-is-the-most-efficient-way-to-get-first-and-last-line-of-a-text-file/3346788
            with open(output_path, "rb") as f:
                f.seek(-2, os.SEEK_END)     # Jump to the second last byte.
                while f.read(1) != b"\n":   # Until EOL is found...
                    f.seek(-2, os.SEEK_CUR) # ...jump back the read byte plus one more.
                last = f.readline()         # Read last line as bytes.
            session_number = 1+int(chr(int.from_bytes(last.split(b',')[0],'little'))) #assumes little endian, and that session index is present in column 0 (as will be standard in the future)
            log_number = 0

    #set up scanning interface in curses (cscreen = curses screen)
    print("Opening scanner interface...")
    if not(cscreen is None):
        cscreen.clear()
        cscreen.nodelay(True)
        if (file_mode):
            cscreen.addstr(0,0,"Playing back input file: '" + input_path +"'...")
        else:
            cscreen.addstr(0,0,"Scanning on filter " + str(arg_filter) + ", address " + str(arg_address) + "...")
        cscreen.addstr(1,0,"Press 'q' to stop.")
        cscreen.addstr(3,0,"System OK.")
        cscreen.refresh()    
    else:
        print("Scanning on filter " + str(arg_filter) + ", address " + str(arg_address) + "...")
    
    if (not file_mode):
        #open USBPcap, throwing all output onto a pipe
        usb_fd_r, usb_fd_w = os.pipe()
        usbpcap_process = subprocess.Popen(usb_args, stdout=usb_fd_w)
        #start receiving usbpcap output and organizing it into packets
        usb_stream = os.fdopen(usb_fd_r, "rb")
        #set up receiver to process raw USB bytestream
        halt_threads = threading.Event()
        receiver = PcapPacketReceiver(usb_stream, loop=True, halt_event=halt_threads)
        
    #prepare deque for waveform visualization; only stores a few of the most recently received waveforms. appended entries cycle out old ones
    #larger deque -> more maximum latency between visualization and actual system state
    #smaller deque -> not sure why this would be a problem (something about losing information if packets aren't received constantly)
    wf_deque = deque(maxlen=1)
    
    #prepare to visualize waveforms
    fig = plt.figure()
    plot_window = pf.screen(title='SSTDR Correlation Waveform')
    
    ######################################################
    ##                  PYGAME SETUP                    ##
    ######################################################

    #initializing pygame
    pygame.init()
    pscreen = pygame.display.set_mode(SCREEN_SIZE)
    pygame.display.set_caption("PV Fault Scanner")

    #loading assets, preparing pre-baked surfaces
    TERMINAL_FONT = pygame.font.Font(None, 40)
    STATUS_FONT = pygame.font.Font(None, 20)

    panel_surf = pygame.image.load(os.path.join("Assets", "PV_panel_CharlesMJames_CC.jpg"))
    panel_surf = pygame.transform.scale(panel_surf, (int(panel_surf.get_width()*PANEL_SCALE), int(panel_surf.get_height()*PANEL_SCALE)))
    panel_rect = panel_surf.get_rect()

    grass_surf = pygame.image.load(os.path.join("Assets", "grass.png"))
    grass_rect = grass_surf.get_rect()

    hazard_surf = pygame.image.load(os.path.join("Assets", "hazard.png"))
    hazard_rect = hazard_surf.get_rect()

    bg_surf = pygame.Surface(pscreen.get_size())
    bg_surf.convert()
    bg_rect = bg_surf.get_rect()
    bg_surf.fill(BG_COLOR)
    """
    for r in range(int(SCREEN_Y / grass_rect.h+1)):
        for c in range(int(SCREEN_X / grass_rect.w+1)):
            bg_surf.blit(grass_surf, grass_rect)
            grass_rect.move_ip(grass_rect.w,0)
        grass_rect.x = 0
        grass_rect.move_ip(0, grass_rect.h)
    """
    line_surf = pygame.Surface((SCREEN_X, BORDER_WIDTH))
    line_surf.fill(COLOR_ORANGE)
    line_rect = line_surf.get_rect()
    line_rect.y = VISUAL_Y - BORDER_WIDTH - int(BORDER_PADDING/2)
    bg_surf.blit(line_surf, line_rect)
    line_surf.fill(COLOR_BLUE)
    line_rect.move_ip(0, BORDER_WIDTH + BORDER_PADDING)
    bg_surf.blit(line_surf, line_rect)

    text_surf = STATUS_FONT.render("Scanning at 24MHz...", True, COLOR_WHITE)
    text_rect = text_surf.get_rect()
    text_rect.move_ip(3,3)
    bg_surf.blit(text_surf, text_rect)

    text_surf = STATUS_FONT.render("Selected Array Layout: " + yaml_path, True, COLOR_WHITE)
    text_rect = text_surf.get_rect()
    text_rect.x = 3
    text_rect.bottom = VISUAL_Y - BORDER_WIDTH - int(0.5*BORDER_PADDING) - 3
    bg_surf.blit(text_surf, text_rect)

    #load panel layout
    panel_layout, panel_ds, panel_length = load_panel_layout(yaml_path)
    panel_cols = panel_rows = 0
    try:
        N = len(panel_ds)
        H = int(N/2)-1
        if (panel_layout['layout'] == 'loop'):
            panel_rows = 2
            panel_cols = int(N/2+0.5)
            r = 0
            PANEL_COORDS = [(c*(PANEL_PADDING[0] + panel_rect.w), r*(PANEL_PADDING[1] + panel_rect.h)) for c in range(0,H+1)]
            if (len(panel_ds)%2):
                r = 0.5
                PANEL_COORDS.append(((H+1)*(PANEL_PADDING[0] + panel_rect.w), r*(PANEL_PADDING[1] + panel_rect.h)))
            r = 1
            PANEL_COORDS = PANEL_COORDS + [(c*(PANEL_PADDING[0] + panel_rect.w), r*(PANEL_PADDING[1] + panel_rect.h)) for c in range(H,-1,-1)]
        
        elif(panel_layout['layout'] == 'home-run'):
            panel_rows = 1
            panel_cols = N
            r = 0
            PANEL_COORDS = [(c*(PANEL_PADDING[0] + panel_rect.w), r*(PANEL_PADDING[1] + panel_rect.h)) for c in range(0,N)]
        else:
            raise Exception("Error: unknown layout field in layout yaml file.")
    except:
        print("Error: invalid layout yaml file.")
        return
    
    ARRAY_SIZE = (panel_cols*(panel_rect.w + PANEL_PADDING[0]), panel_rows*(panel_rect.h + PANEL_PADDING[1]))
    array_surf = pygame.Surface(ARRAY_SIZE, pygame.SRCALPHA)
    array_surf.convert()
    
    for p in PANEL_COORDS:
        panel_rect.topleft = p
        array_surf.blit(panel_surf, panel_rect)

    array_rect = array_surf.get_rect()
    array_rect.center = (int(SCREEN_X*PANEL_SCREEN_X_RATIO), int(VISUAL_Y/2))

    WIRE_COORDS = []
    for p in PANEL_COORDS:
        panel_rect.topleft = p
        WIRE_COORDS.append((panel_rect.center[0] + array_rect.topleft[0], panel_rect.center[1] + array_rect.topleft[1]))
    WIRE_COORDS.insert(0,(0,WIRE_COORDS[0][1]))
    WIRE_COORDS.append((0,WIRE_COORDS[-1][1]))
    pygame.draw.lines(bg_surf, WIRE_COLOR, False, WIRE_COORDS, WIRE_WIDTH)

    term_surf = pygame.Surface((SCREEN_X, TERMINAL_Y - int(BORDER_PADDING/2) - BORDER_WIDTH))
    term_surf.fill(TERMINAL_COLOR)
    term_rect = term_surf.get_rect()
    term_rect.bottom = SCREEN_Y
    
    button_text_p = 5 #padding
    button_outer_p = 5
    button_size = list(STATUS_FONT.size("Measure"))
    button_size[0] += 2*button_text_p
    button_size[1] += 2*button_text_p
    button_rect = pygame.Rect(SCREEN_X-button_size[0]-button_outer_p, button_outer_p, button_size[0], button_size[1])
    button_surf = pygame.Surface(button_size)
    button_text_surf = STATUS_FONT.render("Measure", True, COLOR_WHITE)
    
    ######################################################
    ##              FAULT DETECTION SETUP               ##
    ######################################################

    detector = fault_detection.Detector(fault_detection.METHOD_NONE)
    fault = (fault_detection.FAULT_NONE, 0)
    terminal_waveform = None
    logging = False #if this is true, measured waveforms will be written to a file
    measurement_counter = 0
    
    first_timestamp = None
    first_time_played = None
    input_row_index = 0
    if file_mode:
        input_data = fault_detection.read_csv_ungrouped(input_path)
    
    ######################################################
    ##                      LOOP                        ##
    ######################################################
    
    #set up threads:
    #first child thread: receives and interprets packets using receiver.run()
    with ThreadPoolExecutor(max_workers=3) as executor:
        if not file_mode:
            rec_thread = executor.submit(receiver.run)
        
        valid_waveform_prefix = b'\xaa\xaa\xaa\xad\x00\xbf' #valid waveform regions start with this pattern
        valid_waveform_suffix = 253 #valid waveform regions end with this pattern
        
        payloadString = b''
        byteCount = 0
        WAVEFORM_BYTE_COUNT = 199 #every waveform region contains 199 payload bytes
        
        try:
            while(True):
                #take packet from Q, process in some way
                if file_mode:
                    #TODO change this to whatever the baseline index ought to be
                    if input_row_index == 12:
                        detector.set_baseline(input_data[input_row_index][3:])
                    if input_row_index == 0:
                        first_time_played = time.time()
                        first_timestamp = input_data[0][2]
                    if True:#time.time() - first_time_played >= input_data[input_row_index+1][2] - first_timestamp:
                        input_row_index = input_row_index + 1
                    wf_deque.append(np.array(input_data[input_row_index][3:]))
                    time.sleep(0.25)
                """
                goal is to identify shape of data in intermittent test, and have this
                code recognize when a sequence of packet blocks represents a
                correlation waveform. This waveform should be calculated from payload
                bytes, and either shown for visualization (pyplot?) or fed to matlab
                for processing (which is the ultimate goal).
                """
                if not file_mode and receiver.q.empty() == False:
                    pBlock = receiver.q.get()
                    #commented out because this printing was very very slow, and ruined realtime
                    #if not(cscreen is None):
                        #cscreen.addstr(5,0,"Received packet at timestamp: " + str(pBlock.ts_sec + 0.000001*pBlock.ts_usec)) #show some packet data so it's clear the scanner is working
                        #cscreen.refresh()
                    
                    #if received packet may be in a waveform region of the stream:
                    #criteria: input (to host) from endpoint 3 and function == URB_FUNCTION_BULK_OR_INTERRUPT_TRANSFER
                    if (pBlock.packet.endpoint == 0x83 and pBlock.packet.function == 0x09):
                        #if block has a payload:
                        p = pBlock.packet.payload
                        l = len(p)
                        if (l > 0):
                            payloadString = payloadString + p
                            byteCount = byteCount + l
                            if ((l==1 and p==0xaa) or (l>1 and p[0] == 0xaa) or (b'\xaa' in p)):
                                if not cscreen is None and DEBUG_VERIFICATION:
                                    cscreen.addstr(13,0,"Received start of prefix at timestamp: " + str(pBlock.ts_sec + 0.000001*pBlock.ts_usec))
                                    cscreen.addstr(14,4,"Starting payload: " + str(p))
                                    cscreen.addstr(15,4,"New payload string: " + str(payloadString))
                                    cscreen.refresh()
 
                            #for the first few bytes, compare it to a prefix pattern that all valid waveforms start with.
                            if (VERIFY_WAVEFORMS and byteCount <= len(valid_waveform_prefix)):
                                #TODO: currently assumes prefix bytes arrive one at a time (ie not in the same payload). this is OK the vast majority of the time.
                                if (p[0] != valid_waveform_prefix[byteCount-1]):
                                    #the current waveform is not valid. throw it out.
                                    if not(cscreen is None):
                                        cscreen.addstr(10,0,"Invalid waveform at timestamp: " + str(pBlock.ts_sec + 0.000001*pBlock.ts_usec))
                                        cscreen.addstr(11,4,"Prefix string: " + str(payloadString) + "; Byte count: " + str(byteCount))
                                        if (byteCount > 1):
                                            cscreen.addstr(12,4,"Last long invalid prefix: "+ str(payloadString) + "; Byte count: " + str(byteCount))
                                        cscreen.refresh()
                                    else:
                                        print("Invalid waveform at timestamp: " + str(pBlock.ts_sec + 0.000001*pBlock.ts_usec))
                                        print("Prefix string: " + str(payloadString)+'\n')
                                    if ((l==1 and p==0xaa) or (l>1 and p[0] == 0xaa) or (b'\xaa' in p)): #if the byte that ruined everything may be the start of a valid region, keep it
                                        payloadString = b'' + p
                                        byteCount = l
                                    else:
                                        payloadString = b''
                                        byteCount = 0
                            elif (byteCount >= WAVEFORM_BYTE_COUNT):
                                #perform processing on raw waveform
                                wf = process_waveform_region(payloadString,cscreen)
                                wf_deque.append(wf)
                                if not(cscreen is None):
                                    #show that we've received a waveform
                                    cscreen.addstr(7,0,"Received waveform at timestamp: " + str(pBlock.ts_sec + 0.000001*pBlock.ts_usec))
                                    cscreen.refresh()                                                     
                                #prepare to receive next waveform region                            
                                payloadString = b''
                                byteCount = 0
                    elif (byteCount > 0):
                        payloadString = b''
                        byteCount = 0
                
                elif len(wf_deque) > 0:
                    #q was empty, we have some extra time to visualize things
                    wf = np.array(wf_deque.popleft())
                    if (logging):
                        #write row with session index, log index, timestamp, and measured waveform.
                        with open("SSTDR_waveforms.csv", "a") as f:
                            f.write(str(session_number)+","+str(log_number)+","+str(pBlock.ts_sec + 0.000001*pBlock.ts_usec)+","+str(list(wf))[1:-1]+'\n')
                        if measurement_counter > 0:
                            measurement_counter -= 1
                            if measurement_counter == 0:
                                logging = False                    
                                log_number += 1
                    
                    ###################################################################################################################################
                    #       PYFORMULAS: visualize waveform
                    ###################################################################################################################################
                    #some code from https://stackoverflow.com/questions/40126176/fast-live-plotting-in-matplotlib-pyplot
                    plt.clf()
                    plt.xlabel("Distance (feet)")
                    plt.ylabel("Correlation With Reflection")
                    plt.gcf().subplots_adjust(left=0.15)
                
                    if detector.raw_baseline is None:
                        #plt.plot(fault_detection.FEET_VECTOR, wf_i/max(abs(wf_i)))
                        plt.plot(fault_detection.SPLINE_FEET_VECTOR-detector.spline_feet_offset, detector.last_processed_waveform)
                        #plt.ylim((-1,1))
                        plt.ylim((-(2**15), 2**15))
                        plt.xlim((fault_detection.SPLINE_FEET_VECTOR[0]-detector.spline_feet_offset, fault_detection.SPLINE_FEET_VECTOR[-1]-detector.spline_feet_offset))
                    else:
                        #plot BLS
                        bls = detector.last_processed_waveform - detector.processed_baseline
                        max_f = fault_detection.SPLINE_FEET_VECTOR[np.argmax(bls)]-detector.spline_feet_offset
                        plt.plot(fault_detection.SPLINE_FEET_VECTOR-detector.spline_feet_offset, bls)
                        plt.plot([max_f, max_f], [-750, 750])
                        #plt.ylim((-1,1))
                        plt.ylim((-(750), 750))
                        plt.xlim((fault_detection.SPLINE_FEET_VECTOR[0]-detector.spline_feet_offset, fault_detection.SPLINE_FEET_VECTOR[-1]-detector.spline_feet_offset))
                    
                    fig.canvas.draw()
                    image = np.frombuffer(fig.canvas.tostring_rgb(), dtype=np.uint8)
                    image = image.reshape(fig.canvas.get_width_height()[::-1] + (3,))
                    plot_window.update(image)
                    
                    ###################################################################################################################################
                    #       PYGAME: fault visualization & event queue
                    ###################################################################################################################################
                    fault = detector.detect_faults(wf)
                    
                    for event in pygame.event.get():
                        if event.type == pygame.QUIT or (event.type == pygame.KEYDOWN and event.key == pygame.K_ESCAPE):
                            pygame.display.quit()
                            pygame.quit()
                        if event.type == pygame.MOUSEBUTTONUP:
                            if (button_rect.collidepoint(pygame.mouse.get_pos())):
                                #log for 10 samples. "window capture"
                                logging = True
                                measurement_counter = 10 #counts down to zero
                        if event.type == pygame.KEYDOWN:
                            if event.key == pygame.K_b:
                                detector.set_baseline(wf)#set baseline
                            elif event.key == pygame.K_l:
                                #START/STOP LOGGING.
                                if logging:
                                    #stop logging; increment log index.
                                    log_number = log_number+1
                                    measurement_counter = 0
                                    logging = False
                                else:
                                    #start logging
                                    logging = True
                            elif event.key == pygame.K_a:
                                terminal_waveform = wf #record waveform representing a disconnect at the panel terminal
                            elif event.key == pygame.K_t:
                                if (terminal_waveform is None): terminal_waveform = wf
                                detector.set_terminal(terminal_waveform)#set terminal points based on recorded terminal waveform and current BLSDT
                            elif event.key == pygame.K_LEFT:
                                detector.bls_deviation_thresh = detector.bls_deviation_thresh - 0.01 #adjust deviation threshold for peak location
                            elif event.key == pygame.K_RIGHT:
                                detector.bls_deviation_thresh = detector.bls_deviation_thresh + 0.01
                            elif event.key == pygame.K_w:
                                #log for 10 samples. "window capture"
                                logging = True
                                measurement_counter = 10 #counts down to zero
                                
                    #per-frame logic here
                    is_fault = (fault[0] != fault_detection.FAULT_NONE)
                    fault_d_f = fault[1]
                    
                    if (is_fault):
                        d = 0
                        px = WIRE_COORDS[0][0]
                        py = WIRE_COORDS[0][1]
                        hazard_point = WIRE_COORDS[-1]
                        
                        #determine which panel the fault is AFTER
                        for i in range(len(panel_ds)):
                            if (panel_ds[i] > fault_d_f):
                                break                        
                        #get the distance from the SSTDR positive lead to the pre-fault point ('point' being a point in WIRE_COORDS)
                        if i == 0:
                            pre_d = 0
                        else:
                            pre_d = panel_ds[i-1]
                        #get the distance from the SSTDR positive lead to the post-fault point
                        if i == len(panel_ds):
                            post_d = panel_ds[-1] + panel_layout['home_cable_length'] #point in feet at final SSTDR terminal
                        else:
                            post_d = panel_ds[i]          
                        #get PIXEL locations of pre-fault and post-fault points, then calculate PIXEL location of fault point
                        pre_x, pre_y = WIRE_COORDS[i] #WIRE COORDS has an extra point at i=0 (where x=0), so this chooses the point of the panel/terminal BEFORE the fault
                        post_x, post_y = WIRE_COORDS[i+1] #certainly safe; WIRE_COORDS has two more points than PANEL_COORDS. chooses the point AFTER the fault
                        hsr = (fault_d_f - pre_d)/(post_d - pre_d) #hazard step ratio: ratio at which the fault lies in between post point and pre point, s.t. fault_d = pre_d + hsr*(post_d - pre_d)
                        step = ((post_x-pre_x)**2 + (post_y-pre_y)**2)**0.5 #distance IN PIXELS between post and pre points
                        hazard_rect.center = (pre_x + hsr*(post_x-pre_x), pre_y + hsr*(post_y-pre_y))
                        
                        #subtract from distance to account for panel length; only want to report cable length
                        fault_cable_location = fault_d_f - panel_length*i
                        
                        fault_name = fault_detection.get_fault_name(fault[0])
                        fault_text_surf = TERMINAL_FONT.render(fault_name + " located at " + str(round(fault_cable_location,3)) + " feet", True, TEXT_COLOR)
                    else:
                        fault_text_surf = TERMINAL_FONT.render("System OK", True, TEXT_COLOR)
                    fault_text_rect = fault_text_surf.get_rect()
                    fault_text_rect.center = term_rect.center
                    
                    #param_text_surf = STATUS_FONT.render("BLS deviation threshold:" + str(detector.bls_deviation_thresh), True, COLOR_WHITE)
                    #param_text_surf = STATUS_FONT.render("LPF Cutoff Frequency: 6 MHz", True, COLOR_WHITE) #TODO don't hard code this, allow for live control of cutoff frequency
                    param_text_surf = STATUS_FONT.render("Current Log Number: "+str(log_number), True, COLOR_WHITE)
                    param_text_rect = param_text_surf.get_rect()
                    param_text_rect.bottomright = (SCREEN_X-3, VISUAL_Y - BORDER_WIDTH - int(0.5*BORDER_PADDING) - 3)
                    
                    logging_string = "Logging to 'SSTDR_waveforms.csv'..." if logging else "Not logging."
                    logging_text_surf = STATUS_FONT.render(logging_string, True, COLOR_WHITE)
                    logging_text_rect = logging_text_surf.get_rect()
                    logging_text_rect.bottomright = param_text_rect.topright
                    
                    #buttons: fill with color depending on context
                    #for now just fill orange
                    if (button_rect.collidepoint(pygame.mouse.get_pos())):
                        button_surf.fill(COLOR_ORANGE)
                    else:
                        button_surf.fill(COLOR_BLUE)
                    button_surf.blit(button_text_surf,(button_text_p,button_text_p))
                    
                    #drawing
                    pscreen.blit(bg_surf, bg_rect)
                    pscreen.blit(term_surf, term_rect)
                    pscreen.blit(fault_text_surf, fault_text_rect)
                    pscreen.blit(param_text_surf, param_text_rect)
                    pscreen.blit(logging_text_surf, logging_text_rect)
                    pscreen.blit(array_surf, array_rect)
                    pscreen.blit(button_surf, button_rect)
                    if (is_fault):
                        pscreen.blit(hazard_surf, hazard_rect)
                    pygame.display.flip()
                
                ###################################################################################################################################
                #       CURSES: Check for quit
                ###################################################################################################################################
                if not(cscreen is None):
                    c = cscreen.getch()
                    if (c == ord('q')):
                        cscreen.addstr(0,0,"Quitting: Terminating scanner...")
                        cscreen.refresh()                
                        usbpcap_process.terminate()                
                        cscreen.addstr(0,0, "Stopped scanner. Waiting for threads...")
                        cscreen.refresh()
                        receiver.halt()
                        #while(rec_thread.running()):
                        #    pass
                        usb_stream.close()
                        #executor.shutdown() #performed implicitly by "with" statement
                        cscreen.addstr(0,0, "Finished. Exiting...")
                        break
        except:
            print("Exception Occurred:")
            print('='*40)
            traceback.print_exc(file=sys.stdout)
            print('='*40)
            
    print("All done. :)")
Beispiel #51
0
def DebugPipe():
    fd1, fd2 = os.pipe()
    c1 = DebugConnection(fd1, writable=False)
    c2 = DebugConnection(fd2, readable=False)
    return c1, c2
Beispiel #52
0
 def __init__(self):
     self._rfd, self._wfd = os.pipe()
     self._set = False
     self._forever = False
     self._closed = False
Beispiel #53
0
def _pipe_cloexec() -> Tuple[int, int]:
    r, w = os.pipe()
    set_close_exec(r)
    set_close_exec(w)
    return r, w
Beispiel #54
0
def cleanup(fcn=None, *args, **kwargs):
    # We setup a safe procedure here to ensure that the
    # child will receive a SIGTERM when the parent exits.
    # This is used to cleanup all sorts of things once the
    # VM finishes automatically (like removing file trees,
    # removing tap devices, kill dnsmasq, etc.).
    libc = ctypes.CDLL("libc.so.6")

    if fcn is None:
        # This will only exit when the parent dies,
        # we will not run any function. This will be
        # used generally as the subprocess preexec_fn.
        child_pid = 0
        parent_pid = os.getppid()
    else:
        # Open a pipe to notify the parent when we're
        # ready to handle parent death and run the fcn.
        rpipe, wpipe = os.pipe()

        # Fork a child process.
        # This child process will execute the given code
        # when its parent dies. It's normally used inline.
        child_pid = os.fork()
        parent_pid = os.getppid()

        if child_pid == 0:
            os.close(rpipe)
        else:
            # Wait for the child to finish setup.
            # When it writes to its end of the pipe,
            # we know that it is prepared to handle
            # parent death and run the function.
            os.close(wpipe)
            os.read(rpipe, 1)
            os.close(rpipe)

    if child_pid == 0:
        # Set P_SETSIGDEATH to SIGTERM.
        libc.prctl(1, signal.SIGTERM)

        # Did we catch a race above, where we've
        # missed the re-parenting to init?
        if os.getppid() != parent_pid:
            os.kill(os.getpid(), signal.SIGTERM)

        # Are we finished?
        # In the case of not having a function to
        # execute, we simply return control. This is
        # a pre-exec hook for subprocess, for eaxample.
        if fcn is None:
            return

        # Close descriptors.
        # (Make sure we don't close our pipe).
        for fd in range(3, os.sysconf("SC_OPEN_MAX")):
            try:
                if fd != wpipe:
                    os.close(fd)
            except OSError:
                pass

        # Eat a signal.
        def squash(*args):
            pass

        # Suppress SIGINT. We'll get it when the user
        # hits Ctrl-C, which may be before our parent dies.
        signal.signal(signal.SIGINT, squash)

        def interrupt(*args):
            # Temporarily suppress SIGTERM. We'll enable it
            # once we are ready to wait (and recheck races).
            signal.signal(signal.SIGTERM, squash)
            thread.exit()

        # Temporarily supress SIGTERM, we do this until
        # it's re-enabled in the main wait loop below.
        signal.signal(signal.SIGTERM, squash)

        # Notify that we are ready to go.
        os.write(wpipe, 'o')
        os.close(wpipe)

        while True:
            try:
                # Get ready to receive our SIGTERM.
                # NOTE: When we receive the exception,
                # it will automatically be suppressed.
                signal.signal(signal.SIGTERM, interrupt)

                # Catch our race condition.
                if os.getppid() != parent_pid:
                    break

                # Wait for a signal.
                signal.pause()
            except (SystemExit, KeyboardInterrupt):
                continue

        try:
            fcn(*args, **kwargs)
        except:
            # We eat all exceptions from the
            # cleanup function. If the user wants
            # to generate any output, they may --
            # however by default we silence it.
            pass
        os._exit(0)
Beispiel #55
0
    def TouchFileAsUser(self,user,file_dir,file_dir_orig):
        #file_dir_orig is needed to access the "Warn" or "Offline" information for the file/directory in question from the config file when variable substitution has taken place
        # Define the child var
        child = 0
        user_data = None

        pbs.logmsg(pbs.EVENT_DEBUG3,"User name: %s\tFile dir: %s"%(user,file_dir))
        try:
            #user_data = getpwnam(self.user)
            user_data = getpwnam(user)

            if file_dir.find('<user_home>') != -1:
                file_dir = file_dir.replace('<user_home>',user_data[5])

            pbs.logmsg(pbs.EVENT_DEBUG3,"User name: %s\tDir to write to: %s"% \
                                 (user_data[4],file_dir))

#This is a special case where the user account does not exist on the node.  Offlining here is a good alternative to the job failing to run 20 times and being held, but it can be changed if desired
        except KeyError:
            pbs.logmsg(pbs.EVENT_DEBUG,"Unable to find user: %s"%user)
            #
            return ['Offline','unable to find user: %s'%user]

        # Fork the process for touching a file as the user
        r,w = os.pipe()

        pid = os.fork()
        pbs.logmsg(pbs.EVENT_DEBUG3,"pid: %d"%pid)

        if pid:
            # We are the parent
            os.close(w)

            r = os.fdopen(r) # turn r into a file object

            child = pid

            pbs.logmsg(pbs.EVENT_DEBUG3,"Ready to read from the child process: %d"%pid)
            lines = r.read()

            pbs.logmsg(pbs.EVENT_DEBUG3,lines)
            # Wait for the child process to complete
            os.waitpid(child,0)

            # Close the pipes
            r.close()

            # Check to see if the file was successfully touched
            if lines.find('Successfully touched file') == -1 or lines.find('Failed to remove file') != -1:
                pbs.logmsg(pbs.EVENT_DEBUG3,"Failed to touch/remove file in %s as %s"%(file_dir,user))
                return [self.nhc_cfg["as_user_operations"]["touch_files"][file_dir_orig][1],'Failed to touch/remove file for %s in %s'%(user,file_dir)]
            else:
                pbs.logmsg(pbs.EVENT_DEBUG3,"Successfully touched and removed file for %s in %s"%(user,file_dir))

        else:
            try:
                # Close the reading pipe
                os.close(r)

                # Turn w into a file object
                w = os.fdopen(w,'w')

                # Switch to the user
                w.write("Ready to switch to user: %s\tuid: %s\n"%(user,user_data[2]))
                os.setuid(user_data[2])

                # Change to the user home dir
                w.write("Changing dir to: %s\n"%(file_dir))
                if os.path.isdir(file_dir):
                    os.chdir(file_dir)

                    # Touch a file in the user's home directory
                    touch_file_name = "__user_%s_jobid_%s_host_%s_pbs_test.txt"%(user,self.job_id,self.host)
                    w.write("Ready to touch file: %s\n"%(touch_file_name))
                    touchFileSuccess = self.TouchFile(touch_file_name)

                    if touchFileSuccess:
                        w.write("Successfully touched file\n")

                    try:
                        os.remove(touch_file_name)
                    except OSError:
                        w.write("Failed to remove file: %s"%touch_file_name)
                    except Exception, e:
                        w.write("Remove file exception: %s\n"%(e))
                else:
Beispiel #56
0
    def __enter__(self):
        if self._active:
            raise RuntimeError("Can't re-enter the same log_output!")

        if self.file_like is None:
            raise RuntimeError(
                "file argument must be set by either __init__ or __call__")

        # set up a stream for the daemon to write to
        self.log_file = FileWrapper(self.file_like)

        # record parent color settings before redirecting.  We do this
        # because color output depends on whether the *original* stdout
        # is a TTY.  New stdout won't be a TTY so we force colorization.
        self._saved_color = tty.color._force_color
        forced_color = tty.color.get_color_when()

        # also record parent debug settings -- in case the logger is
        # forcing debug output.
        self._saved_debug = tty._debug

        # OS-level pipe for redirecting output to logger
        read_fd, write_fd = os.pipe()

        read_multiprocess_fd = MultiProcessFd(read_fd)

        # Multiprocessing pipe for communication back from the daemon
        # Currently only used to save echo value between uses
        self.parent_pipe, child_pipe = multiprocessing.Pipe()

        # Sets a daemon that writes to file what it reads from a pipe
        try:
            # need to pass this b/c multiprocessing closes stdin in child.
            input_multiprocess_fd = None
            try:
                if sys.stdin.isatty():
                    input_multiprocess_fd = MultiProcessFd(
                        os.dup(sys.stdin.fileno()))
            except BaseException:
                # just don't forward input if this fails
                pass

            with replace_environment(self.env):
                self.process = multiprocessing.Process(
                    target=_writer_daemon,
                    args=(input_multiprocess_fd, read_multiprocess_fd,
                          write_fd, self.echo, self.log_file, child_pipe,
                          self.filter_fn))
                self.process.daemon = True  # must set before start()
                self.process.start()

        finally:
            if input_multiprocess_fd:
                input_multiprocess_fd.close()
            read_multiprocess_fd.close()

        # Flush immediately before redirecting so that anything buffered
        # goes to the original stream
        sys.stdout.flush()
        sys.stderr.flush()

        # Now do the actual output redirection.
        self.use_fds = _file_descriptors_work(sys.stdout, sys.stderr)
        if self.use_fds:
            # We try first to use OS-level file descriptors, as this
            # redirects output for subprocesses and system calls.

            # Save old stdout and stderr file descriptors
            self._saved_stdout = os.dup(sys.stdout.fileno())
            self._saved_stderr = os.dup(sys.stderr.fileno())

            # redirect to the pipe we created above
            os.dup2(write_fd, sys.stdout.fileno())
            os.dup2(write_fd, sys.stderr.fileno())
            os.close(write_fd)

        else:
            # Handle I/O the Python way. This won't redirect lower-level
            # output, but it's the best we can do, and the caller
            # shouldn't expect any better, since *they* have apparently
            # redirected I/O the Python way.

            # Save old stdout and stderr file objects
            self._saved_stdout = sys.stdout
            self._saved_stderr = sys.stderr

            # create a file object for the pipe; redirect to it.
            pipe_fd_out = os.fdopen(write_fd, 'w')
            sys.stdout = pipe_fd_out
            sys.stderr = pipe_fd_out

        # Unbuffer stdout and stderr at the Python level
        if not self.buffer:
            sys.stdout = Unbuffered(sys.stdout)
            sys.stderr = Unbuffered(sys.stderr)

        # Force color and debug settings now that we have redirected.
        tty.color.set_color_when(forced_color)
        tty._debug = self.debug

        # track whether we're currently inside this log_output
        self._active = True

        # return this log_output object so that the user can do things
        # like temporarily echo some output.
        return self
Beispiel #57
0
 def setUpClass(cls):
     """
     Perform class setup before running the testcase
     Remove shared memory files, start vpp and connect the vpp-api
     """
     gc.collect()  # run garbage collection first
     random.seed()
     if not hasattr(cls, 'logger'):
         cls.logger = getLogger(cls.__name__)
     else:
         cls.logger.name = cls.__name__
     cls.tempdir = tempfile.mkdtemp(
         prefix='vpp-unittest-%s-' % cls.__name__)
     cls.stats_sock = "%s/stats.sock" % cls.tempdir
     cls.file_handler = FileHandler("%s/log.txt" % cls.tempdir)
     cls.file_handler.setFormatter(
         Formatter(fmt='%(asctime)s,%(msecs)03d %(message)s',
                   datefmt="%H:%M:%S"))
     cls.file_handler.setLevel(DEBUG)
     cls.logger.addHandler(cls.file_handler)
     cls.shm_prefix = os.path.basename(cls.tempdir)
     os.chdir(cls.tempdir)
     cls.logger.info("Temporary dir is %s, shm prefix is %s",
                     cls.tempdir, cls.shm_prefix)
     cls.setUpConstants()
     cls.reset_packet_infos()
     cls._captures = []
     cls._zombie_captures = []
     cls.verbose = 0
     cls.vpp_dead = False
     cls.registry = VppObjectRegistry()
     cls.vpp_startup_failed = False
     cls.reporter = KeepAliveReporter()
     # need to catch exceptions here because if we raise, then the cleanup
     # doesn't get called and we might end with a zombie vpp
     try:
         cls.run_vpp()
         cls.reporter.send_keep_alive(cls)
         cls.vpp_stdout_deque = deque()
         cls.vpp_stderr_deque = deque()
         cls.pump_thread_stop_flag = Event()
         cls.pump_thread_wakeup_pipe = os.pipe()
         cls.pump_thread = Thread(target=pump_output, args=(cls,))
         cls.pump_thread.daemon = True
         cls.pump_thread.start()
         if cls.debug_gdb or cls.debug_gdbserver:
             read_timeout = 0
         else:
             read_timeout = 5
         cls.vapi = VppPapiProvider(cls.shm_prefix, cls.shm_prefix, cls,
                                    read_timeout)
         if cls.step:
             hook = StepHook(cls)
         else:
             hook = PollHook(cls)
         cls.vapi.register_hook(hook)
         cls.wait_for_stats_socket()
         cls.statistics = VPPStats(socketname=cls.stats_sock)
         try:
             hook.poll_vpp()
         except VppDiedError:
             cls.vpp_startup_failed = True
             cls.logger.critical(
                 "VPP died shortly after startup, check the"
                 " output to standard error for possible cause")
             raise
         try:
             cls.vapi.connect()
         except Exception:
             try:
                 cls.vapi.disconnect()
             except Exception:
                 pass
             if cls.debug_gdbserver:
                 print(colorize("You're running VPP inside gdbserver but "
                                "VPP-API connection failed, did you forget "
                                "to 'continue' VPP from within gdb?", RED))
             raise
     except Exception:
         try:
             cls.quit()
         except Exception:
             pass
         raise
Beispiel #58
0
def image_search(settings, **spiderargs):
    from .spiders import crawler_setup

    image_url = spiderargs.get('image_url')
    image_data = spiderargs.get('image_data')
    if image_url:
        logger.info('Image-searching for %s' % image_url)
    elif image_data:
        logger.info('Image-searching for (image data)')
    else:
        return

    pipein, pipeout = os.pipe()
    pid = os.fork()
    if pid < 0:
        raise OSError('Forking child process failed.')

    if pid == 0: # child process
        os.close(pipein)
        writer = os.fdopen(pipeout, 'wb')
        statuscode = crawler_setup(settings, writer=writer, **spiderargs)
        writer.flush()
        writer.close()
        if not statuscode:
            statuscode = 0
        os._exit(int(statuscode))
        return # finis

    # parent process
    os.close(pipeout)
    reader = os.fdopen(pipein, 'rb')

    results = {}
    while True:
        # simple line-based protocol
        data = reader.readline()[:-1]
        if not data or len(data) == 0:
            break

        result = None
        try:
            result = json.loads(data)
        except ValueError:
            logger.error('Error decoding Spider data: %s' % (data,))
        if not result:
            continue
        # result must have a search provider
        if not 'provider' in result:
            continue # should not happen

        result = SearchResultItem(result)

        provider = result['provider']
        if not provider in results:
            results[provider] = []

        results[provider].append(result)

    pid, status = os.waitpid(pid, 0)
    reader.close()

    # do not forget about empty results
    providers = ['KarmaDecay', 'Yandex', 'Bing', 'Tineye', 'Google']
    for provider in providers:
        if not provider in results:
            results[provider] = []

    # sort (by provider) for constant key order
    results = OrderedDict(sorted(results.items()))
    return results
    def test_openFileDescriptors(self):
        """
        Processes spawned with spawnProcess() close all extraneous file
        descriptors in the parent.  They do have a stdin, stdout, and stderr
        open.
        """

        # To test this, we are going to open a file descriptor in the parent
        # that is unlikely to be opened in the child, then verify that it's not
        # open in the child.
        source = networkString("""
import sys
sys.path.insert(0, '{0}')
from twisted.internet import process
sys.stdout.write(repr(process._listOpenFDs()))
sys.stdout.flush()""".format(twistedRoot.path))

        r, w = os.pipe()
        self.addCleanup(os.close, r)
        self.addCleanup(os.close, w)

        # The call to "os.listdir()" (in _listOpenFDs's implementation) opens a
        # file descriptor (with "opendir"), which shows up in _listOpenFDs's
        # result.  And speaking of "random" file descriptors, the code required
        # for _listOpenFDs itself imports logger, which imports random, which
        # (depending on your Python version) might leave /dev/urandom open.

        # More generally though, even if we were to use an extremely minimal C
        # program, the operating system would be within its rights to open file
        # descriptors we might not know about in the C library's
        # initialization; things like debuggers, profilers, or nsswitch plugins
        # might open some and this test should pass in those environments.

        # Although some of these file descriptors aren't predictable, we should
        # at least be able to select a very large file descriptor which is very
        # unlikely to be opened automatically in the subprocess.  (Apply a
        # fudge factor to avoid hard-coding something too near a limit
        # condition like the maximum possible file descriptor, which a library
        # might at least hypothetically select.)

        fudgeFactor = 17
        unlikelyFD = (resource.getrlimit(resource.RLIMIT_NOFILE)[0] -
                      fudgeFactor)

        os.dup2(w, unlikelyFD)
        self.addCleanup(os.close, unlikelyFD)

        output = io.BytesIO()

        class GatheringProtocol(ProcessProtocol):
            outReceived = output.write

            def processEnded(self, reason):
                reactor.stop()

        reactor = self.buildReactor()

        reactor.callWhenRunning(reactor.spawnProcess,
                                GatheringProtocol(),
                                pyExe, [pyExe, b"-Wignore", b"-c", source],
                                usePTY=self.usePTY)

        self.runReactor(reactor)
        reportedChildFDs = set(eval(output.getvalue()))

        stdFDs = [0, 1, 2]

        # Unfortunately this assertion is still not *entirely* deterministic,
        # since hypothetically, any library could open any file descriptor at
        # any time.  See comment above.
        self.assertEqual(
            reportedChildFDs.intersection(set(stdFDs + [unlikelyFD])),
            set(stdFDs))
Beispiel #60
0
 def __init__(self):
     super(_LinuxFifo, self).__init__()
     self.rfd, self.wfd = os.pipe()
     # must do before forking, since path contains pid!
     self.rpath = _LinuxFifo._mkFdPath(self.rfd)
     self.wpath = _LinuxFifo._mkFdPath(self.wfd)