def run():
    args = _parse_args()

    try:
        tornado = Popen(['python', '../static-server.py'])
        node = Popen(['node', 'build/server/server.js'])

        sleep(1)

        for url, page_name in PAGE_LIST:
            out_file = os.path.join(BUILD_DIR, page_name + '.html')
            # useless, since template is cached by tornado
            os.unlink(out_file)

            url = TORNADO_SERVER[:-1] + url
            phantom = Popen(['phantomjs', 'ph.js', url], stdout=PIPE)
            styles = phantom.stdout.readline().rstrip()
            load_css_func = _process_js_func(LOAD_CSS_FUNC)
            template = TEMPLATE.format(style=styles, load_css_func=load_css_func)

            with open(out_file, 'w') as f:
                f.write(template)
                print("out file:", out_file)
            phantom.communicate()
    finally:
        try:
            if tornado.poll() is None:
                tornado.terminate()
        except Exception:
            pass
        try:
            if node.poll() is None:
                node.terminate()
        except Exception:
            pass
Beispiel #2
0
class GraphNode:
	def __init__(self, name, command):
		self.name = name
		self.command = command
		self.process = None
		self.stdin = None
		self.stdout = None
		self.outputs = []
	
	def execute(self):
		if self.command is not None:
			self.process = Popen(self.command, shell=True, stdin=PIPE, stdout=PIPE, stderr=None)
			make_nonblocking(self.process.stdout.fileno())
			make_nonblocking(self.process.stdin.fileno())
			self.stdin = GraphNodeStream(self, self.process.stdin, 'stdin')
			self.stdout = GraphNodeStream(self, self.process.stdout, 'stdout')
	
	# Returns False iff this node will never produce more data
	def is_live(self):
		if self.process is not None:
			self.process.poll()
		return self.command is None or (self.process is not None and self.process.returncode is None)
	
	def is_readable(self):
		return self.stdout.is_live()
	
	def is_writable(self):
		return self.stdin.is_live()
	
	def __repr__(self):
		if self.command is None:
			return "(" + self.name + ")"
		return "(" + self.name + ": " + self.command + ")"
Beispiel #3
0
class Player:
    def __init__(self, name, cmd):
        self.name = name
        self.forts = set()
        self.marches = set()
        args = cmd.split(' ')
        self.process = Popen(list(args) + [name], stdin=PIPE, stdout=PIPE,
                             universal_newlines=True)

    def capture(self, fort):
        if fort.owner:
            fort.owner.forts.remove(fort)
        self.forts.add(fort)
        fort.owner = self

    def is_defeated(self):
        return (not self.forts) and (not self.marches)

    def send_state(self):
        if not self.process.poll():
            self.process.stdin.write(show_visible(self.forts))
            self.process.stdin.write('\n')
            self.process.stdin.flush()

    def read_commands(self, game):
        if not self.process.poll():
            try:
                read_commands(game, self, self.process.stdout)
            except StopIteration:
                self.process.kill()
Beispiel #4
0
def run(target, timeout=TIMEOUT, retry=RETRY,
        preserveChildren=False, verbose=False):
    """Run a process but kill it after timeout
    @param target: the command to launch (str)
    @param timeout: timeout in seconds before start killing process (int)
    @param retry: Number of time we will try to kill the process with SIGTERM and SIGKILL (int)
    @param preserveChildren: Do we need to also kill process children ? Default is True (bool)
    @param verbose: Print what happened on standard output (bool)
    @return: 0 if everything is ok. 1 if process was killed, 2 if something goes wrong.
    """
    
    rc=0
    # Some sanity checks
    if timeout<0 or retry<1:
        print "Timeout must be a positive integer and number of retry must be greater or equal than 1"
        return 2
    if verbose:
        print "running %s" % target

    process=Popen(target, shell=True)
    endTime=time()+timeout
    while True:
        # Timeout
        if time()>endTime:
            rc=1
            if verbose:
                print "Process timeout"
            break
        # Process finish
        if process.poll() is not None:
            if verbose:
                print "Process finish before timeout"
            break
        # Wait a little before looping
        sleep(0.05)
    if process.poll() is not None:
        if verbose:
            print "process correctly finished"
    else:
        if preserveChildren:
            pids=[]
        else:
            pids=getChildrenPid(process.pid, verbose)
        pids.append(process.pid)
        for i in xrange(retry):
            for signal in (15, 9): # SIGTERM then SIGKILL
                for pid in pids:
                    if verbose:
                        print "kill %s with signal %s" % (pid, signal)
                    try:
                        os.kill(pid, signal)
                    except OSError, e:
                        if e.errno==1:
                            print "Not authorized to kill %s" % pid
                        elif e.errno==3:
                            # No such process - already dead
                            pass
                        else:
                            print "Error while killing %s:\n%s" % (pid, e)
                sleep(i)
    def testBasic(self):
        iface = self.tap.name
        record = PacketRecord()

        # start capture
        process = Popen([APP, iface, CAPTURE_FILE], stdout=DEV_NULL, stderr=DEV_NULL)

        # send packets
        for i in range(PACKET_COUNT):
            packet = IP(dst="www.google.com")/ICMP()
            sendp(packet, iface=iface, verbose=False)
            record.add_sent(packet)

        # wait for stragglers
        time.sleep(1)

        # stop capture
        process.terminate()
        # hack: send one more packet to make sure capture closes immediately
        sendp(IP(), iface=iface, verbose=False)
        process.poll()

        # verify capture file
        for packet in rdpcap(CAPTURE_FILE):
            record.add_received(packet)
        self.assertTrue(record.verify())
Beispiel #6
0
  def exec_and_wait(self, cmd, timelife = 10):
    """
    exec_and_wait is syncronous with a timelife given by
    parameter whether is reached the command request is returning and error.
    retruncode is an Operating System err code, != 0 are error codes
    """

    proc = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True)
    t_nought = time.time()
    seconds_passed = 0
    self.pid = proc.pid
    self.stderr = proc.stderr
    self.stdout = proc.stdout.read().rstrip('\n')
    self.returncode = proc.poll()
    while(self.returncode != 0 and seconds_passed < timelife):
      seconds_passed = time.time() - t_nought
      self.returncode = proc.poll()

    if seconds_passed >= timelife:
      self.returncode = 1
      self.stdout = "Timelife expired, connection aborted"
      return

    self.returncode = 0
    return
    def testLocal(self):
        # note: ignore_zero_padding necessary until '0' padding simulation bug is resolved
        record = PacketRecord(ignore_zero_padding=True)

        # start capture
        process = Popen([CLIENT_APP, DFE_IP, DFE_NETMASK, '-l', CAPTURE_FILE], env=self.env, \
            stdout=DEV_NULL, stderr=DEV_NULL)
        self.processes.append(process)

        # send packets
        for i in range(PACKET_COUNT):
            packet = IP(dst='127.0.0.2')/ICMP()
            sendp(packet, iface=self.iface, verbose=False)
            record.add_sent(packet)

        # wait for stragglers
        time.sleep(1)

        # make sure still running
        process.poll()
        self.assertTrue(process.returncode == None)

        # stop capture
        process.terminate()
        # hack: send one more packet to make sure capture closes immediately
        sendp(IP(), iface=self.iface, verbose=False)
        process.wait()

        # verify capture CAPTURE_FILE
        for packet in rdpcap(CAPTURE_FILE):
            record.add_received(packet)
        self.assertTrue(record.verify())
def _launch_kernel(cmd):
    """start an embedded kernel in a subprocess, and wait for it to be ready
    
    Returns
    -------
    kernel, kernel_manager: Popen instance and connected KernelManager
    """
    kernel = Popen([sys.executable, '-c', cmd], stdout=PIPE, stderr=PIPE, env=env)
    connection_file = os.path.join(IPYTHONDIR,
                                    'profile_default',
                                    'security',
                                    'kernel-%i.json' % kernel.pid
    )
    # wait for connection file to exist, timeout after 5s
    tic = time.time()
    while not os.path.exists(connection_file) and kernel.poll() is None and time.time() < tic + 5:
        time.sleep(0.1)
    
    if not os.path.exists(connection_file):
        if kernel.poll() is None:
            kernel.terminate()
        raise IOError("Connection file %r never arrived" % connection_file)
    
    if kernel.poll() is not None:
        raise IOError("Kernel failed to start")
    
    km = BlockingKernelManager(connection_file=connection_file)
    km.load_connection_file()
    km.start_channels()
    
    return kernel, km
Beispiel #9
0
    def run_and_capture(self):
        """
        Run a command and capture exceptions. This is a blocking call
        :returns: tuple of exitcode, error (or None)
        :rtype: int, str | None
        """

        subproc = Popen([self._executable,  self._command] + self._args,
                        stderr=PIPE)
        err = ''
        while subproc.poll() is None:
            line = subproc.stderr.readline().decode('utf-8')
            err += line
            sys.stderr.write(line)
            sys.stderr.flush()

        exitcode = subproc.poll()
        # We only want to catch exceptions, not other stderr messages
        # (such as "task does not exist", so we look for the 'Traceback'
        # string.  This only works for python, so we'll need to revisit
        # this in the future when we support subcommands written in other
        # languages.
        err = ('Traceback' in err and err) or None

        return exitcode, err
Beispiel #10
0
class RunServer(object):

    def __init__(self, instance_name, port):
        self.instance_name = instance_name
        self.port = port
        self.lan_ip = get_lan_ip()
        self.process = None
        self.out = None

    def start(self):
        self.stop()
        cmd = [sys.executable, 'manage_%s.py' % self.instance_name,
               'runserver', '--noreload', '--traceback', '0.0.0.0:%d' % self.port]
        self.process = Popen(cmd)
        sleep(3.0)
        if self.process.poll() is not None:
            self.stop()
            raise RunException(ugettext("Error to start!"))
        self.open_url()

    def open_url(self):
        webbrowser.open_new("http://%(ip)s:%(port)d" %
                            {'ip': self.lan_ip, 'port': self.port})

    def stop(self):
        if self.is_running():
            self.process.terminate()
        self.process = None
        self.out = None

    def is_running(self):
        return (self.process is not None) and (self.process.poll() is None)
Beispiel #11
0
def run(cmd):
	handle = Popen(cmd, shell=True, stdout=PIPE, stderr=STDOUT)
	while handle.poll() is None:
		print(handle.stdout.read(1).decode('utf-8'), end='')
	print(handle.stdout.read(1).decode('utf-8'), end='') # Last byte, if there is any.
	handle.stdout.close()
	return handle.poll()
Beispiel #12
0
 def run(self):
     try:
         if osflag:
             proc=Popen(self.cmd,shell=False,stdin=None,stdout=PIPE,\
                 stderr=STDOUT,bufsize=0)
         else:
             from subprocess import STARTUPINFO
             si=STARTUPINFO()
             si.dwFlags|=1
             si.wShowWindow=0
             proc=Popen(self.cmd,shell=False,stdin=None,stdout=PIPE,\
                 stderr=STDOUT,bufsize=0,startupinfo=si)
         while 1:
             if self.stop_flag:
                 if osflag: proc.send_signal(signal.SIGKILL)
                 else: proc.kill()
                 break
             if osflag:
                 if proc.stdout in select.select([proc.stdout],[],[],1)[0]:
                     line=proc.stdout.readline()
                 else: line=' \n'
             else: line=proc.stdout.readline()
             if not len(line): break
             else:
                 if count(line,'ttl') or count(line,'TTL'): self.retries=0
                 else: self.retries=self.retries+1
                 line=' '
             sleep(0.5)
         proc.poll()
     except: pass
Beispiel #13
0
    def _popen(self, command, timeout, timeout_msg, rc_non_zero_msg, common_msg=""):
        """ Runs a command in background and returns its return_code, stdout and stderr.
        stdout and stderr will be None if return code = 0
        """
        stdout, stderr = None, None

        # Run the command
        p = Popen(command, stdout=PIPE, stderr=PIPE)

        # Sleep as long as requested and poll for results
        sleep(timeout)
        p.poll()

        if p.returncode is None:
            msg = timeout_msg + common_msg + "command:[{}]".format(command)
            raise Exception(msg.format(timeout))
        else:
            if p.returncode != 0:
                stdout, stderr = p.communicate()
                msg = (
                    rc_non_zero_msg
                    + common_msg
                    + "command:[{}], return code:[{}], stdout:[{}], stderr:[{}] ".format(
                        command, p.returncode, stdout, stderr
                    )
                )
                raise Exception(msg)

        return p.returncode
 def compile(self, filename):
     if (self.debug_output == 1):
         print("compile " + self.processor)
     else:
         chemin = os.path.dirname(filename)
         fichier = open(sys.path[0] + "/tmp/stdout", 'w+')
         sortie = Popen([sys.path[0] + "/tools/bin/sdcc",
                     "-mpic16",
                     "--denable-peeps",
                     "--obanksel=9",
                     "--opt-code-size",
                     "--optimize-cmp",
                     "--optimize-df",
                     "-p" + self.processor,                        
                     "-I" + sys.path[0] + "/include",
                     "-I" + chemin + "/",
                     "-c",
                     "-c",
                     "-o" + sys.path[0] + "/source/main.o",
                     sys.path[0] + "/source/main.c"],
                     stdout=fichier, stderr=STDOUT)
         sortie.communicate()
         if sortie.poll()!=0:
             fichier.seek(0)
             self.displaymsg(fichier.read(), 0)
         fichier.seek(0)
         line=fichier.readline()
         if line.find("error")!=-1:
             number=line[line.find(":")+1:line.find("error")-2]
             self.editor.highlightline(int(number)-1,'pink')
             self.displaymsg("error line"+" "+number+"\n",1)
             self.displaymsg(line[line.find("error")+10:len(line)],0)
         fichier.close()
         return sortie.poll()
Beispiel #15
0
def test_difftool(git_repo, request):
    nbdime.gitdifftool.main(['config', '--enable'])
    cmd = get_output('git config --get --local difftool.nbdime.cmd').strip()

    # pick a non-random port so we can connect later, and avoid opening a browser
    port = 62021
    cmd = cmd + ' --port=%i --browser=disabled' % port
    call(['git', 'config', 'difftool.nbdime.cmd', cmd])

    # avoid global diff driver config from disabling difftool:
    with open('.gitattributes', 'w') as f:
        f.write('*.ipynb\tdiff=notnbdime')

    p = Popen(['git', 'difftool', '--tool=nbdime', 'base'])
    def _term():
        try:
            p.terminate()
        except OSError:
            pass
    request.addfinalizer(_term)
    
    # 3 is the number of notebooks in this diff
    url = 'http://127.0.0.1:%i' % port
    for i in range(3):
        _wait_up(url, check=lambda : p.poll() is None)
        # server started
        r = requests.get(url + '/difftool')
        r.raise_for_status()
        # close it
        r = requests.post(url + '/api/closetool', headers={'exit_code': '0'})
        r.raise_for_status()
        time.sleep(0.25)
    # wait for exit
    p.wait()
    assert p.poll() == 0
Beispiel #16
0
def main(args=sys.argv[1:]):
    parser = argparse.ArgumentParser(description="Adapt socket communication.")
    parser.add_argument("--server")
    parser.add_argument("--port", type=int)
    parser.add_argument("--legacy", nargs=2)
    config = parser.parse_args(args)

    if config.legacy:
        address = (config.legacy[0], int(config.legacy[1]))
    else:
        address = (config.server, config.port)
    sock = socket.socket()
    sock.connect(address)

    proc = Popen("simple_engine", stdin=PIPE, stdout=PIPE,
                 universal_newlines=True)
    com = _ProcCom(proc, sock)
    com.start()

    while proc.poll() is None:
        msg = sock.recv(4096)
        if len(msg) == 0:
            break
        proc.stdin.write(msg)
        proc.stdin.flush()
    if proc.poll() is None:
        proc.stdin.write("quit\n")
        proc.stdin.flush()
    com.stop.set()
    sock.close()
Beispiel #17
0
    def worker(self, cmd_q, res_q):
        while True:
            name, cmd = cmd_q.get(True)

            self.log.debug("External : launch %s" % cmd)

            try:
                proc = Popen(
                    cmd,
                    stdin=None, stdout=PIPE, stderr=STDOUT, close_fds=True)

                output = ""
                while True:
                    out, err = proc.communicate()
                    proc.poll()
                    output += out.decode('utf-8')
                    if proc.returncode is not None:
                        break

            except:
                self.log.debug(traceback.format_exc())

            # Truncate multi-line output
            try:
                output = output[:output.index("\n")]
            except:
                pass

            res_q.put_nowait(
                (name, proc.returncode, output, int(time.time())))
            cmd_q.task_done()
    def upload_using_curl(self, source_directory, target_directory):
        logging.debug("Curl upload!")
        import os

        filesToUpload = os.listdir(source_directory)
        for file in filesToUpload:
            logging.warn(source_directory)
            logging.warn(file)
            filePath = os.path.join(source_directory, file)
            curlcommand = "curl --ftp-create-dirs -T {filepath} --keepalive-time 5 --user {user}:{password} ftp://{host}{remotedir}/".format(
                user=self.user,
                password=self.password,
                port=self.port,
                host=self.hostname,
                remotedir=target_directory,
                filepath=filePath,
            )
            logging.debug(curlcommand)
            p = Popen(curlcommand)
            stdout, stderr = p.communicate()
            logging.debug(stdout)
            p.poll()
            curlReturnValue = p.returncode
            if curlReturnValue != 0:
                logging.error("CURL has encurred into an error! the program execution will stop!")
                raise Exception("CURL returned invalid status code")
Beispiel #19
0
def local_popen( f, cmdline ):
   if os.name != 'nt':
      cmdline="./" + cmdline

   if execution==0:
      print cmdline
   else:
      p=Popen( cmdline, shell=True, stdout=PIPE, stderr=STDOUT )

      r=p.poll()
      while r == None:
         r=p.poll()
      pipe=p.stdout

      if r != 0:
         print "---- TESTING " + cmdline.split()[3] + "... FAILED(" + str(p.returncode) +") !"
         for line in pipe.readlines():
            f.write(str(line))
      else:
         found=0
         for line in pipe.readlines():
            f.write(str(line))
            if "TESTING" in line :
               found = 1
               print line,
         if found == 0:
            print cmdline.split()[0] + " " + cmdline.split()[3] + ": FAILED(Unexpected error)"
            
   f.flush();
   return 0
Beispiel #20
0
def run_tsmuxer_2(msg, argv1, argv2):
    out = StringIO.StringIO()
    argv = ['tsMuxeR', argv1, argv2]
    p = Popen(argv, stdout=PIPE, stderr=PIPE)
    write("%s ...\n" % msg)
    while True:
        line = p.stdout.readline()
        if line == '':
            if p.poll() is not None:
                break   #reached final line of output
            else:
                time.sleep(0.01) # sleep 10 milliseconds
        else:
            out.write(line)
            progress = extract_progress(line)
            if progress is not None:
                write("\r%5.1f%% completed" % progress)
    
    write("\n")

    try:
        assert p.poll() == 0
        return True
    except AssertionError:
        print "ERROR: Muxing failed"
        print "Args:", argv
        out = out.read()
        print "STDOUT:"
        print out
        err = p.stderr.read()
        print "STDERR:"
        print err
        return False
Beispiel #21
0
class ExecuteHook(BaseHook):
    """
    Hook that executes an arbitary command in the setup phase.
    If the command is still running during teardown, it will
    be terminated.
    """
    def __init__(self, args):
        """
        :param args: sequence of program arguments,
                     see :class:`subprocess.Popen` for details
        :type args: string or sequence
        """
        super(ExecuteHook, self).__init__()
        self.args = args
        self.proc = None

    def setup(self):
        self.proc = Popen(shlex.split(self.args) if
                isinstance(self.args, str) else self.args)

    def teardown(self):
        self.proc.poll()
        if self.proc.returncode is None:
            self.proc.terminate()
            self.proc.wait()
Beispiel #22
0
def setup_kernel(cmd):
    """start an embedded kernel in a subprocess, and wait for it to be ready

    Returns
    -------
    kernel_manager: connected KernelManager instance
    """
    kernel = Popen([sys.executable, "-c", cmd], stdout=PIPE, stderr=PIPE, env=env)
    connection_file = os.path.join(IPYTHONDIR, "profile_default", "security", "kernel-%i.json" % kernel.pid)
    # wait for connection file to exist, timeout after 5s
    tic = time.time()
    while not os.path.exists(connection_file) and kernel.poll() is None and time.time() < tic + SETUP_TIMEOUT:
        time.sleep(0.1)

    if kernel.poll() is not None:
        o, e = kernel.communicate()
        e = py3compat.cast_unicode(e)
        raise IOError("Kernel failed to start:\n%s" % e)

    if not os.path.exists(connection_file):
        if kernel.poll() is None:
            kernel.terminate()
        raise IOError("Connection file %r never arrived" % connection_file)

    client = BlockingKernelClient(connection_file=connection_file)
    client.load_connection_file()
    client.start_channels()

    try:
        yield client
    finally:
        client.stop_channels()
        kernel.terminate()
Beispiel #23
0
    def start(self):
        """Start the process."""
        import shlex
        from subprocess import Popen

        if self.process:
            raise RuntimeError("Process already started.")
        if self.host and self.network_ping():
            raise RuntimeError("A process is already running on port %s" %
                               self.port)

        if isinstance(self.cmd, basestring):
            cmd = shlex.split(self.cmd)
        else:
            cmd = self.cmd
        self.null = open('/dev/null', 'w')
        process = Popen(cmd, stdout=self.null, stderr=self.null, close_fds=True)

        if not self.host:
            time.sleep(0.35)
            if process.poll():
                output = process.communicate()[0]
                raise RuntimeError("Did not start server!  Woe!\n" + output)
            self.process = process
            return

        start = time.time()
        while process.poll() is None and time.time() - start < 15:
            if self.network_ping():
                break
        else:
            output = process.communicate()[0]
            raise RuntimeError("Did not start server!  Woe!\n" + output)
        self.process = process
Beispiel #24
0
def yes_proc(args, yes="yes"):
    proc = Popen(args, stdin=PIPE)
    while proc.returncode is None:
        proc.communicate(yes)
        proc.poll()

    return proc.returncode == 0
Beispiel #25
0
def wait_process_finish(scan, profile_fname):
    '''
    Start w3af process in new process, and wait while it terminate
    '''
    process = Popen([settings.W3AF_RUN,
                    '--no-update', '-P',
                    profile_fname],
                    stdout=PIPE,
                    stderr=PIPE)
    scan.pid = process.pid
    scan.save()
    while process.returncode is None:
        scan.last_updated = datetime.today()
        scan.save()
        sleep(W3AF_POLL_PERIOD) # wail 
        process.poll() # get process status
    scan.set_task_status_free()
    finish_time = datetime.now()
    scan.finish = finish_time
    scan.save()
    target = scan.scan_task.target
    target.last_scan = finish_time
    target.save()
    logger.info('w3af Process return code %s' % process.returncode)
    return process.returncode
Beispiel #26
0
class TestServer(unittest.TestCase):
    server = 'WSGIRefServer'
    port = 12643

    def setUp(self):
        # Start servertest.py in a subprocess
        cmd = [sys.executable, serverscript, self.server, str(self.port)]
        cmd += sys.argv[1:] # pass cmdline arguments to subprocesses
        self.p = Popen(cmd, stdout=PIPE, stderr=PIPE)
        # Wait for the socket to accept connections
        for i in xrange(100):
            time.sleep(0.1)
            # Check if the process has died for some reason
            if self.p.poll() != None: break
            if ping('127.0.0.1', self.port): break

    def tearDown(self):
        while self.p.poll() is None:
            os.kill(self.p.pid, signal.SIGINT)
            time.sleep(0.1)
            if self.p.poll() is None:
                os.kill(self.p.pid, signal.SIGTERM)
        for stream in (self.p.stdout, self.p.stderr):
            for line in stream:
                if tob('Warning') in line \
                or tob('Error') in line:
                    print line.strip().decode('utf8')

    def fetch(self, url):
        try:
            return urllib2.urlopen('http://127.0.0.1:%d/%s' % (self.port, url)).read()
        except Exception, e:
            return repr(e)
 def start(self, timeout = 20):
     if self.is_running():
         raise Exception('%s is already running' % self.name)
     if not self.run_cmd:
         raise Exception('No starting command registered for %s' % self.name)
     if type(self.run_cmd) is types.FunctionType:
         self.run_cmd()
         return
     with tempfile.TemporaryFile() as stdout_file, tempfile.TemporaryFile() as stderr_file:
         proc = Popen(shlex.split('%s -p %s' % (self.run_cmd, self.port)), cwd = self.dir, close_fds = True,
                      stdout = stdout_file, stderr = stderr_file)
         if timeout > 0:
             poll_rate = 0.1
             for i in range(int(timeout/poll_rate)):
                 if self.is_running():
                     break
                 sleep(poll_rate)
                 if bool(proc.poll()): # process ended with error
                     stdout_file.seek(0)
                     stderr_file.seek(0)
                     raise Exception('Run of %s ended unexpectfully: %s' % (self.name, [proc.returncode, stdout_file.read().decode(errors = 'replace'), stderr_file.read().decode(errors = 'replace')]))
                 elif proc.poll() == 0: # process runs other process, and ended
                     break
         if self.is_running():
             if self.check_connectivity():
                 return True
             raise Exception('Daemon process is running, but no connectivity')
         raise Exception('%s failed to run.' % self.name)
Beispiel #28
0
class Inetd:
  def __init__(self, inq, outq):
    self.inq=inq
    self.outq=outq

  def start(self):
    self.proc=Popen(['fossil', 'http', '/home/blanu/fossil/'], stdin=PIPE, stdout=PIPE)
#    self.proc=Popen(['fossil', 'help'], stdin=PIPE, stdout=PIPE)

    t=Thread(target=self.processIn)
    t.setDaemon(True)
    t.start()

    t2=Thread(target=self.processOut)
#    t2.setDaemon(True)
    t2.start()

  def processIn(self):
    self.proc.poll()
    while not self.proc.returncode:
      data=self.inq.get()
      print('inetd -> '+str(data))
      self.proc.stdin.write(data)
      self.proc.stdin.flush()
      self.proc.poll()

  def processOut(self):
    print('waiting to finish')
#    self.proc.wait()
    print('finished')
    print('waiting to read from inetd')
    data=self.proc.stdout.read()
    print('inetd <- '+str(data))
    self.outq.put(data)
Beispiel #29
0
class Engine:
    def __init__(self, cmdline):
        self.proc = Popen(cmdline, stdin = PIPE, stdout = PIPE)
        self.proc_com = _ProcCom(self.proc)
        self.proc_com.start()

    def send(self, msg):
        self.proc.stdin.write(msg)
        self.proc.stdin.flush()

    def readline(self, timeout=None):
        if self.proc.poll():
            return ""
        try:
            msg = self.proc_com.outq.get(timeout=timeout)
        except Empty:
            raise socket.timeout()
        return msg

    def cleanup(self):
        self.proc_com.stop.set()
        if self.proc.poll() is None:
            if sys.platform == 'win32':
                import ctypes
                handle = int(self.proc._handle)
                ctypes.windll.kernel32.TerminateProcess(handle, 0)
            else:
                os.kill(self.proc.pid, signal.SIGTERM)
class CMatProcess:
    def __init__(self, processPath='./cmatprocess/cmatprocess', nCores=4):
        assert processPath is not None

        self.cmd = [processPath, str(nCores)]

        self.p = Popen(self.cmd, stdin=PIPE, stdout=PIPE,
                       stderr=PIPE, bufsize=0)

        def exitHandler():
            if self.p.poll() is None:
                self.p.kill()
        atexit.register(exitHandler)

    def processPaths(self, imgPaths, f):
        rc = self.p.poll()
        if rc is not None and rc != 0:
            raise Exception("Error from process!")

        self.p.stdin.write(f + ' ' + ' '.join(imgPaths) + '\n')
        try:
            n = int(self.p.stdout.readline())
            result = []
            for i in range(n):
                fileLocation = self.p.stdout.readline().strip()
                if len(fileLocation) > 3:
                    result.append(cv2.imread(fileLocation))

            return result
        except Exception as e:
            self.p.kill()
            stdout, stderr = self.p.communicate()
            print("Error from process output: " + stdout + '\n' + stderr)
            sys.exit(-1)
Beispiel #31
0
    def run(self):
        self.started_event()
        mutex.lock()
        log_entry = EventLogModel(category='borg-run',
                                  subcommand=self.cmd[1],
                                  profile=self.params.get(
                                      'profile_name', None))
        log_entry.save()
        logger.info('Running command %s', ' '.join(self.cmd))

        p = Popen(self.cmd,
                  stdout=PIPE,
                  stderr=PIPE,
                  bufsize=1,
                  universal_newlines=True,
                  env=self.env,
                  cwd=self.cwd,
                  start_new_session=True)

        self.process = p

        # Prevent blocking of stdout/err. Via https://stackoverflow.com/a/7730201/3983708
        os.set_blocking(p.stdout.fileno(), False)
        os.set_blocking(p.stderr.fileno(), False)

        def read_async(fd):
            try:
                return fd.read()
            except (IOError, TypeError):
                return ''

        stdout = []
        while True:
            # Wait for new output
            select.select([p.stdout, p.stderr], [], [], 0.1)

            stdout.append(read_async(p.stdout))
            stderr = read_async(p.stderr)
            if stderr:
                for line in stderr.split('\n'):
                    try:
                        parsed = json.loads(line)
                        if parsed['type'] == 'log_message':
                            self.app.backup_log_event.emit(
                                f'{parsed["levelname"]}: {parsed["message"]}')
                            level_int = getattr(logging, parsed["levelname"])
                            logger.log(level_int, parsed["message"])
                        elif parsed['type'] == 'file_status':
                            self.app.backup_log_event.emit(
                                f'{parsed["path"]} ({parsed["status"]})')
                        elif parsed['type'] == 'archive_progress':
                            msg = (
                                f"Files: {parsed['nfiles']}, "
                                f"Original: {pretty_bytes(parsed['original_size'])}, "
                                f"Deduplicated: {pretty_bytes(parsed['deduplicated_size'])}, "
                                f"Compressed: {pretty_bytes(parsed['compressed_size'])}"
                            )
                            self.app.backup_progress_event.emit(msg)
                    except json.decoder.JSONDecodeError:
                        msg = line.strip()
                        if msg:  # Log only if there is something to log.
                            self.app.backup_log_event.emit(msg)
                            logger.warning(msg)

            if p.poll() is not None:
                time.sleep(0.1)
                stdout.append(read_async(p.stdout))
                break

        result = {
            'params': self.params,
            'returncode': self.process.returncode,
            'cmd': self.cmd,
        }
        stdout = ''.join(stdout)

        try:
            result['data'] = json.loads(stdout)
        except ValueError:
            result['data'] = stdout

        log_entry.returncode = p.returncode
        log_entry.repo_url = self.params.get('repo_url', None)
        log_entry.save()

        self.process_result(result)
        self.finished_event(result)
        mutex.unlock()
Beispiel #32
0
def run_with_timeout(command, timeoutSecs, inputStream = "", combinedOutput=True):
    """
    Runs the given process until it exits or the given time out is reached.

    Returns a tuple: (bool:timeout, str:stdout, str:stderr, int:exitcode)
    """
    timePassed = 0.0
    increment = 0.01

    stderrFD, errFile = tempfile.mkstemp()
    if combinedOutput:
        stdoutFD, outFile = stderrFD, errFile
    else:
        stdoutFD, outFile = tempfile.mkstemp()

    process =  Popen(command, shell=True, stdin=PIPE, 
                     stdout=stdoutFD, stderr=stderrFD, close_fds=False)

    if process == None:
        print "Could not create process"
        sys.exit(1)
    try:
        if inputStream != "":
            for line in inputStream:
                process.stdin.write(line.strip() + '\n')
                process.stdin.flush()
            process.stdin.close()

        while True:
            status = process.poll()
            if status != None:
                # Process terminated succesfully.
                stdoutSize = os.lseek(stdoutFD, 0, 2)
                stderrSize = os.lseek(stderrFD, 0, 2)

                os.lseek(stdoutFD, 0, 0)
                os.lseek(stderrFD, 0, 0)

                stdoutContents = os.read(stdoutFD, stdoutSize)
                os.close(stdoutFD)
                os.remove(outFile)

                if not combinedOutput:
                    stderrContents = os.read(stderrFD, stderrSize)
                    os.close(stderrFD)
                    os.remove(errFile)
                else:
                    stderrContents = stdoutContents

                return (False, stdoutContents, stderrContents, process.returncode)

            if timePassed < timeoutSecs:
                time.sleep(increment)
                timePassed = timePassed + increment
            else:
                # time out, kill the process.
                stdoutSize = os.lseek(stdoutFD, 0, 2)
                stderrSize = os.lseek(stderrFD, 0, 2)

                os.lseek(stdoutFD, 0, 0)
                stdoutContents = os.read(stdoutFD, stdoutSize)
                os.close(stdoutFD)
                os.remove(outFile)

                if not combinedOutput:
                    os.lseek(stderrFD, 0, 0)
                    stderrContents = os.read(stderrFD, stderrSize)
                    os.close(stderrFD)
                    os.remove(errFile)
                else:
                    stderrContents = stdoutContents
                os.kill(process.pid, signal.SIGTSTP)
                return (True, stdoutContents, stderrContents, process.returncode)
    except Exception, e:
        # if something threw exception (e.g. ctrl-c)
        os.kill(process.pid, signal.SIGTSTP)
        try:
            # time out, kill the process.
            # time out, kill the process.
            stdoutSize = os.lseek(stdoutFD, 0, 2)
            stderrSize = os.lseek(stderrFD, 0, 2)

            os.lseek(stdoutFD, 0, 0)
            stdoutContents = os.read(stdoutFD, stdoutSize)
            os.close(stdoutFD)
            os.remove(outFile)

            if not combinedOutput:
                os.lseek(stderrFD, 0, 0)
                stderrContents = os.read(stderrFD, stderrSize)
            else:
                os.close(stderrFD)
                os.remove(errFile)
                stderrContents = stdoutContents

            os.kill(process.pid, signal.SIGTSTP)                
        except:
            pass

        return (False, stdoutContents, stderrContents, process.returncode)
def gradeGroup(group_dir, question_ID):

    #student_question_ID = student_ID + question_ID
    os.chdir("./Groups/" + group_dir)

    #-----------------------------------------------------
    # Edit the .bochsrc-no-gui, replace the partport1 line to this question timestamped output file
    # We do this to make sure that Autograde doesn't mistakenly take the same file for the next question
    #-----------------------------------------------------
    #   fraction of a second time stamp
    QTimestamp = str(time.time()).replace(".", "")
    #   This question specific student_stdout file name
    student_stdout_default = "student_stdout"
    student_stdout_Q_specific = student_stdout_default + "_" + QTimestamp

    bochsrcDefault = open(os.getcwd() + "/.bochsrc-no-gui", 'r')
    bochsrcAutograde = open(os.getcwd() + "/.bochsrc_autograde_" + QTimestamp,
                            'w')
    for line in bochsrcDefault:
        bochsrcAutograde.write(
            line.replace(student_stdout_default, student_stdout_Q_specific))
    bochsrcDefault.close()
    bochsrcAutograde.close()
    #-----------------------------------------------------

    # TODO: Improvment: If a compilation error occurs to a student folder, let the script save the output
    #                   of "make" into a disk file (just like it does for Bochs output)

    # According to the number of cores in your machine, you can change the -j number to compile faster
    # example: I use -j4 on my quad core machine
    makeProc = subprocess.Popen("make -j2",
                                shell=True,
                                stdout=PIPE,
                                stderr=PIPE)
    #makeProc.wait()
    makeProc.communicate(
    )  #wait() caused the deadlock with stdout, err = PIPE, online documentation warns against that, and suggested the use of communicate()

    question_grade = -100
    if (os.path.isfile(os.getcwd() + "/obj/kern/bochs.img") == True):

        myCommand = 'bochs-no-gui -q -f .bochsrc_autograde_' + QTimestamp
        #proc = Popen(myCommand, shell=True, stdout=PIPE, stderr=PIPE, stdin=PIPE, preexec_fn=os.setsid)
        #si = subprocess.STARTUPINFO()
        #si.dwFlags |= subprocess.STARTF_USESTDHANDLES
        #proc = Popen(myCommand, shell=True, startupinfo=si, stdout=PIPE, stderr=STDOUT) #shell=True, stdout=PIPE, stderr=PIPE, stdin=PIPE)

        proc = Popen("exec " + myCommand,
                     shell=True,
                     stdout=PIPE,
                     stderr=PIPE,
                     stdin=PIPE)

        #===== WARNING: Unix only work-around: non-blocking streams
        # default read() and readline() in Python block until a
        # valid output is available.
        # This makes it difficult to stop the bochs process
        # when we reach the FOS>, because we will not know when to check
        # as long as read() is working and doesn't return
        #
        # The workaround using fcntl non-blocking flag is Unix only solution
        # we should find an equivalent library to other platforms, in case
        # we migrate the course to windows or Mac

        #import fcntl
        #file_flags = fcntl.fcntl(proc.stdout.fileno(), fcntl.F_GETFL)
        #fcntl.fcntl(proc.stdout.fileno(), fcntl.F_SETFL, file_flags | os.O_NONBLOCK)

        # UPDATE: We didn't use this non-blocking work-around, and we returned
        #         to using the default blocking read(1), but we made sure
        #         to always check for the FOS> after every character read
        #         from the stream. if for some unusual reason the FOS>
        #         prompt isn't printed, we will have to manually close the
        #         bochs window to allow the script to continue working.
        #         - JUST MAKE SURE to run the script with python - u (unbuffered)
        #=============
        stream_data = ""
        question_grade = 0

        #A closure (inner function) that I will use later to terminate any stucking threads or processes
        # and finialize remainig resources
        def finializeAndCleanUp():
            try:
                if (proc.poll() is
                        None):  #kill the Bochs process if not already killed
                    proc.kill()
                    #os.kill(proc.pid, signal.SIGKILL)
                thefile.close()
            except Exception, msg:
                print msg

        try:
            startTimeOfQuestion = datetime.now()

            while os.path.isfile(os.getcwd() + "/" +
                                 student_stdout_Q_specific) == False:
                #keep waiting till we find the new student_stdout file generated by Bochs
                if ((datetime.now() - startTimeOfQuestion).seconds > 40):
                    print "Timeout, no console output from Bochs"
                    shutil.rmtree('./obj', ignore_errors=True)
                    os.chdir("../../")
                    if (proc.poll() is None
                        ):  #kill the Bochs process if not already killed
                        proc.kill()
                    return -100, (datetime.now() - startTimeOfQuestion).seconds
                time.sleep(0.001)

            #TODO: Consider removing the try catch statements as a running time optimization
            thefile = open(os.getcwd() + "/" + student_stdout_Q_specific,
                           "r")  #+ "/Students"+"/"+student_dir+

            while True:
                timeNow = datetime.now()
                if ((timeNow - startTimeOfQuestion).seconds > 60):
                    question_grade = -6
                    finializeAndCleanUp()
                    print "Timeout"
                    break
                try:
                    #TODO: Revise this part to enable time out for project correction

                    #def signal_handler(signum, frame):
                    #    raise Exception("Timed out!")
                    #signal.signal(signal.SIGALRM, signal_handler)
                    #signal.alarm(60*1)   # 1 Min
                    #try:
                    #    chunk = thefile.read(1)
                    #except Exception, msg:
                    #    question_grade = -6
                    #    finializeAndCleanUp()
                    #    print msg
                    #    break

                    #time.sleep(0.00005)  #this slowed down the loop consideralby, taking about 5 seconds
                    chunk = thefile.read(1)
                    if not chunk:
                        continue
                    #else:
                    #    sys.stdout.write(chunk)
                    #    sys.stdout.flush()

                    stream_data += chunk
                    #print stream_data
                    if "InitialWSError1" in stream_data:
                        question_grade = -4
                        finializeAndCleanUp()
                        break
                    elif "InitialWSError2" in stream_data:
                        question_grade = -5
                        finializeAndCleanUp()
                        break
                    elif "[EVAL_FINAL]panic" in stream_data:
                        if tests[
                                question_ID] == 'tia' and "kernel [EVAL_FINAL]panic" in stream_data:
                            question_grade = 1
                            finializeAndCleanUp()
                            break
                        else:
                            question_grade = -7  #Panic Case
                            finializeAndCleanUp()
                            break
                    elif stream_data.count('!! FCIS says HELLO !!') > 1:
                        question_grade = -3  #Restart Case
                        finializeAndCleanUp()
                        break
                    elif outputs[
                            question_ID] == "TESTING MULTIPLE TIMES" and stream_data.count(
                                '[AUTO_GR@DING]Congratulations!! The array is sorted correctly'
                            ) == 3:
                        question_grade = 1
                        finializeAndCleanUp()
                        break
                    elif outputs[
                            question_ID] == "TESTING MULTIPLE TIMES S2" and "[AUTO_GR@DING]Fibonacci #30 = 1346269" in stream_data and stream_data.count(
                                '[AUTO_GR@DING]Congratulations!! The array is sorted correctly'
                            ) == 4:
                        question_grade = 1
                        finializeAndCleanUp()
                        break
                    elif outputs[
                            question_ID] == "TESTING MULTIPLE TIMES S3" and stream_data.count(
                                '[AUTO_GR@DING]Congratulations!! The array is sorted correctly'
                            ) == 2:
                        question_grade = 1
                        finializeAndCleanUp()
                        break
                    elif outputs[
                            question_ID] == "TESTING MULTIPLE TIMES S4" and stream_data.count(
                                '[AUTO_GR@DING]Congratulations!! The array is sorted correctly'
                            ) == 1:
                        question_grade = 1
                        finializeAndCleanUp()
                        break
                    elif outputs[
                            question_ID] == "TESTING ENVFREE SC 2" and stream_data.count(
                                '[AUTO_GR@DING]Congratulations!! test Scenario 2 completed successfully.'
                            ) == 1 and stream_data.count(
                                '[AUTO_GR@DING]Congratulations!! The array is sorted correctly'
                            ) == 2:
                        question_grade = 1
                        finializeAndCleanUp()
                        break
                    elif outputs[question_ID] in stream_data:
                        question_grade = 1
                        finializeAndCleanUp()
                        break
                    elif "FOS>" in stream_data:
                        finializeAndCleanUp()
                        break

                    elif proc.poll(
                    ) is not None:  #if process is finished, break the loop
                        finializeAndCleanUp()
                        break
                    #elif chunk is None:
                    #break
                    #elif chunk == "":
                    #break

                except IOError:
                    if proc.poll() is not None:
                        #print "exiting loop, io error"
                        finializeAndCleanUp()
                        break

                except Exception, msg:
                    #print msg
                    if proc.poll() is not None:
                        #print "exiting loop"
                        finializeAndCleanUp()
                        break

        except IOError, msg:
            #print "exiting, io error opening student_stdout_..."
            print msg
            finializeAndCleanUp()
Beispiel #34
0
    def shebang(self, line, cell):
        """Run a cell via a shell command
        
        The `%%script` line is like the #! line of script,
        specifying a program (bash, perl, ruby, etc.) with which to run.
        
        The rest of the cell is run by that program.
        
        Examples
        --------
        ::
        
            In [1]: %%script bash
               ...: for i in 1 2 3; do
               ...:   echo $i
               ...: done
            1
            2
            3
        """
        argv = arg_split(line, posix=not sys.platform.startswith('win'))
        args, cmd = self.shebang.parser.parse_known_args(argv)

        try:
            p = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE)
        except OSError as e:
            if e.errno == errno.ENOENT:
                print("Couldn't find program: %r" % cmd[0])
                return
            else:
                raise

        if not cell.endswith('\n'):
            cell += '\n'
        cell = cell.encode('utf8', 'replace')
        if args.bg:
            self.bg_processes.append(p)
            self._gc_bg_processes()
            if args.out:
                self.shell.user_ns[args.out] = p.stdout
            if args.err:
                self.shell.user_ns[args.err] = p.stderr
            self.job_manager.new(self._run_script, p, cell, daemon=True)
            if args.proc:
                self.shell.user_ns[args.proc] = p
            return

        try:
            out, err = p.communicate(cell)
        except KeyboardInterrupt:
            try:
                p.send_signal(signal.SIGINT)
                time.sleep(0.1)
                if p.poll() is not None:
                    print("Process is interrupted.")
                    return
                p.terminate()
                time.sleep(0.1)
                if p.poll() is not None:
                    print("Process is terminated.")
                    return
                p.kill()
                print("Process is killed.")
            except OSError:
                pass
            except Exception as e:
                print("Error while terminating subprocess (pid=%i): %s" \
                    % (p.pid, e))
            return
        out = py3compat.bytes_to_str(out)
        err = py3compat.bytes_to_str(err)
        if args.out:
            self.shell.user_ns[args.out] = out
        else:
            sys.stdout.write(out)
            sys.stdout.flush()
        if args.err:
            self.shell.user_ns[args.err] = err
        else:
            sys.stderr.write(err)
            sys.stderr.flush()
Beispiel #35
0
class FirefoxBinary(object):

    NO_FOCUS_LIBRARY_NAME = "x_ignore_nofocus.so"

    def __init__(self, firefox_path=None, log_file=None):
        """
        Creates a new instance of Firefox binary.

        :Args:
         - firefox_path - Path to the Firefox executable. By default, it will be detected from the standard locations.
         - log_file - A file object to redirect the firefox process output to. It can be sys.stdout.
                      Please note that with parallel run the output won't be synchronous.
                      By default, it will be redirected to /dev/null.
        """
        self._start_cmd = firefox_path
        # We used to default to subprocess.PIPE instead of /dev/null, but after
        # a while the pipe would fill up and Firefox would freeze.
        self._log_file = log_file or open(os.devnull, "wb")
        self.command_line = None
        if self._start_cmd is None:
            self._start_cmd = self._get_firefox_start_cmd()
        if not self._start_cmd.strip():
            raise WebDriverException(
                "Failed to find firefox binary. You can set it by specifying "
                "the path to 'firefox_binary':\n\nfrom "
                "selenium.webdriver.firefox.firefox_binary import "
                "FirefoxBinary\n\nbinary = "
                "FirefoxBinary('/path/to/binary')\ndrive = "
                "webdriver.Firefox(firefox_binary=binary)")
        # Rather than modifying the environment of the calling Python process
        # copy it and modify as needed.
        self._firefox_env = os.environ.copy()
        self._firefox_env["MOZ_CRASHREPORTER_DISABLE"] = "1"
        self._firefox_env["MOZ_NO_REMOTE"] = "1"
        self._firefox_env["NO_EM_RESTART"] = "1"

    def add_command_line_options(self, *args):
        self.command_line = args

    def launch_browser(self, profile, timeout=30):
        """Launches the browser for the given profile name.
        It is assumed the profile already exists.
        """
        self.profile = profile

        self._start_from_profile_path(self.profile.path)
        self._wait_until_connectable(timeout=timeout)

    def kill(self):
        """Kill the browser.

        This is useful when the browser is stuck.
        """
        if self.process:
            self.process.kill()
            self.process.wait()

    def _start_from_profile_path(self, path):
        self._firefox_env["XRE_PROFILE_PATH"] = path

        if platform.system().lower() == 'linux':
            self._modify_link_library_path()
        command = [self._start_cmd, "-foreground"]
        if self.command_line is not None:
            for cli in self.command_line:
                command.append(cli)
        self.process = Popen(
            command, stdout=self._log_file, stderr=STDOUT,
            env=self._firefox_env)

    def _wait_until_connectable(self, timeout=30):
        """Blocks until the extension is connectable in the firefox."""
        count = 0
        while not utils.is_connectable(self.profile.port):
            if self.process.poll() is not None:
                # Browser has exited
                raise WebDriverException(
                    "The browser appears to have exited "
                    "before we could connect. If you specified a log_file in "
                    "the FirefoxBinary constructor, check it for details.")
            if count >= timeout:
                self.kill()
                raise WebDriverException(
                    "Can't load the profile. Possible firefox version mismatch. "
                    "You must use GeckoDriver instead for Firefox 48+. Profile "
                    "Dir: %s If you specified a log_file in the "
                    "FirefoxBinary constructor, check it for details."
                    % (self.profile.path))
            count += 1
            time.sleep(1)
        return True

    def _find_exe_in_registry(self):
        try:
            from _winreg import OpenKey, QueryValue, HKEY_LOCAL_MACHINE, HKEY_CURRENT_USER
        except ImportError:
            from winreg import OpenKey, QueryValue, HKEY_LOCAL_MACHINE, HKEY_CURRENT_USER
        import shlex
        keys = (r"SOFTWARE\Classes\FirefoxHTML\shell\open\command",
                r"SOFTWARE\Classes\Applications\firefox.exe\shell\open\command")
        command = ""
        for path in keys:
            try:
                key = OpenKey(HKEY_LOCAL_MACHINE, path)
                command = QueryValue(key, "")
                break
            except OSError:
                try:
                    key = OpenKey(HKEY_CURRENT_USER, path)
                    command = QueryValue(key, "")
                    break
                except OSError:
                    pass
        else:
            return ""

        if not command:
            return ""

        return shlex.split(command)[0]

    def _get_firefox_start_cmd(self):
        """Return the command to start firefox."""
        start_cmd = ""
        if platform.system() == "Darwin":
            start_cmd = "/Applications/Firefox.app/Contents/MacOS/firefox-bin"
            # fallback to homebrew installation for mac users
            if not os.path.exists(start_cmd):
                start_cmd = os.path.expanduser("~") + start_cmd
        elif platform.system() == "Windows":
            start_cmd = (self._find_exe_in_registry() or self._default_windows_location())
        elif platform.system() == 'Java' and os._name == 'nt':
            start_cmd = self._default_windows_location()
        else:
            for ffname in ["firefox", "iceweasel"]:
                start_cmd = self.which(ffname)
                if start_cmd is not None:
                    break
            else:
                # couldn't find firefox on the system path
                raise RuntimeError(
                    "Could not find firefox in your system PATH." +
                    " Please specify the firefox binary location or install firefox")
        return start_cmd

    def _default_windows_location(self):
        program_files = [os.getenv("PROGRAMFILES", r"C:\Program Files"),
                         os.getenv("PROGRAMFILES(X86)", r"C:\Program Files (x86)")]
        for path in program_files:
            binary_path = os.path.join(path, r"Mozilla Firefox\firefox.exe")
            if os.access(binary_path, os.X_OK):
                return binary_path
        return ""

    def _modify_link_library_path(self):
        existing_ld_lib_path = os.environ.get('LD_LIBRARY_PATH', '')

        new_ld_lib_path = self._extract_and_check(
            self.profile, self.NO_FOCUS_LIBRARY_NAME, "x86", "amd64")

        new_ld_lib_path += existing_ld_lib_path

        self._firefox_env["LD_LIBRARY_PATH"] = new_ld_lib_path
        self._firefox_env['LD_PRELOAD'] = self.NO_FOCUS_LIBRARY_NAME

    def _extract_and_check(self, profile, no_focus_so_name, x86, amd64):

        paths = [x86, amd64]
        built_path = ""
        for path in paths:
            library_path = os.path.join(profile.path, path)
            if not os.path.exists(library_path):
                os.makedirs(library_path)
            import shutil
            shutil.copy(os.path.join(
                os.path.dirname(__file__),
                path,
                self.NO_FOCUS_LIBRARY_NAME),
                library_path)
            built_path += library_path + ":"

        return built_path

    def which(self, fname):
        """Returns the fully qualified path by searching Path of the given
        name"""
        for pe in os.environ['PATH'].split(os.pathsep):
            checkname = os.path.join(pe, fname)
            if os.access(checkname, os.X_OK) and not os.path.isdir(checkname):
                return checkname
        return None
Beispiel #36
0
def network(s, **kwargs):
    """pysimm.apps.zeopp.network

    Perform 1. Pore diameters; 2. Channel identification and dimensionality; 3. Surface area;
            4. Accessible volume; 5. Pore size distribution calculation using zeo++ v2.2

    with options to do 6. Probe-occupiable volume; 7. Stochastic ray tracing; 8. Blocking spheres;
                       9. Distance grids; 10. Structure analysis

    Args:
        s: pysimm System object or filename of file in CSSR | CUC | V1 | CIF format
        atype_name: True to use atom type as atom name (usually need radii and mass info), False to use atom element
        radii: file name that contain atom radii data (rad.rad)
        mass: file name that contain atom mass data (mass.mass)
        probe_radius: radius of a probe used in sampling of surface (1.2 A)
        chan_radius: radius of a probe used to determine accessibility of void space (1.2 A)
        num_samples: number of Monte Carlo samples per unit cell (50000)
        option to include in the simulation: set True to activate
            ha: default=True, for using high accuracy,
            res: default=True, for diameters of the largest included sphere, the largest free sphere and the largest included sphere along free sphere path
            chan: default=True, for channel systems characterized by dimensionality as well as Di, Df and Dif
            sa: default=True, for surface area accessible to a spherical probe, characterized by
                      accessible surface area (ASA) and non-accessible surface area (NASA)
            vol: default=True, for accessible volume (AV) and non-accessible volume (NAV)
            volpo: default=False, for accessible proce-occupiable volume (POAV) and non-accessible probe-occupiable volume (PONAV)
            psd: default=True, for the "deriviative distribution" (change of AV w.r.t probe size) reported in the histogram file with 1000 bins of size of 0.1 Ang
            ray_atom: default=False
            block: default=False
            extra: user provided options, such as -gridG, -gridBOV, -strinfo, -oms, etc.
            
    ZEOpp_EXEC: path to zeo++ executable (network)

    Returns:
        None
    """
    global ZEOpp_EXEC

    if ZEOpp_EXEC is None:
        print('Please specify the environment variable '
              'ZEOpp_EXEC'
              ' that points to '
              'zeo++ executable (network)')
        exit(1)

    probe_radius = kwargs.get('probe_radius', 1.2)
    chan_radius = kwargs.get('chan_radius', 1.2)
    num_samples = kwargs.get('num_samples', 50000)
    atype_name = kwargs.get('atype_name', False)

    ha = kwargs.get('ha', True)
    res = kwargs.get('res', True)
    chan = kwargs.get('chan', True)
    sa = kwargs.get('sa', True)
    vol = kwargs.get('vol', True)
    psd = kwargs.get('psd', True)
    volpo = kwargs.get('volpo', False)
    ray_atom = kwargs.get('ray_atom', False)
    block = kwargs.get('block', False)
    extra = kwargs.get('extra')

    nanohub = kwargs.get('nanohub')

    if isinstance(s, system.System):
        if atype_name:
            s.write_cssr('zeopp_data.cssr', aname=1)
        else:
            s.write_cssr('zeopp_data.cssr')
        input_file = 'zeopp_data.cssr'
    elif isinstance(s, str):
        input_file = s

    args = ZEOpp_EXEC

    if 'radii' in kwargs.keys():
        args += ' -r ' + kwargs.get('radii')
    if 'mass' in kwargs.keys():
        args += ' -mass ' + kwargs.get('mass')

    if ha:
        args += ' -ha'
    if res:
        args += ' -res'
    if chan:
        args += ' -chan ' + str(probe_radius)
    if sa:
        args += ' -sa ' + str(chan_radius) + ' ' + str(
            probe_radius) + ' ' + str(num_samples)
    if vol:
        args += ' -vol ' + str(chan_radius) + ' ' + str(
            probe_radius) + ' ' + str(num_samples)
    if psd:
        args += ' -psd ' + str(chan_radius) + ' ' + str(
            probe_radius) + ' ' + str(num_samples)
    if volpo:
        args += ' -volpo ' + str(chan_radius) + ' ' + str(
            probe_radius) + ' ' + str(num_samples)
    if ray_atom:
        args += ' -ray_atom ' + str(chan_radius) + ' ' + str(
            probe_radius) + ' ' + str(num_samples)
    if block:
        args += ' -block ' + str(probe_radius) + ' ' + str(num_samples)
    if extra:
        args += ' ' + extra

    args += ' ' + input_file

    arg_list = shlex.split(args)

    print('%s: starting simulation using zeo++' % strftime('%H:%M:%S'))

    if nanohub:
        print('%s: sending zeo++ simulation to computer cluster' %
              strftime('%H:%M:%S'))
        sys.stdout.flush()
        cmd = ('submit -n 1 -w %s ' % (24 * 60)) + ZEOpp_EXEC + args
        cmd = shlex.split(cmd)
        exit_status, stdo, stde = RapptureExec(cmd)
    else:
        p = Popen(arg_list, stdin=PIPE, stdout=PIPE, stderr=PIPE)
        while True:
            stout = p.stdout.readline()
            if stout == '' and p.poll() is not None:
                break
            if stout:
                print(stout.strip())
        # print(stout)
        sterr = p.stderr.readlines()
        print(sterr)

    print('%s: zeo++ simulation successful' % strftime('%H:%M:%S'))
Beispiel #37
0
    def execute_code(self, event=None):
        if not getattr(self.shared, "src_image", None):
            self.execblink("No image", "#ffff00")
            self.status("No image given")
            return

        if event != "RUN":
            thr = Thread(target=self.execute_code, args=("RUN", ))
            thr.start()
            return

        self.status("Starting...")
        self.profiler_canvas.delete("top-text")
        self.profiler_canvas.delete("pieslice")
        self.update()

        self.shared.profiler_info = {}
        code = self.codeblock._text.get('1.0', 'end')
        tempcode = tempfile.mkstemp('.py', 'detcher-')[1]
        with open(tempcode, 'w') as fd:
            fd.write(code)
        self.status("Preparing...")
        tempimg = tempfile.mkstemp('.png', 'detcher-')[1]
        self.shared.src_image.save(tempimg)

        self.execbutton.config(text="Kill", fg="#ff0000", state="normal")
        self.status("Running...")

        proc = Popen([sys.executable, "-u", "-B", sys.argv[0], "--IS-HELPER"],
                     stdin=PIPE,
                     stdout=PIPE)

        self.shared.proc = proc
        proc.stdin.write(tempcode.encode() + b"\n")
        proc.stdin.write(tempimg.encode() + b"\n")
        proc.stdin.write(f"{self.shared.seed}\n".encode())
        proc.stdin.flush()

        err = [None, None]
        out, tmpdir = None, None
        line = b'-- -- --'

        while proc.poll() is None or len(line):
            line = proc.stdout.readline()
            if not line:
                continue
            if proc.poll():
                print("[INFO] Process finished but buffer is not empty")
            ln = line.decode().rstrip()
            if ln.startswith("\x1b"):
                cmd = ln.lstrip("\x1b")
                params = cmd.split(":")
                print(f">> {cmd}")
                name = params[0]
                if name == "OKDONE":
                    out = ":".join(params[1:])
                if name == "DR":
                    ok, name, cpu, real = params[1:]
                    cpu, real = map(float, [cpu, real])
                    self.after_call(name, [cpu, real])
                if name == "ERR":
                    err[0] = ":".join(params[1:])
                if name == "ERRF":
                    err[1] = ":".join(params[1:])
                if name == "ERR_NOTB":
                    err = [":".join(params[1:]), None]
                if name == "TMP":
                    tmpdir = ":".join(params[1:])
                if name == "DBG":
                    self.status(":".join(params[1:]), "#7878fa")
            else:
                print(f"[SUB][STDOUT]: {ln}")
                self.status(f"[sub] {ln.rstrip()}", "#7878fa")
        proc.wait()

        exitcode = proc.poll()
        print(f"[SUB] -> {exitcode}")

        self.shared.proc = None
        if exitcode:
            short_e, e_file = err
            trace = "-- no trace --"
            if e_file:
                with open(e_file, "r") as fd:
                    trace = fd.read()
                rm_file(e_file)
            self.execblink("Terminated" if exitcode < 0 else "Error",
                           "#ff0000")
            self.status(f"{short_e or 'Terminated'}", "#ff0000")
            print(trace)
            self.shared.proc = None
            if tmpdir:
                shutil.rmtree(tmpdir)
            return

        self.status("Loading image...")
        self.shared.out_image = Image.open(out).copy()
        self.status("Cleaning files...")
        rm_file(tempcode)
        rm_file(tempimg)
        if tmpdir:
            shutil.rmtree(tmpdir)
        self.status("Updating...")
        self.update_image()
        self.execbutton.config(text="Execute", fg="#78fa78")
        self.status("Done.")
Beispiel #38
0
def lcall(cmd,
          logger,
          outlvl=logging.INFO,
          errlvl=logging.ERROR,
          timeout=None,
          **kwargs):
    """
    Variant of subprocess.call that accepts a logger instead of stdout/stderr,
    and logs stdout messages via logger.debug and stderr messages via
    logger.error.
    """
    start = time.time()
    if "close_fds" not in kwargs:
        kwargs["close_fds"] = close_fds
    os.environ["PYTHONIOENCODING"] = "UTF-8"
    rout, wout = os.pipe()
    rerr, werr = os.pipe()
    proc = Popen(cmd, stdout=wout, stderr=werr, **kwargs)
    log_level = {rout: outlvl, rerr: errlvl}
    pending = {rout: "", rerr: ""}
    terminated = False
    killed = False

    def check_io():
        logged = 0
        rlist, _, xlist = select.select([rout, rerr], [], [], 0.2)
        if xlist:
            return logged
        for io in rlist:
            buff = os.read(io, 32768)
            buff = bdecode(buff)
            if six.PY2:
                buff = buff.decode("utf8")
            if buff in ('', b''):
                continue
            buff = pending[io] + buff
            while True:
                l = buff.split("\n", 1)
                if len(l) == 1:
                    pending[io] = l[0]
                    break
                line, buff = l
                if logger:
                    logger.log(log_level[io], "| " + line)
                elif log_level[io] < logging.ERROR:
                    print(line)
                else:
                    print(line, file=sys.stderr)
                logged += 1
        return logged

    # keep checking stdout/stderr until the proc exits
    while proc.poll() is None:
        check_io()
        ellapsed = time.time() - start
        if timeout and ellapsed > timeout:
            if not terminated:
                if logger:
                    logger.error(
                        "execution timeout (%.1f seconds). send SIGTERM." %
                        timeout)
                else:
                    print("execution timeout (%.1f seconds). send SIGTERM." %
                          timeout,
                          file=sys.stderr)
                proc.terminate()
                terminated = True
            elif not killed and ellapsed > timeout * 2:
                if logger:
                    logger.error(
                        "SIGTERM handling timeout (%.1f seconds). send SIGKILL."
                        % timeout)
                else:
                    print(
                        "SIGTERM handling timeout (%.1f seconds). send SIGKILL."
                        % timeout,
                        file=sys.stderr)
                proc.kill()
                killed = True

    while True:
        # check again to catch anything after the process exits
        logged = check_io()
        if logged == 0:
            break
    for io in rout, rerr:
        line = pending[io]
        if line:
            if logger:
                logger.log(log_level[io], "| " + line)
            elif log_level[io] < logging.ERROR:
                print(line)
            else:
                print(line, file=sys.stderr)
    os.close(rout)
    os.close(rerr)
    os.close(wout)
    os.close(werr)
    return proc.returncode
Beispiel #39
0
    def start(self, cb=None):
        def which(program, paths):
            def is_exe(fpath):
                return os.path.exists(fpath) and os.access(fpath, os.X_OK)

            for path in paths:
                if not os.path.isdir(path):
                    continue
                exe_file = os.path.join(u(path, fs_encoding), program)
                if is_exe(exe_file):
                    return file_quote(exe_file)
            return None

        def convert_environment_variables(env):
            """
            This function is use to convert environment variable to string
            because environment variable must be string in popen
            :param env: Dict of environment variable
            :return: Encoded environment variable as string
            """
            encoding = sys.getdefaultencoding()
            if encoding is None or encoding == 'ascii':
                encoding = 'utf-8'
            temp_env = dict()
            for key, value in env.items():
                if not isinstance(key, str):
                    key = key.encode(encoding)
                if not isinstance(value, str):
                    value = value.encode(encoding)
                temp_env[key] = value
            return temp_env

        if self.stime is not None:
            if self.etime is None:
                raise Exception(_('The process has already been started.'))
            raise Exception(
                _('The process has already finished and cannot be restarted.'))

        executor = file_quote(
            os.path.join(os.path.dirname(u(__file__)), u'process_executor.py'))
        paths = os.environ['PATH'].split(os.pathsep)
        interpreter = None

        if os.name == 'nt':
            paths.insert(0, os.path.join(u(sys.prefix), u'Scripts'))
            paths.insert(0, u(sys.prefix))

            interpreter = which(u'pythonw.exe', paths)
            if interpreter is None:
                interpreter = which(u'python.exe', paths)

            if interpreter is None and current_app.PGADMIN_RUNTIME:
                # We've faced an issue with Windows 2008 R2 (x86) regarding,
                # not honouring the environment variables set under the Qt
                # (e.g. runtime), and also setting PYTHONHOME same as
                # sys.executable (i.e. pgAdmin4.exe).
                #
                # As we know, we're running it under the runtime, we can assume
                # that 'venv' directory will be available outside of 'bin'
                # directory.
                #
                # We would try out luck to find python executable based on that
                # assumptions.
                bin_path = os.path.dirname(sys.executable)

                venv = os.path.realpath(os.path.join(bin_path, u'..\\venv'))

                interpreter = which(u'pythonw.exe', [venv])
                if interpreter is None:
                    interpreter = which(u'pythonw.exe', [venv])

                if interpreter is not None:
                    # Our assumptions are proven right.
                    # Let's append the 'bin' directory to the PATH environment
                    # variable. And, also set PYTHONHOME environment variable
                    # to 'venv' directory.
                    os.environ['PATH'] = bin_path + ';' + os.environ['PATH']
                    os.environ['PYTHONHOME'] = venv
        else:
            # Let's not use sys.prefix in runtime.
            # 'sys.prefix' is not identified on *nix systems for some unknown
            # reason, while running under the runtime.
            # We're already adding '<installation path>/pgAdmin 4/venv/bin'
            # directory in the PATH environment variable. Hence - it will
            # anyway be the redundant value in paths.
            if not current_app.PGADMIN_RUNTIME:
                paths.insert(0, os.path.join(u(sys.prefix), u'bin'))
            python_binary_name = 'python{0}'.format(sys.version_info[0]) \
                if sys.version_info[0] >= 3 else 'python'
            interpreter = which(u(python_binary_name), paths)

        p = None
        cmd = [
            interpreter if interpreter is not None else 'python', executor,
            self.cmd
        ]
        cmd.extend(self.args)

        current_app.logger.info(
            u"Executing the process executor with the arguments: %s", str(cmd))

        # Make a copy of environment, and add new variables to support
        env = os.environ.copy()
        env['PROCID'] = self.id
        env['OUTDIR'] = self.log_dir
        env['PGA_BGP_FOREGROUND'] = "1"

        if self.env:
            env.update(self.env)

        if cb is not None:
            cb(env)

        if os.name == 'nt':
            DETACHED_PROCESS = 0x00000008
            from subprocess import CREATE_NEW_PROCESS_GROUP

            # We need to redirect the standard input, standard output, and
            # standard error to devnull in order to allow it start in detached
            # mode on
            stdout = os.devnull
            stderr = stdout
            stdin = open(os.devnull, "r")
            stdout = open(stdout, "a")
            stderr = open(stderr, "a")

            p = Popen(cmd,
                      close_fds=False,
                      env=env,
                      stdout=stdout.fileno(),
                      stderr=stderr.fileno(),
                      stdin=stdin.fileno(),
                      creationflags=(CREATE_NEW_PROCESS_GROUP
                                     | DETACHED_PROCESS))
        else:

            def preexec_function():
                import signal
                # Detaching from the parent process group
                os.setpgrp()
                # Explicitly ignoring signals in the child process
                signal.signal(signal.SIGINT, signal.SIG_IGN)

            # if in debug mode, wait for process to complete and
            # get the stdout and stderr of popen.
            if config.CONSOLE_LOG_LEVEL <= logging.DEBUG:
                p = Popen(cmd,
                          close_fds=True,
                          stdout=PIPE,
                          stderr=PIPE,
                          stdin=None,
                          preexec_fn=preexec_function,
                          env=env)

                output, errors = p.communicate()
                output = output.decode() \
                    if hasattr(output, 'decode') else output
                errors = errors.decode() \
                    if hasattr(errors, 'decode') else errors
                current_app.logger.debug(
                    'Process Watcher Out:{0}'.format(output))
                current_app.logger.debug(
                    'Process Watcher Err:{0}'.format(errors))
            else:
                p = Popen(cmd,
                          close_fds=True,
                          stdout=None,
                          stderr=None,
                          stdin=None,
                          preexec_fn=preexec_function,
                          env=env)

        self.ecode = p.poll()

        # Execution completed immediately.
        # Process executor cannot update the status, if it was not able to
        # start properly.
        if self.ecode is not None and self.ecode != 0:
            # There is no way to find out the error message from this process
            # as standard output, and standard error were redirected to
            # devnull.
            p = Process.query.filter_by(pid=self.id,
                                        user_id=current_user.id).first()
            p.start_time = p.end_time = get_current_time()
            if not p.exit_code:
                p.exit_code = self.ecode
            p.process_state = PROCESS_FINISHED
            db.session.commit()
        else:
            # Update the process state to "Started"
            p = Process.query.filter_by(pid=self.id,
                                        user_id=current_user.id).first()
            p.process_state = PROCESS_STARTED
            db.session.commit()
Beispiel #40
0
 def check_coap_client():
     p = Popen('coap-client', stdout=PIPE, stderr=PIPE, shell=True)
     time.sleep(0.05)
     if p.poll() == 127:
         print('coap-client not install. Please follow instructions from the README file')
         sys.exit(1)
Beispiel #41
0
def Server (args):
    # FIXME: this  is convenience hack, one should  decide either args
    # is a list or a string:
    if type (args) == type (""):
        args = shlex.split (args)

    # Root process such as naive backup can still enter this directory
    # and read/write to the fifo pipes:
    tmp = mkdtemp()

    # Just one possible choice of names:
    inp = os.path.join (tmp, "%inp")
    out = os.path.join (tmp, "%out")

    os.mkfifo (inp)
    os.mkfifo (out)

    # args[]  is supposed  to include  the executable  and  flags, the
    # subcommand   and   two  more   positional   arguments  will   be
    # appended. The  "server" subcommand must be  the first positional
    # argument in order to be interpreted as such:
    proc = Popen (args + ["server", inp, out])

    # This  function  takes  input  text  and returns  output  of  the
    # subprocess, both communicated via the fifos:
    def server (x):
        try:
            with open (inp, "w") as f:
                f.write (x)
            with open (out, "r") as f:
                y = f.read ()
        except KeyboardInterrupt as e:
            #
            # FIXME:  If the IO  is screwed,  we will  not be  able to
            # regularly shut  down the daemon.  Kill  it with SIGTERM.
            # Some minimal testing showed that proc.poll() will return
            # something  trueish  afterwards.   Also  if you  catch  a
            # generic  Exception here  the chances  are slim  that you
            # will catch  an asynchrounous KeyboardInterrupt  here and
            # now (see StackOverflow).
            #
            proc.terminate()
            proc.wait()
            raise e
        return y

    # This is a valid Func:
    func = protocol (server)

    # If an  exception happens in the  body of the  "with" statment we
    # will note it here:
    try:
        #
        # The control returns to the
        #
        #   with Server(...) as func:
        #
        # statement body:
        #
        yield func
    finally:
        #
        # When  leaving  the   "with"  statement  body  execute  this.
        # Executed  in  any  case,   even  if  exceptions  such  as  a
        # KeyboardInterrupt occur.  We do  not want zombie daemons and
        # stray files.
        #
        # Tell the daemon process to terminate.  This will not work if
        # user hit C-c while the  server() was already doing IO on the
        # fifos. In this case  the process should have been terminated
        # there.
        #
        if not proc.poll():
            # Regular shutdown sequence:
            with open (inp, "w") as f:
                f.write ("#f")  # convention
            proc.wait ()

        os.unlink (inp)
        os.unlink (out)
        os.rmdir (tmp)
Beispiel #42
0
class CheckUpdates(base.ThreadPoolText):
    """Shows number of pending updates in different unix systems"""
    orientations = base.ORIENTATION_HORIZONTAL
    defaults = [
        ("distro", "Arch", "Name of your distribution"),
        ("custom_command", None, "Custom shell command for checking updates (counts the lines of the output)"),
        ("custom_command_modify", (lambda x: x), "Lambda function to modify line count from custom_command"),
        ("update_interval", 60, "Update interval in seconds."),
        ('execute', None, 'Command to execute on click'),
        ("display_format", "Updates: {updates}", "Display format if updates available"),
        ("colour_no_updates", "ffffff", "Colour when there's no updates."),
        ("colour_have_updates", "ffffff", "Colour when there are updates."),
        ("restart_indicator", "", "Indicator to represent reboot is required. (Ubuntu only)"),
        ("no_update_string", "", "String to display if no updates available")
    ]

    def __init__(self, **config):
        base.ThreadPoolText.__init__(self, "", **config)
        self.add_defaults(CheckUpdates.defaults)

        # Helpful to have this as a variable as we can shorten it for testing
        self.execute_polling_interval = 1

        # format: "Distro": ("cmd", "number of lines to subtract from output")
        self.cmd_dict = {"Arch": ("pacman -Qu", 0),
                         "Arch_checkupdates": ("checkupdates", 0),
                         "Arch_Sup": ("pacman -Sup", 1),
                         "Arch_yay": ("yay -Qu", 0),
                         "Debian": ("apt-show-versions -u -b", 0),
                         "Ubuntu": ("aptitude search ~U", 0),
                         "Fedora": ("dnf list updates -q", 1),
                         "FreeBSD": ("pkg_version -I -l '<'", 0),
                         "Mandriva": ("urpmq --auto-select", 0)
                         }

        if self.custom_command:
            # Use custom_command
            self.cmd = self.custom_command

        else:
            # Check if distro name is valid.
            try:
                self.cmd = self.cmd_dict[self.distro][0]
                self.custom_command_modify = (lambda x: x - self.cmd_dict[self.distro][1])
            except KeyError:
                distros = sorted(self.cmd_dict.keys())
                logger.error(self.distro + ' is not a valid distro name. ' +
                             'Use one of the list: ' + str(distros) + '.')
                self.cmd = None

        if self.execute:
            self.add_callbacks({'Button1': self.do_execute})

    def _check_updates(self):
        # type: () -> str
        try:
            updates = self.call_process(self.cmd, shell=True)
        except CalledProcessError:
            updates = ""
        num_updates = self.custom_command_modify(len(updates.splitlines()))

        if num_updates < 0:
            num_updates = 0
        if num_updates == 0:
            self.layout.colour = self.colour_no_updates
            return self.no_update_string
        num_updates = str(num_updates)

        if self.restart_indicator and os.path.exists('/var/run/reboot-required'):
            num_updates += self.restart_indicator

        self.layout.colour = self.colour_have_updates
        return self.display_format.format(**{"updates": num_updates})

    def poll(self):
        # type: () -> str
        if not self.cmd:
            return "N/A"
        return self._check_updates()

    def do_execute(self):
        self._process = Popen(self.execute, shell=True)
        self.timeout_add(self.execute_polling_interval, self._refresh_count)

    def _refresh_count(self):
        if self._process.poll() is None:
            self.timeout_add(self.execute_polling_interval, self._refresh_count)

        else:
            self.timer_setup()
Beispiel #43
0
    def _read_progress(self, proc: subprocess.Popen, pbar: List[Any],
                       idx: int) -> bytes:
        """
        Update tqdm progress bars according to CmdStan console progress msgs.
        Poll process to get CmdStan console outputs,
        check for output lines that start with 'Iteration: '.
        NOTE: if CmdStan output messages change, this will break.
        """
        pattern = (
            r'^Iteration\:\s*(\d+)\s*/\s*(\d+)\s*\[\s*\d+%\s*\]\s*\((\S*)\)$')
        pattern_compiled = re.compile(pattern, flags=re.IGNORECASE)
        pbar_warmup, pbar_sampling = pbar
        num_warmup = pbar_warmup.total
        num_sampling = pbar_sampling.total
        count_warmup = 0
        count_sampling = 0
        stdout = b''

        try:
            # iterate while process is sampling
            while proc.poll() is None:
                output = proc.stdout.readline()
                stdout += output
                output = output.decode('utf-8').strip()
                refresh_warmup = True
                if output.startswith('Iteration'):
                    match = re.search(pattern_compiled, output)
                    if match:
                        # check if pbars need reset
                        if num_warmup is None or num_sampling is None:
                            total_count = int(match.group(2))
                            if num_warmup is None and num_sampling is None:
                                num_warmup = total_count // 2
                                num_sampling = total_count - num_warmup
                                pbar_warmup.total = num_warmup
                                pbar_sampling.total = num_sampling
                            elif num_warmup is None:
                                num_warmup = total_count - num_sampling
                                pbar_warmup.total = num_warmup
                            else:
                                num_sampling = total_count - num_warmup
                                pbar_sampling.total = num_sampling
                        # raw_count = warmup + sampling
                        raw_count = int(match.group(1))
                        if match.group(3).lower() == 'warmup':
                            count, count_warmup = (
                                raw_count - count_warmup,
                                raw_count,
                            )
                            pbar_warmup.update(count)
                        elif match.group(3).lower() == 'sampling':
                            # refresh warmup and close the progress bar
                            if refresh_warmup:
                                pbar_warmup.update(num_warmup - count_warmup)
                                pbar_warmup.refresh()
                                pbar_warmup.close()
                                refresh_warmup = False
                            # update values to full
                            count, count_sampling = (
                                raw_count - num_warmup - count_sampling,
                                raw_count - num_warmup,
                            )
                            pbar_sampling.update(count)

            # read and process rest of the stdout if needed
            warmup_cumulative_count = 0
            sampling_cumulative_count = 0
            for output in proc.stdout:
                stdout += output
                output = output.decode('utf-8').strip()
                if output.startswith('Iteration'):
                    match = re.search(pattern_compiled, output)
                    if match:
                        # check if pbars need reset
                        if num_warmup is None or num_sampling is None:
                            total_count = int(match.group(2))
                            if num_warmup is None and num_sampling is None:
                                num_warmup = total_count // 2
                                num_sampling = total_count - num_warmup
                                pbar_warmup.total = num_warmup
                                pbar_sampling.total = num_sampling
                            elif num_warmup is None:
                                num_warmup = total_count - num_sampling
                                pbar_warmup.total = num_warmup
                            else:
                                num_sampling = total_count - num_warmup
                                pbar_sampling.total = num_sampling
                        # raw_count = warmup + sampling
                        raw_count = int(match.group(1))
                        if match.group(3).lower() == 'warmup':
                            count, count_warmup = (
                                raw_count - count_warmup,
                                raw_count,
                            )
                            warmup_cumulative_count += count
                        elif match.group(3).lower() == 'sampling':
                            count, count_sampling = (
                                raw_count - num_warmup - count_sampling,
                                raw_count - num_warmup,
                            )
                            sampling_cumulative_count += count
            # update warmup pbar if needed
            if warmup_cumulative_count:
                pbar_warmup.update(warmup_cumulative_count)
                pbar_warmup.refresh()
            # update sampling pbar if needed
            if sampling_cumulative_count:
                pbar_sampling.update(sampling_cumulative_count)
                pbar_sampling.refresh()

        except Exception as e:
            self._logger.warning(
                'Chain %s: Failed to read the progress on the fly. Error: %s',
                idx,
                e,
            )
        # close both pbar
        pbar_warmup.close()
        pbar_sampling.close()

        # return stdout
        return stdout
Beispiel #44
0
class ConfigurableHTTPProxy(Proxy):
    """Proxy implementation for the default configurable-http-proxy.

    This is the default proxy implementation
    for running the nodejs proxy `configurable-http-proxy`.

    If the proxy should not be run as a subprocess of the Hub,
    (e.g. in a separate container),
    set::
    
        c.ConfigurableHTTPProxy.should_start = False
    """

    proxy_process = Any()
    client = Instance(AsyncHTTPClient, ())

    debug = Bool(False,
                 help="Add debug-level logging to the Proxy.",
                 config=True)
    auth_token = Unicode(help="""The Proxy auth token

        Loaded from the CONFIGPROXY_AUTH_TOKEN env variable by default.
        """, ).tag(config=True)
    check_running_interval = Integer(5, config=True)

    @default('auth_token')
    def _auth_token_default(self):
        token = os.environ.get('CONFIGPROXY_AUTH_TOKEN', None)
        if not token:
            self.log.warning('\n'.join([
                "",
                "Generating CONFIGPROXY_AUTH_TOKEN. Restarting the Hub will require restarting the proxy.",
                "Set CONFIGPROXY_AUTH_TOKEN env or JupyterHub.proxy_auth_token config to avoid this message.",
                "",
            ]))
            token = utils.new_token()
        return token

    api_url = Unicode(
        'http://127.0.0.1:8001',
        config=True,
        help="""The ip (or hostname) of the proxy's API endpoint""")
    command = Command('configurable-http-proxy',
                      config=True,
                      help="""The command to start the proxy""")

    @gen.coroutine
    def start(self):
        public_server = Server.from_url(self.public_url)
        api_server = Server.from_url(self.api_url)
        env = os.environ.copy()
        env['CONFIGPROXY_AUTH_TOKEN'] = self.auth_token
        cmd = self.command + [
            '--ip',
            public_server.ip,
            '--port',
            str(public_server.port),
            '--api-ip',
            api_server.ip,
            '--api-port',
            str(api_server.port),
            '--error-target',
            url_path_join(self.hub.url, 'error'),
        ]
        if self.app.subdomain_host:
            cmd.append('--host-routing')
        if self.debug:
            cmd.extend(['--log-level', 'debug'])
        if self.ssl_key:
            cmd.extend(['--ssl-key', self.ssl_key])
        if self.ssl_cert:
            cmd.extend(['--ssl-cert', self.ssl_cert])
        if self.app.statsd_host:
            cmd.extend([
                '--statsd-host', self.app.statsd_host, '--statsd-port',
                str(self.app.statsd_port), '--statsd-prefix',
                self.app.statsd_prefix + '.chp'
            ])
        # Warn if SSL is not used
        if ' --ssl' not in ' '.join(cmd):
            self.log.warning(
                "Running JupyterHub without SSL."
                "  I hope there is SSL termination happening somewhere else..."
            )
        self.log.info("Starting proxy @ %s", public_server.bind_url)
        self.log.debug("Proxy cmd: %s", cmd)
        try:
            self.proxy_process = Popen(cmd, env=env, start_new_session=True)
        except FileNotFoundError as e:
            self.log.error(
                "Failed to find proxy %r\n"
                "The proxy can be installed with `npm install -g configurable-http-proxy`"
                % self.command)
            raise

        def _check_process():
            status = self.proxy_process.poll()
            if status is not None:
                e = RuntimeError("Proxy failed to start with exit code %i" %
                                 status)
                # py2-compatible `raise e from None`
                e.__cause__ = None
                raise e

        for server in (public_server, api_server):
            for i in range(10):
                _check_process()
                try:
                    yield server.wait_up(1)
                except TimeoutError:
                    continue
                else:
                    break
            yield server.wait_up(1)
        _check_process()
        self.log.debug("Proxy started and appears to be up")
        pc = PeriodicCallback(self.check_running,
                              1e3 * self.check_running_interval)
        pc.start()

    def stop(self):
        self.log.info("Cleaning up proxy[%i]...", self.proxy_process.pid)
        if self.proxy_process.poll() is None:
            try:
                self.proxy_process.terminate()
            except Exception as e:
                self.log.error("Failed to terminate proxy process: %s", e)

    @gen.coroutine
    def check_running(self):
        """Check if the proxy is still running"""
        if self.proxy_process.poll() is None:
            return
        self.log.error(
            "Proxy stopped with exit code %r", 'unknown'
            if self.proxy_process is None else self.proxy_process.poll())
        yield self.start()
        yield self.restore_routes()

    def _routespec_to_chp_path(self, routespec):
        """Turn a routespec into a CHP API path

        For host-based routing, CHP uses the host as the first path segment.
        """
        path = self.validate_routespec(routespec)
        # CHP always wants to start with /
        if not path.startswith('/'):
            path = '/' + path
        # BUG: CHP doesn't seem to like trailing slashes on some endpoints (DELETE)
        if path != '/' and path.endswith('/'):
            path = path.rstrip('/')
        return path

    def _routespec_from_chp_path(self, chp_path):
        """Turn a CHP route into a route spec
        
        In the JSON API, CHP route keys are unescaped,
        so re-escape them to raw URLs and ensure slashes are in the right places.
        """
        # chp stores routes in unescaped form.
        # restore escaped-form we created it with.
        routespec = quote(chp_path, safe='@/')
        if self.host_routing:
            # host routes don't start with /
            routespec = routespec.lstrip('/')
        # all routes should end with /
        if not routespec.endswith('/'):
            routespec = routespec + '/'
        return routespec

    def api_request(self, path, method='GET', body=None, client=None):
        """Make an authenticated API request of the proxy."""
        client = client or AsyncHTTPClient()
        url = url_path_join(self.api_url, 'api/routes', path)

        if isinstance(body, dict):
            body = json.dumps(body)
        self.log.debug("Proxy: Fetching %s %s", method, url)
        req = HTTPRequest(
            url,
            method=method,
            headers={'Authorization': 'token {}'.format(self.auth_token)},
            body=body,
        )

        return client.fetch(req)

    def add_route(self, routespec, target, data):
        body = data or {}
        body['target'] = target
        body['jupyterhub'] = True
        path = self._routespec_to_chp_path(routespec)
        return self.api_request(
            path,
            method='POST',
            body=body,
        )

    def delete_route(self, routespec):
        path = self._routespec_to_chp_path(routespec)
        return self.api_request(path, method='DELETE')

    def _reformat_routespec(self, routespec, chp_data):
        """Reformat CHP data format to JupyterHub's proxy API."""
        target = chp_data.pop('target')
        chp_data.pop('jupyterhub')
        return {
            'routespec': routespec,
            'target': target,
            'data': chp_data,
        }

    @gen.coroutine
    def get_all_routes(self, client=None):
        """Fetch the proxy's routes."""
        resp = yield self.api_request('', client=client)
        chp_routes = json.loads(resp.body.decode('utf8', 'replace'))
        all_routes = {}
        for chp_path, chp_data in chp_routes.items():
            routespec = self._routespec_from_chp_path(chp_path)
            if 'jupyterhub' not in chp_data:
                # exclude routes not associated with JupyterHub
                self.log.debug("Omitting non-jupyterhub route %r", routespec)
                continue
            all_routes[routespec] = self._reformat_routespec(
                routespec, chp_data)
        return all_routes
Beispiel #45
0
    def fastMerge(self):
        filelist = []
        outputfile = self._outputFiles[0].value()
        for file in self._inputFiles:
            if file:
                value = file.value()
                if type(value).__name__ == 'list':
                    filelist += value
            else:
                filelist.append(value)
        print "Files to Merge: %s" % filelist

        # First run mergePOOL.exe to get events.pool
        cmd = ['mergePOOL.exe', '-o', 'events.pool.root']
        for file in filelist:
            cmd.extend(['-i', file])
        cmd.extend([
            '-e', 'MetaData', '-e', 'MetaDataHdrDataHeaderForm', '-e',
            'MetaDataHdrDataHeader', '-e', 'MetaDataHdr'
        ])
        print "Will execute hybrid merge step 1: %s" % cmd

        p = Popen(cmd, stdout=PIPE, stderr=STDOUT, close_fds=True)
        while p.poll() is None:
            line = p.stdout.readline()
            if line:
                print "mergePOOL.exe Report: %s" % line.strip()
        rc = p.returncode
        print "1st mergePOOL (event data) finished with code %s" % rc
        if rc == 1:
            print "mergePOOL.exe finished with unknown status (upgrade your RootFileTools to a newer version)"
        elif rc != 0:
            raise TransformError(
                "mergePOOL.exe (event merge) encountered a problem",
                error='TRF_MERGEERR')

        # Second merge with metadata.pool to produce final output
        cmd = ['mergePOOL.exe', '-o', 'events.pool.root', '-i', outputfile]
        print "Will execute hybrid merge step 2: %s" % cmd

        p = Popen(cmd, stdout=PIPE, stderr=STDOUT, close_fds=True)
        while p.poll() is None:
            line = p.stdout.readline()
            if line:
                print "mergePOOL.exe Report: %s" % line.strip()
        rc = p.returncode
        print "2nd mergePOOL (metadata) finished with code %s" % rc
        if rc == 1:
            print "mergePOOL.exe finished with unknown status (upgrade your RootFileTools to a newer version) - assuming all is ok"
        elif rc != 0:
            raise TransformError(
                "mergePOOL.exe (final merge) encountered a problem",
                error='TRF_MERGEERR')

        # Finish hybrid merge by moving the full file to the final output location
        shutil.move('events.pool.root', outputfile)

        # Now fix the metadata, which has been left by POOL as the _stub_ file's metadata
        # so it has the wrong GUID in the PFC
        print 'Now fixing metadata in PFC for %s' % outputfile
        try:
            check_call(['FCdeletePFN', '-p', outputfile])
            correctGUID = None
            p = Popen(['pool_extractFileIdentifier.py', outputfile],
                      stdout=PIPE,
                      stderr=STDOUT,
                      close_fds=True,
                      bufsize=1)
            while p.poll() is None:
                line = p.stdout.readline()
                words = line.split()
                if len(words) >= 2 and outputfile in words[1]:
                    correctGUID = words[0]
            if correctGUID == None or p.returncode != 0:
                raise TransformError(
                    "pool_extractFileIdentifier.py failed to get merged file GUID",
                    error='TRF_MERGEERR')
            print 'GUID is %s' % correctGUID
            check_call([
                'FCregisterPFN', '-p', outputfile, '-t', 'ROOT_All', '-g',
                correctGUID
            ])
        except CalledProcessError, e:
            print 'Attempt to fix PFC with new merged file information failed: %s' % e
Beispiel #46
0
class SystemSSHTransport(Transport):
    def __init__(
        self,
        host: str = "",
        port: int = 22,
        auth_username: str = "",
        auth_private_key: str = "",
        auth_password: str = "",
        auth_strict_key: bool = True,
        timeout_socket: int = 5,
        timeout_transport: int = 5,
        timeout_ops: int = 10,
        timeout_exit: bool = True,
        keepalive: bool = False,
        keepalive_interval: int = 30,
        keepalive_type: str = "",
        keepalive_pattern: str = "\005",
        comms_prompt_pattern: str = r"^[a-z0-9.\-@()/:]{1,32}[#>$]$",
        comms_return_char: str = "\n",
        comms_ansi: bool = False,
        ssh_config_file: str = "",
        ssh_known_hosts_file: str = "",
    ) -> None:
        """
        SystemSSHTransport Object

        Inherit from Transport ABC
        SSH2Transport <- Transport (ABC)

        If using this driver, and passing a ssh_config_file (or setting this argument to `True`),
        all settings in the ssh config file will be superseded by any arguments passed here!

        SystemSSHTransport *always* prefers public key auth if given the option! If auth_private_key
        is set in the provided arguments OR if ssh_config_file is passed/True and there is a key for
        ANY match (i.e. `*` has a key in ssh config file!!), we will use that key! If public key
        auth fails and a username and password is set (manually or by ssh config file), password
        auth will be attempted.

        Note that comms_prompt_pattern, comms_return_char and comms_ansi are only passed here to
        handle "in channel" authentication required by SystemSSH -- these are assigned to private
        attributes in this class and ignored after authentication. If you wish to modify these
        values on a "live" scrapli connection, modify them in the Channel object, i.e.
        `conn.channel.comms_prompt_pattern`. Additionally timeout_ops is passed and assigned to
        _timeout_ops to use the same timeout_ops that is used in Channel to decorate the
        authentication methods here.

        Args:
            host: host ip/name to connect to
            port: port to connect to
            auth_username: username for authentication
            auth_private_key: path to private key for authentication
            auth_password: password for authentication
            auth_strict_key: True/False to enforce strict key checking (default is True)
            timeout_socket: timeout for ssh session to start -- this directly maps to ConnectTimeout
                ssh argument; see `man ssh_config`
            timeout_transport: timeout for transport in seconds. since system ssh is using popen/pty
                we can't really set a timeout directly, so this value governs the time timeout
                decorator for the transport read and write methods
            timeout_ops: timeout for telnet channel operations in seconds -- this is also the
                timeout for finding and responding to username and password prompts at initial
                login. This is assigned to a private attribute and is ignored after authentication
                is completed.
            timeout_exit: True/False close transport if timeout encountered. If False and keepalives
                are in use, keepalives will prevent program from exiting so you should be sure to
                catch Timeout exceptions and handle them appropriately
            keepalive: whether or not to try to keep session alive
            keepalive_interval: interval to use for session keepalives
            keepalive_type: network|standard -- 'network' sends actual characters over the
                transport channel. This is useful for network-y type devices that may not support
                'standard' keepalive mechanisms. 'standard' is not currently implemented for
                system ssh
            keepalive_pattern: pattern to send to keep network channel alive. Default is
                u'\005' which is equivalent to 'ctrl+e'. This pattern moves cursor to end of the
                line which should be an innocuous pattern. This will only be entered *if* a lock
                can be acquired. This is only applicable if using keepalives and if the keepalive
                type is 'network'
            comms_prompt_pattern: prompt pattern expected for device, same as the one provided to
                channel -- system ssh needs to know this to know how to decide if we are properly
                sending/receiving data -- i.e. we are not stuck at some password prompt or some
                other failure scenario. If using driver, this should be passed from driver (Scrape,
                or IOSXE, etc.) to this Transport class. This is assigned to a private attribute and
                is ignored after authentication is completed.
            comms_return_char: return character to use on the channel, same as the one provided to
                channel -- system ssh needs to know this to know what to send so that we can probe
                the channel to make sure we are authenticated and sending/receiving data. If using
                driver, this should be passed from driver (Scrape, or IOSXE, etc.) to this Transport
                class. This is assigned to a private attribute and is ignored after authentication
                is completed.
            comms_ansi: True/False strip comms_ansi characters from output; this value is assigned
                self._comms_ansi and is ignored after authentication. We only need it for transport
                on the off chance (maybe never?) that username/password prompts contain ansi
                characters, otherwise "comms_ansi" is really a channel attribute and is treated as
                such. This is assigned to a private attribute and is ignored after authentication
                is completed.
            ssh_config_file: string to path for ssh config file
            ssh_known_hosts_file: string to path for ssh known hosts file

        Returns:
            N/A  # noqa: DAR202

        Raises:
            N/A

        """
        super().__init__(
            host,
            port,
            timeout_socket,
            timeout_transport,
            timeout_exit,
            keepalive,
            keepalive_interval,
            keepalive_type,
            keepalive_pattern,
        )

        self.auth_username: str = auth_username
        self.auth_private_key: str = auth_private_key
        self.auth_password: str = auth_password
        self.auth_strict_key: bool = auth_strict_key

        self._timeout_ops: int = timeout_ops
        self._comms_prompt_pattern: str = comms_prompt_pattern
        self._comms_return_char: str = comms_return_char
        self._comms_ansi: bool = comms_ansi
        self._process_ssh_config(ssh_config_file)
        self.ssh_known_hosts_file: str = ssh_known_hosts_file

        self.session: Union[Popen[bytes], PtyProcess]  # pylint: disable=E1136
        self.lib_auth_exception = ScrapliAuthenticationFailed
        self._isauthenticated = False

        self.open_cmd = ["ssh", self.host]
        self._build_open_cmd()

        # create stdin/stdout fd in case we can use pipes for session
        self._stdin_fd = -1
        self._stdout_fd = -1

    def _process_ssh_config(self, ssh_config_file: str) -> None:
        """
        Method to parse ssh config file

        Ensure ssh_config_file is valid (if providing a string path to config file), or resolve
        config file if passed True. Search config file for any private key, if ANY matching key is
        found and user has not provided a private key, set `auth_private_key` to the value of the
        found key. This is because we prefer to use `open_pipes` over `open_pty`!

        Args:
            ssh_config_file: string path to ssh config file; passed down from `Scrape`, or the
                `NetworkDriver` or subclasses of it, in most cases.

        Returns:
            N/A  # noqa: DAR202

        Raises:
            N/A

        """
        ssh = SSHConfig(ssh_config_file)
        self.ssh_config_file = ssh.ssh_config_file
        host_config = ssh.lookup(self.host)
        if not self.auth_private_key and host_config.identity_file:
            self.auth_private_key = os.path.expanduser(
                host_config.identity_file.strip())

    def _build_open_cmd(self) -> None:
        """
        Method to craft command to open ssh session

        Args:
            N/A

        Returns:
            N/A  # noqa: DAR202

        Raises:
            N/A

        """
        self.open_cmd.extend(["-p", str(self.port)])
        self.open_cmd.extend(["-o", f"ConnectTimeout={self.timeout_socket}"])
        if self.auth_private_key:
            self.open_cmd.extend(["-i", self.auth_private_key])
        if self.auth_username:
            self.open_cmd.extend(["-l", self.auth_username])
        if self.auth_strict_key is False:
            self.open_cmd.extend(["-o", "StrictHostKeyChecking=no"])
            self.open_cmd.extend(["-o", "UserKnownHostsFile=/dev/null"])
        else:
            self.open_cmd.extend(["-o", "StrictHostKeyChecking=yes"])
            if self.ssh_known_hosts_file:
                self.open_cmd.extend(
                    ["-o", f"UserKnownHostsFile={self.ssh_known_hosts_file}"])
        if self.ssh_config_file:
            self.open_cmd.extend(["-F", self.ssh_config_file])
        else:
            self.open_cmd.extend(["-F", "/dev/null"])

    def open(self) -> None:
        """
        Parent method to open session, authenticate and acquire shell

        If possible it is preferable to use the `_open_pipes` method, but we can only do this IF we
        can authenticate with public key authorization (because we don't have to spawn a PTY; if no
        public key we have to spawn PTY to deal w/ authentication prompts). IF we get a private key
        provided, use pipes method, otherwise we will just deal with `_open_pty`. `_open_pty` is
        less preferable because we have to spawn a PTY and cannot as easily tell if SSH
        authentication is successful. With `_open_pipes` we can read stderr which contains the
        output from the verbose flag for SSH -- this contains a message that indicates success of
        SSH auth. In the case of `_open_pty` we have to read from the channel directly like in the
        case of telnet... so it works, but its just a bit less desirable.

        Args:
            N/A

        Returns:
            N/A  # noqa: DAR202

        Raises:
            ScrapliAuthenticationFailed: if all authentication means fail

        """
        self.session_lock.acquire()

        # If authenticating with private key prefer to use open pipes
        # _open_pipes uses subprocess Popen which is preferable to opening a pty
        # if _open_pipes fails and no password available, raise failure, otherwise try password auth
        if self.auth_private_key:
            open_pipes_result = self._open_pipes()
            if open_pipes_result:
                return
            if not self.auth_password or not self.auth_username:
                msg = (
                    f"Public key authentication to host {self.host} failed. Missing username or"
                    " password unable to attempt password authentication.")
                LOG.critical(msg)
                raise ScrapliAuthenticationFailed(msg)

        # If public key auth fails or is not configured, open a pty session
        if not self._open_pty():
            msg = f"Authentication to host {self.host} failed"
            LOG.critical(msg)
            raise ScrapliAuthenticationFailed(msg)

        if self.keepalive:
            self._session_keepalive()

    def _open_pipes(self) -> bool:
        """
        Private method to open session with subprocess.Popen

        Args:
            N/A

        Returns:
            bool: True/False session was opened and authenticated

        Raises:
            N/A

        """
        # copy the open_cmd as we don't want to update the objects open_cmd until we know we can
        # authenticate. add verbose output and disable batch mode (disables passphrase/password
        # queries). If auth is successful update the object open_cmd to represent what was used
        open_cmd = self.open_cmd.copy()
        open_cmd.append("-v")
        open_cmd.extend(["-o", "BatchMode=yes"])

        stdout_master_pty, stdout_slave_pty = pty.openpty()
        stdin_master_pty, stdin_slave_pty = pty.openpty()

        self.session = Popen(
            open_cmd,
            bufsize=0,
            shell=False,
            stdin=stdin_slave_pty,
            stdout=stdout_slave_pty,
            stderr=PIPE,
        )
        # close the slave fds, don't need them anymore
        os.close(stdin_slave_pty)
        os.close(stdout_slave_pty)
        LOG.debug(f"Session to host {self.host} spawned")

        try:
            self._pipes_isauthenticated(self.session)
        except TimeoutError:
            # If auth fails, kill the popen session, also need to manually close the stderr pipe
            # for some reason... unclear why, but w/out this it will hang open
            if self.session.stderr is not None:
                stderr_fd = self.session.stderr.fileno()
                os.close(stderr_fd)
            self.session.kill()
            return False

        LOG.debug(f"Authenticated to host {self.host} with public key")

        # set stdin/stdout to the new master pty fds
        self._stdin_fd = stdin_master_pty
        self._stdout_fd = stdout_master_pty

        self.open_cmd = open_cmd
        self.session_lock.release()
        return True

    @operation_timeout("_timeout_ops",
                       "Timed out determining if session is authenticated")
    def _pipes_isauthenticated(self, pipes_session: "PopenBytes") -> bool:
        """
        Private method to check initial authentication when using subprocess.Popen

        Since we always run ssh with `-v` we can simply check the stderr (where verbose output goes)
        to see if `Authenticated to [our host]` is in the output.

        Args:
            pipes_session: Popen pipes session object

        Returns:
            bool: True/False session was authenticated

        Raises:
            ScrapliTimeout: if `Operation timed out` in stderr output

        """
        if pipes_session.stderr is None:
            raise ScrapliTimeout(
                f"Could not read stderr while connecting to host {self.host}")

        output = b""
        while True:
            output += pipes_session.stderr.read(65535)
            if f"Authenticated to {self.host}".encode() in output:
                self._isauthenticated = True
                return True
            if "Operation timed out".encode() in output:
                raise ScrapliTimeout(
                    f"Timed opening connection to host {self.host}")

    def _open_pty(self) -> bool:
        """
        Private method to open session with PtyProcess

        Args:
            N/A

        Returns:
            bool: True/False session was opened and authenticated

        Raises:
            N/A

        """
        self.session = PtyProcess.spawn(self.open_cmd)
        LOG.debug(f"Session to host {self.host} spawned")
        self.session_lock.release()
        self._pty_authenticate(self.session)
        if not self._pty_isauthenticated(self.session):
            return False
        LOG.debug(f"Authenticated to host {self.host} with password")
        return True

    @operation_timeout("_timeout_ops",
                       "Timed out looking for SSH login password prompt")
    def _pty_authenticate(self, pty_session: PtyProcess) -> None:
        """
        Private method to check initial authentication when using pty_session

        Args:
            pty_session: PtyProcess session object

        Returns:
            N/A  # noqa: DAR202

        Raises:
            ScrapliAuthenticationFailed: if we receive an EOFError -- this usually indicates that
                host key checking is enabled and failed.

        """
        self.session_lock.acquire()
        output = b""
        while True:
            try:
                output += pty_session.read()
            except EOFError:
                msg = f"Failed to open connection to host {self.host}"
                if b"Host key verification failed" in output:
                    msg = f"Host key verification failed for host {self.host}"
                elif b"Operation timed out" in output:
                    msg = f"Timed out connecting to host {self.host}"
                raise ScrapliAuthenticationFailed(msg)
            if self._comms_ansi:
                output = strip_ansi(output)
            if b"password" in output.lower():
                LOG.debug("Found password prompt, sending password")
                pty_session.write(self.auth_password.encode())
                pty_session.write(self._comms_return_char.encode())
                self.session_lock.release()
                break

    @operation_timeout("_timeout_ops",
                       "Timed out determining if session is authenticated")
    def _pty_isauthenticated(self, pty_session: PtyProcess) -> bool:
        """
        Check if session is authenticated

        This is very naive -- it only knows if the sub process is alive and has not received an EOF.
        Beyond that we lock the session and send the return character and re-read the channel.

        Args:
            pty_session: PtyProcess session object

        Returns:
            bool: True if authenticated, else False

        Raises:
            N/A

        """
        LOG.debug(
            "Attempting to determine if PTY authentication was successful")
        if pty_session.isalive() and not pty_session.eof():
            prompt_pattern = get_prompt_pattern("", self._comms_prompt_pattern)
            self.session_lock.acquire()
            pty_session.write(self._comms_return_char.encode())
            fd_ready, _, _ = select([pty_session.fd], [], [], 0)
            if pty_session.fd in fd_ready:
                output = b""
                while True:
                    output += pty_session.read()
                    # we do not need to deal w/ line replacement for the actual output, only for
                    # parsing if a prompt-like thing is at the end of the output
                    output = re.sub(b"\r", b"", output)
                    # always check to see if we should strip ansi here; if we don't handle this we
                    # may raise auth failures for the wrong reason which would be confusing for
                    # users
                    if b"\x1B" in output:
                        output = strip_ansi(output)
                    channel_match = re.search(prompt_pattern, output)
                    if channel_match:
                        self.session_lock.release()
                        self._isauthenticated = True
                        return True
                    if b"password" in output.lower():
                        # if we see "password" we know auth failed (hopefully in all scenarios!)
                        return False
                    if output:
                        LOG.debug(
                            f"Cannot determine if authenticated, \n\tRead: {repr(output)}"
                        )
        self.session_lock.release()
        return False

    def close(self) -> None:
        """
        Close session and socket

        Args:
            N/A

        Returns:
            N/A  # noqa: DAR202

        Raises:
            N/A

        """
        self.session_lock.acquire()
        if isinstance(self.session, Popen):
            self.session.kill()
        elif isinstance(self.session, PtyProcess):
            self.session.kill(1)
        LOG.debug(f"Channel to host {self.host} closed")
        self.session_lock.release()

    def isalive(self) -> bool:
        """
        Check if session is alive and session is authenticated

        Args:
            N/A

        Returns:
            bool: True if session is alive and session authenticated, else False

        Raises:
            N/A

        """
        if isinstance(self.session, Popen):
            if self.session.poll() is None and self._isauthenticated:
                return True
        elif isinstance(self.session, PtyProcess):
            if self.session.isalive(
            ) and self._isauthenticated and not self.session.eof():
                return True
        return False

    @operation_timeout("timeout_transport",
                       "Transport timeout during read operation.")
    def read(self) -> bytes:
        """
        Read data from the channel

        Args:
            N/A

        Returns:
            bytes: bytes output as read from channel

        Raises:
            N/A

        """
        read_bytes = 65535
        if isinstance(self.session, Popen):
            return os.read(self._stdout_fd, read_bytes)
        if isinstance(self.session, PtyProcess):
            return self.session.read(read_bytes)
        return b""

    @operation_timeout("timeout_transport",
                       "Transport timeout during write operation.")
    def write(self, channel_input: str) -> None:
        """
        Write data to the channel

        Args:
            channel_input: string to send to channel

        Returns:
            N/A  # noqa: DAR202

        Raises:
            N/A

        """
        if isinstance(self.session, Popen):
            os.write(self._stdin_fd, channel_input.encode())
        elif isinstance(self.session, PtyProcess):
            self.session.write(channel_input.encode())

    def set_timeout(self, timeout: Optional[int] = None) -> None:
        """
        Set session timeout

        Note that this modifies the objects `timeout_transport` value directly as this value is
        what controls the timeout decorator for read/write methods. This is slightly different
        behavior from ssh2/paramiko/telnet in that those transports modify the session value and
        leave the objects `timeout_transport` alone.

        Args:
            timeout: timeout in seconds

        Returns:
            N/A  # noqa: DAR202

        Raises:
            N/A

        """
        if isinstance(timeout, int):
            set_timeout = timeout
        else:
            set_timeout = self.timeout_transport
        self.timeout_transport = set_timeout

    def _keepalive_standard(self) -> None:
        """
        Send "out of band" (protocol level) keepalives to devices.

        Args:
            N/A

        Returns:
            N/A  # noqa: DAR202

        Raises:
            NotImplementedError: always, because this is not implemented for telnet

        """
        raise NotImplementedError(
            "'standard' keepalive mechanism not yet implemented for system.")
Beispiel #47
0
class Local(Runner):
    """
    Execute a command on the local system in a subprocess.

    .. note::
        When Invoke itself is executed without a controlling terminal (e.g.
        when ``sys.stdin`` lacks a useful ``fileno``), it's not possible to
        present a handle on our PTY to local subprocesses. In such situations,
        `Local` will fallback to behaving as if ``pty=False`` (on the theory
        that degraded execution is better than none at all) as well as printing
        a warning to stderr.

        To disable this behavior, say ``fallback=False``.
    """
    def __init__(self, context):
        super(Local, self).__init__(context)
        # Bookkeeping var for pty use case
        self.status = None

    def should_use_pty(self, pty=False, fallback=True):
        use_pty = False
        if pty:
            use_pty = True
            # TODO: pass in & test in_stream, not sys.stdin
            if not has_fileno(sys.stdin) and fallback:
                if not self.warned_about_pty_fallback:
                    sys.stderr.write(
                        "WARNING: stdin has no fileno; falling back to non-pty execution!\n"
                    )  # noqa
                    self.warned_about_pty_fallback = True
                use_pty = False
        return use_pty

    def read_proc_stdout(self, num_bytes):
        # Obtain useful read-some-bytes function
        if self.using_pty:
            # Need to handle spurious OSErrors on some Linux platforms.
            try:
                data = os.read(self.parent_fd, num_bytes)
            except OSError as e:
                # Only eat this specific OSError so we don't hide others
                if "Input/output error" not in str(e):
                    raise
                # The bad OSErrors happen after all expected output has
                # appeared, so we return a falsey value, which triggers the
                # "end of output" logic in code using reader functions.
                data = None
        else:
            data = os.read(self.process.stdout.fileno(), num_bytes)
        return data

    def read_proc_stderr(self, num_bytes):
        # NOTE: when using a pty, this will never be called.
        # TODO: do we ever get those OSErrors on stderr? Feels like we could?
        return os.read(self.process.stderr.fileno(), num_bytes)

    def _write_proc_stdin(self, data):
        # NOTE: parent_fd from os.fork() is a read/write pipe attached to our
        # forked process' stdout/stdin, respectively.
        fd = self.parent_fd if self.using_pty else self.process.stdin.fileno()
        # Try to write, ignoring broken pipes if encountered (implies child
        # process exited before the process piping stdin to us finished;
        # there's nothing we can do about that!)
        try:
            return os.write(fd, data)
        except OSError as e:
            if 'Broken pipe' not in str(e):
                raise

    def start(self, command, shell, env):
        if self.using_pty:
            if pty is None:  # Encountered ImportError
                sys.exit(
                    "You indicated pty=True, but your platform doesn't support the 'pty' module!"
                )  # noqa
            cols, rows = pty_size()
            self.pid, self.parent_fd = pty.fork()
            # If we're the child process, load up the actual command in a
            # shell, just as subprocess does; this replaces our process - whose
            # pipes are all hooked up to the PTY - with the "real" one.
            if self.pid == 0:
                # TODO: both pty.spawn() and pexpect.spawn() do a lot of
                # setup/teardown involving tty.setraw, getrlimit, signal.
                # Ostensibly we'll want some of that eventually, but if
                # possible write tests - integration-level if necessary -
                # before adding it!
                #
                # Set pty window size based on what our own controlling
                # terminal's window size appears to be.
                # TODO: make subroutine?
                winsize = struct.pack('HHHH', rows, cols, 0, 0)
                fcntl.ioctl(sys.stdout.fileno(), termios.TIOCSWINSZ, winsize)
                # Use execve for bare-minimum "exec w/ variable # args + env"
                # behavior. No need for the 'p' (use PATH to find executable)
                # for now.
                # TODO: see if subprocess is using equivalent of execvp...
                os.execve(shell, [shell, '-c', command], env)
        else:
            self.process = Popen(
                command,
                shell=True,
                executable=shell,
                env=env,
                stdout=PIPE,
                stderr=PIPE,
                stdin=PIPE,
            )

    @property
    def process_is_finished(self):
        if self.using_pty:
            # NOTE:
            # https://github.com/pexpect/ptyprocess/blob/4058faa05e2940662ab6da1330aa0586c6f9cd9c/ptyprocess/ptyprocess.py#L680-L687
            # implies that Linux "requires" use of the blocking, non-WNOHANG
            # version of this call. Our testing doesn't verify this, however,
            # so...
            # NOTE: It does appear to be totally blocking on Windows, so our
            # issue #351 may be totally unsolvable there. Unclear.
            pid_val, self.status = os.waitpid(self.pid, os.WNOHANG)
            return pid_val != 0
        else:
            return self.process.poll() is not None

    def returncode(self):
        if self.using_pty:
            # No subprocess.returncode available; use WIFEXITED/WIFSIGNALED to
            # determine whch of WEXITSTATUS / WTERMSIG to use.
            # TODO: is it safe to just say "call all WEXITSTATUS/WTERMSIG and
            # return whichever one of them is nondefault"? Probably not?
            # NOTE: doing this in an arbitrary order should be safe since only
            # one of the WIF* methods ought to ever return True.
            code = None
            if os.WIFEXITED(self.status):
                code = os.WEXITSTATUS(self.status)
            elif os.WIFSIGNALED(self.status):
                code = os.WTERMSIG(self.status)
                # Match subprocess.returncode by turning signals into negative
                # 'exit code' integers.
                code = -1 * code
            return code
            # TODO: do we care about WIFSTOPPED? Maybe someday?
        else:
            return self.process.returncode

    def stop(self):
        # No explicit close-out required (so far).
        pass
Beispiel #48
0
class Wasm3():
    def __init__(self, exe):
        self.exe = exe
        self.p = None
        self.loaded = None
        self.timeout = args.timeout
        self.autorestart = True

        self.run()

    def run(self):
        if self.p:
            self.terminate()

        cmd = shlex.split(self.exe)

        #print(f"wasm3: Starting {' '.join(cmd)}")

        self.q = Queue()
        self.p = Popen(cmd, bufsize=0, stdin=PIPE, stdout=PIPE, stderr=STDOUT)

        def _read_output(out, queue):
            for data in iter(lambda: out.read(1024), b''):
                queue.put(data)
            queue.put(None)

        self.t = Thread(target=_read_output, args=(self.p.stdout, self.q))
        self.t.daemon = True
        self.t.start()

        try:
            self._read_until("wasm3> ")
        except Exception as e:
            print(f"wasm3: Could not start: {e}")

    def restart(self):
        print(f"wasm3: Restarting")
        for i in range(10):
            try:
                self.run()
                try:
                    if self.loaded:
                        self.load(self.loaded)
                except Exception as e:
                    pass
                break
            except Exception as e:
                print(f"wasm3: {e} => retry")
                time.sleep(0.1)

    def init(self):
        return self._run_cmd(f":init\n")

    def version(self):
        return self._run_cmd(f":version\n")

    def load(self, fn):
        self.loaded = None
        with open(fn, "rb") as f:
            wasm = f.read()
        res = self._run_cmd(f":load-hex {len(wasm)}\n{wasm.hex()}\n")
        self.loaded = fn
        return res

    def invoke(self, cmd):
        return self._run_cmd(":invoke " + " ".join(map(str, cmd)) + "\n")

    def _run_cmd(self, cmd):
        if self.autorestart and not self._is_running():
            self.restart()
        self._flush_input()

        #print(f"wasm3: {cmd.strip()}")
        self._write(cmd)
        return self._read_until("wasm3> ")

    def _read_until(self, token):
        buff = ""
        tout = time.time() + self.timeout
        error = None

        while time.time() < tout:
            try:
                data = self.q.get(timeout=0.1)
                if data == None:
                    error = "Crashed"
                    break
                buff = buff + data.decode("utf-8")
                idx = buff.rfind(token)
                if idx >= 0:
                    return buff[0:idx].strip()
            except Empty:
                pass
        else:
            error = "Timeout"

        self.terminate()
        raise Exception(error)

    def _write(self, data):
        self.p.stdin.write(data.encode("utf-8"))
        self.p.stdin.flush()

    def _is_running(self):
        return self.p and (self.p.poll() == None)

    def _flush_input(self):
        while not self.q.empty():
            self.q.get()

    def terminate(self):
        self.p.stdin.close()
        self.p.terminate()
        self.p.wait(timeout=1.0)
        self.p = None
Beispiel #49
0
    def runAsWithSudo(self, user="", password="", silent=True):
        """
        Use pty method to run "su" to run a command as another user...

        Required parameters: user, password, command

        @author: Roy Nielsen
        """
        self.logger.log(lp.DEBUG, "Starting runAsWithSudo: ")
        self.logger.log(lp.DEBUG, "\tuser: \"" + str(user) + "\"")
        self.logger.log(lp.DEBUG, "\tcmd : \"" + str(self.command) + "\"")
        if re.match("^\s+$", user) or re.match("^\s+$", password) or \
           not user or not password or \
           not self.command:
            self.logger.log(lp.WARNING, "Cannot pass in empty parameters...")
            self.logger.log(lp.WARNING, "user = \"" + str(user) + "\"")
            self.logger.log(lp.WARNING, "check password...")
            if not silent:
                self.logger.log(lp.WARNING, "command: " + str(self.command))
            return 255
        else:
            output = ""

            internal_command = [
                "/usr/bin/su",
                str("-m"),
                str(user).strip(),
                str("-c")
            ]

            if isinstance(self.command, list):
                cmd = []
                for i in range(len(self.command)):
                    try:
                        cmd.append(str(self.command[i]('utf-8')))
                    except UnicodeDecodeError:
                        cmd.append(str(self.command[i]))

                internal_command.append(
                    str("/usr/bin/sudo -S -s '" + " ".join(cmd) + "'"))
            elif isinstance(self.command, str):
                try:
                    internal_command.append(
                        str("/usr/bin/sudo -E -S -s " + "'" +
                            str(self.command('utf-8')) + "'"))
                except UnicodeDecodeError:
                    internal_command.append(
                        str("/usr/bin/sudo -E -S -s " + "'" +
                            str(self.command) + "'"))

            try:
                (master, slave) = pty.openpty()
            except SubprocessError as err:
                self.logger.log(lp.WARNING,
                                "Error trying to open pty: " + str(err))
                self.logger.log(lp.WARNING, traceback.format_exc())
                self.logger.log(lp.WARNING, str(err))
                raise err
            else:
                try:
                    proc = Popen(internal_command,
                                 stdin=slave,
                                 stdout=slave,
                                 stderr=slave,
                                 close_fds=True,
                                 text=self.text)
                except SubprocessError as err:
                    self.logger.log(
                        lp.WARNING,
                        "Error opening process to pty: " + str(err))
                    self.logger.log(lp.WARNING, traceback.format_exc())
                    self.logger.log(lp.WARNING, str(err))
                    raise err
                else:
                    #####
                    # Catch the su password prompt
                    # prompt = os.read(master, 512)
                    self.waitnoecho(master, 3)
                    self.prompt = os.read(master, 512)

                    #####
                    # pass in the password
                    os.write(master, password.strip() + "\n")

                    #####
                    # catch the password
                    self.prompt = os.read(master, 512)

                    #####
                    # Wait for the next password prompt
                    self.waitnoecho(master, 3)

                    #####
                    # catch the password prompt
                    self.prompt = os.read(master, 512)

                    #####
                    # Enter the sudo password
                    os.write(master, password + "\n")

                    #####
                    # Catch the password
                    os.read(master, 512)

                    # output = tmp + output
                    while True:
                        #####
                        # timeout of 0 means "poll"
                        ready, _, _ = select.select([master], [], [], 0)
                        if ready:
                            line = os.read(master, 512)
                            #####
                            # Warning, uncomment at your own risk - several
                            # programs print empty lines that will cause this
                            # to break and the output will be all goofed up.
                            # if not line :
                            #    break
                            # print output.rstrip()
                            output = output + line
                        elif proc.poll() is not None:
                            break
                        # print output.strip()
                    os.close(master)
                    os.close(slave)
                    self.libc.sync()
                    proc.wait()
                    self.libc.sync()
                    self.stdout = proc.stdout
                    self.stderr = proc.stderr
                    self.retcode = proc.returncode
                    # print output.strip()
            if not silent:
                self.logger.log(
                    lp.DEBUG, "\n\nLeaving runAs with Sudo: \"" +
                    str(self.stdout) + "\"\n\n")
        self.command = None
        return self.stdout, self.stderr, self.retcode
class Capture(object):
    default_port = 9900  # Default port to run CaptureJS at
    check_interval = 0.05  # Frequency (seconds) to check if self.started
    # Set engine configurations for PhantomJS and Puppeteer
    engines = AttrDict(
        phantomjs=AttrDict(cmd='phantomjs --ssl-protocol=any',
                           script='capture.js',
                           first_line=b'PhantomJS.*capture\\.js',
                           name='Capture',
                           version='1.0'),
        chrome=AttrDict(cmd='node',
                        script='chromecapture.js',
                        first_line=b'node\\.js.*chromecapture\\.js',
                        name='ChromeCapture',
                        version='1.1'),
    )
    '''
    Create a proxy for capture.js. Typical usage::

        capture = Capture()
        with open('screenshot.png', 'wb') as handle:
            handle.write(capture.png('https://gramener.com/'))
        with open('screenshot.pdf', 'wb') as handle:
            handle.write(capture.pdf('https://gramener.com/'))

    The constructor accepts these optional parameters:

    :arg int port: port where capture.js is running. Default: 9900
    :arg string url: URL:port where PhantomJS is running with capture.js.
        Default: ``http://localhost:<port>/``
    :arg string cmd: Command to run PhantomJS with capture.js at the specified
        port. Default: ``phantomjs $GRAMEXPATH/apps/capture/capture.js --port=<port>``
    :arg int timeout: Seconds to wait for PhantomJS to timeout. Default: 10

    The constructor runs :meth:`Capture.start` in a new thread, which checks if
    capture.js is running at ``url``. If not, it runs ``cmd`` and checks again.
    Until capture.js is detected, all capture methods will fail.
    '''
    def __init__(self,
                 port=None,
                 url=None,
                 engine=None,
                 cmd=None,
                 timeout=DEFAULT_TIMEOUT):
        # Set default values for port, url and cmd
        self.engine = self.engines['phantomjs' if engine is None else engine]
        port = self.default_port if port is None else port
        if url is None:
            url = 'http://*****:*****@ {self.url}')
            thread.daemon = True
            thread.start()

    def _start(self):
        '''
        Check if capture is already running at ``url``. If not, start ``cmd``
        and check again. Print logs from ``cmd``.
        '''
        self.started = False
        script = self.engine.script
        try:
            # Check if capture.js is at the url specified
            app_log.info('Pinging %s at %s', script, self.url)
            r = requests.get(self.url, timeout=self.timeout)
            self._validate_server(r)
            self.started = True
        except requests.ReadTimeout:
            # If capture.js doesn't respond immediately, we haven't started
            app_log.error('url: %s timed out', self.url)
        except requests.ConnectionError:
            # Try starting the process again
            app_log.info('Starting %s via %s', script, self.cmd)
            self.close()
            # self.cmd is taken from the YAML configuration. Safe to run
            self.proc = Popen(shlex.split(self.cmd),
                              stdout=PIPE,
                              stderr=STDOUT)  # nosec
            self.proc.poll()
            atexit.register(self.close)
            # TODO: what if readline() does not return quickly?
            line = self.proc.stdout.readline().strip()
            if not self.first_line_re.search(line):
                return app_log.error('cmd: %s invalid. Returned "%s"',
                                     self.cmd, line)
            app_log.info('Pinging %s at %s', script, self.url)
            try:
                r = requests.get(self.url, timeout=self.timeout)
                self._validate_server(r)
                pid = self.proc.pid
                app_log.info(line.decode('utf-8') + ' live (pid=%s)', pid)
                self.started = True
                # Keep logging capture.js output until proc is killed by another thread
                while hasattr(self, 'proc'):
                    line = self.proc.stdout.readline().strip()
                    if len(line) == 0:
                        app_log.info('%s terminated: pid=%d', script, pid)
                        self.started = False
                        break
                    # Capture won't print anything, unless there's a problem, or if debug is on.
                    # So log it at warning level not info.
                    app_log.warning(line.decode('utf-8'))
            except Exception:
                app_log.exception('Ran %s. But %s not at %s', self.cmd, script,
                                  self.url)
        except Exception:
            app_log.exception('Cannot start Capture')

    def close(self):
        '''Stop capture.js if it has been started by this object'''
        if hasattr(self, 'proc'):
            try:
                process = psutil.Process(self.proc.pid)
                for proc in process.children(recursive=True):
                    proc.kill()
                process.kill()
            except psutil.NoSuchProcess:
                app_log.info('%s PID %d already killed', self.engine.script,
                             self.proc.pid)
                pass
            delattr(self, 'proc')

    def _validate_server(self, response):
        # Make sure that the response we got is from the right version of capture.js
        server = response.headers.get('Server', '')
        parts = server.split('/', 2)
        script = self.engine.script
        if not len(parts) == 2 or parts[0] != self.engine.name or parts[
                1] < self.engine.version:
            raise RuntimeError('Server: %s at %s is not %s' %
                               (server, self.url, script))

    @tornado.gen.coroutine
    def capture_async(self, headers=None, **kwargs):
        '''
        Returns a screenshot of the URL. Runs asynchronously in Gramex. Arguments
        are same as :py:func:`capture`
        '''
        # If ?start is provided, start server and wait until timeout
        if 'start' in kwargs:
            self.start()
            end_time = time.time() + self.timeout
            while not self.started and time.time() < end_time:
                yield tornado.gen.sleep(self.check_interval)
        if not self.started:
            raise RuntimeError('%s not started. See logs' % self.engine.script)
        r = yield self.browser.fetch(self.url,
                                     method='POST',
                                     body=urlencode(kwargs, doseq=True),
                                     raise_error=False,
                                     connect_timeout=self.timeout,
                                     request_timeout=self.timeout,
                                     headers=headers)
        if r.code == OK:
            self._validate_server(r)
        raise tornado.gen.Return(r)

    def capture(self, url, **kwargs):
        '''
        Return a screenshot of the URL.

        :arg str url: URL to take a screenshot of
        :arg str ext: format of output. Can be pdf, png, gif or jpg
        :arg str selector: Restrict screenshot to (optional) CSS selector in URL
        :arg int delay: milliseconds (or expression) to wait for before taking a screenshot
        :arg str format: A3, A4, A5, Legal, Letter or Tabloid. Defaults to A4. For PDF
        :arg str layout: A3, A4, A5, Legal, 16x9, 16x10, 4x3. Defaults to 4x3. For PPTX
        :arg str orientation: portrait or landscape. Defaults to portrait. For PDF
        :arg str header: header for the page. For PDF
        :arg str footer: footer for the page. For PDF
        :arg int width: screen width. Default: 1200. For PNG/GIF/JPG
        :arg int height: screen height. Default: 768. For PNG/GIF/JPG
        :arg float scale: zooms the screen by a factor. For PNG/GIF/JPG
        :arg int dpi: dots (pixels) per inch. For PPTX
        :arg str title: slide title. For PPTX
        :arg int debug: sets log level for HTTP requests (2) and responses (1)
        :return: a bytestring with the binary contents of the screenshot
        :rtype: bytes
        :raises RuntimeError: if capture.js is not running or fails
        '''
        # Ensure that we're connecting to the right version of capture.js
        if not self.started:
            end_time = time.time() + self.timeout
            while not self.started and time.time() < end_time:
                time.sleep(self.check_interval)
            if not self.started:
                raise RuntimeError('%s not started. See logs' %
                                   self.engine.script)
        kwargs['url'] = url
        r = requests.post(self.url, data=kwargs, timeout=self.timeout)
        if r.status_code == OK:
            self._validate_server(r)
            return r.content
        else:
            raise RuntimeError('%s error: %s' %
                               (self.engine.script, r.content))

    def pdf(self, url, **kwargs):
        '''An alias for :meth:`Capture.capture` with ``ext='pdf'``.'''
        kwargs['ext'] = 'pdf'
        return self.capture(url, **kwargs)

    def png(self, url, **kwargs):
        '''An alias for :meth:`Capture.capture` with ``ext='png'``.'''
        kwargs['ext'] = 'png'
        return self.capture(url, **kwargs)

    def pptx(self, url, **kwargs):
        '''An alias for :meth:`Capture.capture` with ``ext='pptx'``.'''
        kwargs['ext'] = 'pptx'
        return self.capture(url, **kwargs)

    def jpg(self, url, **kwargs):
        '''An alias for :meth:`Capture.capture` with ``ext='jpg'``.'''
        kwargs['ext'] = 'jpg'
        return self.capture(url, **kwargs)

    def gif(self, url, **kwargs):
        '''An alias for :meth:`Capture.capture` with ``ext='gif'``.'''
        kwargs['ext'] = 'gif'
        return self.capture(url, **kwargs)
Beispiel #51
0
    def waitNpassThruStdout(self, chk_string=None, respawn=False, silent=True):
        """
        Use the subprocess module to execute a command, returning
        the output of the command

        Author: Roy Nielsen
        """
        self.stdout = ''
        self.stderr = ''
        self.retcode = 999
        if self.command:
            try:
                proc = Popen(self.command,
                             stdout=PIPE,
                             stderr=PIPE,
                             shell=self.myshell,
                             env=self.environ,
                             close_fds=self.cfds,
                             text=self.text)
                if proc:
                    while True:
                        #####
                        # process stdout
                        myout = proc.stdout.readline()
                        if myout == '' and proc.poll() is not None:
                            break
                        tmpline = str(myout)
                        tmpline = str(tmpline).strip()
                        self.stdout += str(tmpline) + "\n"

                        # if tmpline and not silent:
                        #     self.logger.log(lp.DEBUG, str(tmpline))

                        if isinstance(chk_string, str):
                            if not chk_string:
                                continue
                            else:
                                if re.search(chk_string, tmpline):
                                    try:
                                        proc.stdout.close()
                                    except SubprocessError as err:
                                        self.logger.log(
                                            lp.INFO, traceback.format_exc())
                                    try:
                                        proc.stderr.close()
                                    except SubprocessError as err:
                                        self.logger.log(
                                            lp.INFO, traceback.format_exc())

                                    if respawn:
                                        pass
                                    else:
                                        self.logger.log(
                                            lp.INFO, "chk_string found" +
                                            "... exiting process.")
                                    break

                        if isinstance(chk_string, list):
                            if not chk_string:
                                continue
                            else:
                                found = False
                                for mystring in chk_string:
                                    if chk_string(mystring, tmpline):
                                        try:
                                            proc.stdout.close()
                                        except SubprocessError as err:
                                            self.logger.log(
                                                lp.INFO,
                                                traceback.format_exc())

                                        try:
                                            proc.stderr.close()
                                        except SubprocessError as err:
                                            self.logger.log(
                                                lp.INFO,
                                                traceback.format_exc())

                                        proc.stderr.close()
                                        if respawn:
                                            pass
                                        else:
                                            self.logger.log(
                                                lp.INFO,
                                                "chk_string " + "found... " +
                                                "exiting process.")
                                            found = True
                                            break
                                if found:
                                    break

                    while True:
                        myerr = proc.stderr.readline()
                        if myerr == '' and proc.poll() is not None:
                            break
                        tmpline = str(myerr).strip()
                        self.stderr += tmpline + "\n"

                        if tmpline and not silent:
                            self.logger.log(lp.DEBUG, str(tmpline))

                        if isinstance(chk_string, str):
                            if not chk_string:
                                continue
                            else:
                                if re.search(chk_string, tmpline):
                                    try:
                                        proc.stdout.close()
                                    except SubprocessError as err:
                                        self.logger.log(
                                            lp.INFO, traceback.format_exc())

                                    try:
                                        proc.stderr.close()
                                    except SubprocessError as err:
                                        self.logger.log(
                                            lp.INFO, traceback.format_exc())

                                    if respawn:
                                        pass
                                    else:
                                        self.logger.log(
                                            lp.INFO, "chk_string found" +
                                            "... exiting process.")
                                    break

                        if isinstance(chk_string, list):
                            if not chk_string:
                                continue
                            else:
                                found = False
                                for mystring in chk_string:
                                    if chk_string(mystring, tmpline):
                                        try:
                                            proc.stdout.close()
                                        except SubprocessError as err:
                                            self.logger.log(
                                                lp.INFO,
                                                traceback.format_exc())

                                        try:
                                            proc.stderr.close()
                                        except SubprocessError as err:
                                            self.logger.log(
                                                lp.INFO,
                                                traceback.format_exc())

                                        if respawn:
                                            pass
                                        else:
                                            self.logger.log(
                                                lp.INFO,
                                                "chk_string " + "found... " +
                                                "exiting process.")
                                            found = True
                                            break
                                if found:
                                    break

                proc.wait()
                try:
                    proc.stdout.close()
                except SubprocessError as err:
                    self.logger.log(lp.INFO, traceback.format_exc())
                try:
                    proc.stderr.close()
                except SubprocessError as err:
                    self.logger.log(lp.INFO, traceback.format_exc())

                self.retcode = proc.returncode
                self.libc.sync()

            except SubprocessError as err:
                if not silent:
                    self.logger.log(lp.WARNING,
                                    "command: " + str(self.printcmd))
                self.logger.log(lp.WARNING, "stderr: " + str(self.stderr))
                self.logger.log(lp.WARNING, traceback.format_exc())
                self.logger.log(lp.WARNING, str(err))
                raise err
            else:
                if not silent:
                    self.logger.log(lp.DEBUG, "Done with: " + self.printcmd)
            finally:
                self.retcode = proc.returncode
                if not silent:
                    self.logger.log(lp.DEBUG,
                                    "Done with command: " + self.printcmd)
                self.logger.log(lp.DEBUG, "stdout: " + str(self.stdout))
                self.logger.log(lp.DEBUG, "stderr: " + str(self.stderr))
                self.logger.log(lp.DEBUG, "retcode: " + str(self.retcode))
        else:
            self.logger.log(lp.WARNING,
                            "Cannot run a command that is empty...")
            self.stdout = None
            self.stderr = None
            self.retcode = None

        self.command = None
        return self.stdout, self.stderr, self.retcode
Beispiel #52
0
def launch_gateway():
    if "PYSPARK_GATEWAY_PORT" in os.environ:
        gateway_port = int(os.environ["PYSPARK_GATEWAY_PORT"])
    else:
        SPARK_HOME = os.environ["SPARK_HOME"]
        # Launch the Py4j gateway using Spark's run command so that we pick up the
        # proper classpath and settings from spark-env.sh
        on_windows = platform.system() == "Windows"
        script = "./bin/spark-submit.cmd" if on_windows else "./bin/spark-submit"
        submit_args = os.environ.get("PYSPARK_SUBMIT_ARGS", "pyspark-shell")
        if os.environ.get("SPARK_TESTING"):
            submit_args = "--conf spark.ui.enabled=false " + submit_args
        command = [os.path.join(SPARK_HOME, script)] + shlex.split(submit_args)

        # Start a socket that will be used by PythonGatewayServer to communicate its port to us
        callback_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        callback_socket.bind(('127.0.0.1', 0))
        callback_socket.listen(1)
        callback_host, callback_port = callback_socket.getsockname()
        env = dict(os.environ)
        env['_PYSPARK_DRIVER_CALLBACK_HOST'] = callback_host
        env['_PYSPARK_DRIVER_CALLBACK_PORT'] = str(callback_port)

        # Launch the Java gateway.
        # We open a pipe to stdin so that the Java gateway can die when the pipe is broken
        if not on_windows:
            # Don't send ctrl-c / SIGINT to the Java gateway:
            def preexec_func():
                signal.signal(signal.SIGINT, signal.SIG_IGN)

            proc = Popen(command, stdin=PIPE, preexec_fn=preexec_func, env=env)
        else:
            # preexec_fn not supported on Windows
            proc = Popen(command, stdin=PIPE, env=env)

        gateway_port = None
        # We use select() here in order to avoid blocking indefinitely if the subprocess dies
        # before connecting
        while gateway_port is None and proc.poll() is None:
            timeout = 1  # (seconds)
            readable, _, _ = select.select([callback_socket], [], [], timeout)
            if callback_socket in readable:
                gateway_connection = callback_socket.accept()[0]
                # Determine which ephemeral port the server started on:
                gateway_port = read_int(gateway_connection.makefile(mode="rb"))
                gateway_connection.close()
                callback_socket.close()
        if gateway_port is None:
            raise Exception(
                "Java gateway process exited before sending the driver its port number"
            )

        # In Windows, ensure the Java child processes do not linger after Python has exited.
        # In UNIX-based systems, the child process can kill itself on broken pipe (i.e. when
        # the parent process' stdin sends an EOF). In Windows, however, this is not possible
        # because java.lang.Process reads directly from the parent process' stdin, contending
        # with any opportunity to read an EOF from the parent. Note that this is only best
        # effort and will not take effect if the python process is violently terminated.
        if on_windows:
            # In Windows, the child process here is "spark-submit.cmd", not the JVM itself
            # (because the UNIX "exec" command is not available). This means we cannot simply
            # call proc.kill(), which kills only the "spark-submit.cmd" process but not the
            # JVMs. Instead, we use "taskkill" with the tree-kill option "/t" to terminate all
            # child processes in the tree (http://technet.microsoft.com/en-us/library/bb491009.aspx)
            def killChild():
                Popen([
                    "cmd", "/c", "taskkill", "/f", "/t", "/pid",
                    str(proc.pid)
                ])

            atexit.register(killChild)

    # Connect to the gateway
    gateway = JavaGateway(GatewayClient(port=gateway_port), auto_convert=True)

    # Import the classes used by PySpark
    java_import(gateway.jvm, "org.apache.spark.SparkConf")
    java_import(gateway.jvm, "org.apache.spark.api.java.*")
    java_import(gateway.jvm, "org.apache.spark.api.python.*")
    java_import(gateway.jvm, "org.apache.spark.mllib.api.python.*")
    # TODO(davies): move into sql
    java_import(gateway.jvm, "org.apache.spark.sql.*")
    java_import(gateway.jvm, "org.apache.spark.sql.hive.*")
    java_import(gateway.jvm, "scala.Tuple2")

    return gateway
Beispiel #53
0
class Process(object):
    ''' Represents a running/ran process '''

    @staticmethod
    def devnull():
        ''' Helper method for opening devnull '''
        return open('/dev/null', 'w')

    @staticmethod
    def call(command, cwd=None, shell=False):
        '''
            Calls a command (either string or list of args).
            Returns tuple:
                (stdout, stderr)
        '''
        if type(command) is not str or ' ' in command or shell:
            shell = True
            if Configuration.verbose > 1:
                Color.pe("\n {C}[?] {W} Executing (Shell): {B}%s{W}" % command)
        else:
            shell = False
            if Configuration.verbose > 1:
                Color.pe("\n {C}[?]{W} Executing: {B}%s{W}" % command)

        pid = Popen(command, cwd=cwd, stdout=PIPE, stderr=PIPE, shell=shell)
        pid.wait()
        (stdout, stderr) = pid.communicate()

        # Python 3 compatibility
        if type(stdout) is bytes: stdout = stdout.decode('utf-8')
        if type(stderr) is bytes: stderr = stderr.decode('utf-8')


        if Configuration.verbose > 1 and stdout is not None and stdout.strip() != '':
            Color.pe("{P} [stdout] %s{W}" % '\n [stdout] '.join(stdout.strip().split('\n')))
        if Configuration.verbose > 1 and stderr is not None and stderr.strip() != '':
            Color.pe("{P} [stderr] %s{W}" % '\n [stderr] '.join(stderr.strip().split('\n')))

        return (stdout, stderr)

    @staticmethod
    def exists(program):
        ''' Checks if program is installed on this system '''
        p = Process(['which', program])
        stdout = p.stdout().strip()
        stderr = p.stderr().strip()

        if stdout == '' and stderr == '':
            return False

        return True

    def __init__(self, command, devnull=False, stdout=PIPE, stderr=PIPE, cwd=None, bufsize=0):
        ''' Starts executing command '''

        if type(command) is str:
            # Commands have to be a list
            command = command.split(' ')

        self.command = command

        if Configuration.verbose > 1:
            Color.pe("\n {C}[?] {W} Executing: {B}%s{W}" % ' '.join(command))

        self.out = None
        self.err = None
        if devnull:
            sout = Process.devnull()
            serr = Process.devnull()
        else:
            sout = stdout
            serr = stderr

        self.start_time = time.time()

        self.pid = Popen(command, stdout=sout, stderr=serr, cwd=cwd, bufsize=bufsize)

    def __del__(self):
        '''
            Ran when object is GC'd.
            If process is still running at this point, it should die.
        '''
        try:
            if self.pid and self.pid.poll() is None:
                self.interrupt()
        except AttributeError:
            pass

    def stdout(self):
        ''' Waits for process to finish, returns stdout output '''
        self.get_output()
        if Configuration.verbose > 1 and self.out is not None and self.out.strip() != '':
            Color.pe("{P} [stdout] %s{W}" % '\n [stdout] '.join(self.out.strip().split('\n')))
        return self.out

    def stderr(self):
        ''' Waits for process to finish, returns stderr output '''
        self.get_output()
        if Configuration.verbose > 1 and self.err is not None and self.err.strip() != '':
            Color.pe("{P} [stderr] %s{W}" % '\n [stderr] '.join(self.err.strip().split('\n')))
        return self.err

    def stdoutln(self):
        return self.pid.stdout.readline()

    def stderrln(self):
        return self.pid.stderr.readline()

    def get_output(self):
        ''' Waits for process to finish, sets stdout & stderr '''
        if self.pid.poll() is None:
            self.pid.wait()
        if self.out is None:
            (self.out, self.err) = self.pid.communicate()

        if type(self.out) is bytes:
            self.out = self.out.decode('utf-8')

        if type(self.err) is bytes:
            self.err = self.err.decode('utf-8')

        return (self.out, self.err)

    def poll(self):
        ''' Returns exit code if process is dead, otherwise "None" '''
        return self.pid.poll()

    def wait(self):
        self.pid.wait()

    def running_time(self):
        ''' Returns number of seconds since process was started '''
        return int(time.time() - self.start_time)

    def interrupt(self, wait_time=2.0):
        '''
            Send interrupt to current process.
            If process fails to exit within `wait_time` seconds, terminates it.
        '''
        try:
            pid = self.pid.pid
            cmd = self.command
            if type(cmd) is list:
                cmd = ' '.join(cmd)

            if Configuration.verbose > 1:
                Color.pe('\n {C}[?] {W} sending interrupt to PID %d (%s)' % (pid, cmd))

            os.kill(pid, signal.SIGINT)

            start_time = time.time()  # Time since Interrupt was sent
            while self.pid.poll() is None:
                # Process is still running
                time.sleep(0.1)
                if time.time() - start_time > wait_time:
                    # We waited too long for process to die, terminate it.
                    if Configuration.verbose > 1:
                        Color.pe('\n {C}[?] {W} Waited > %0.2f seconds for process to die, killing it' % wait_time)
                    os.kill(pid, signal.SIGTERM)
                    self.pid.terminate()
                    break

        except OSError as e:
            if 'No such process' in e.__str__():
                return
            raise e  # process cannot be killed
Beispiel #54
0
    def runAs(self, user="", password="", silent=True):
        """
        Use pexpect to run "su" to run a command as another user...

        Required parameters: user, password, command

        @author: Roy Nielsen
        """
        self.stdout = ""
        self.stderr = ""
        self.retcode = 999
        # pretcode = 0
        if re.match("^\s*$", user) or \
           re.match("^\s*$", password) or \
           not self.command:
            self.logger.log(lp.WARNING, "Cannot pass in empty parameters...")
            self.logger.log(lp.WARNING, "user = \"" + str(user) + "\"")
            self.logger.log(lp.WARNING, "check password...")
            if not silent:
                self.logger.log(lp.WARNING,
                                "command = \"" + str(self.command) + "\"")
            return 255
        else:
            output = ""
            internal_command = ["/usr/bin/su", "-", str(user.strip()), "-c"]

            if isinstance(self.command, list):
                internal_command.append(" ".join(self.command))
                if not silent:
                    self.logger.log(
                        lp.DEBUG, "Trying to execute: \"" +
                        " ".join(internal_command) + "\"")
            elif isinstance(self.command, str):
                internal_command += self.command
                if not silent:
                    self.logger.log(
                        lp.DEBUG, "Trying to execute: \"" +
                        " ".join(internal_command) + "\"")

            (master, slave) = pty.openpty()

            proc = Popen(internal_command,
                         stdin=slave,
                         stdout=slave,
                         stderr=slave,
                         close_fds=True,
                         text=self.text)

            prompt = os.read(master, 10)

            if re.match("^Password:"******"\n")
                line = os.read(master, 512)
                output = output + line
                while True:
                    #####
                    # timeout of 0 means "poll"
                    ready, _, _ = select.select([master], [], [], 0)
                    if ready:
                        line = os.read(master, 512)
                        #####
                        # Warning, uncomment at your own risk - several
                        # programs print empty lines that will cause this to
                        # break and the output will be all goofed up.
                        # if not line :
                        #    break
                        # print output.rstrip()
                        output = output + line
                    elif proc.poll() is not None:
                        break
                os.close(master)
                os.close(slave)
                self.libc.sync()
                proc.wait()
                self.libc.sync()
                self.stdout = proc.stdout
                self.stderr = proc.stderr
                self.retcode = proc.returncode
            else:
                output = prompt
                while True:
                    #####
                    # timeout of 0 means "poll"
                    ready, _, _ = select.select([master], [], [], 0)
                    if ready:
                        line = os.read(master, 512)
                        #####
                        # Warning, uncomment at your own risk - several
                        # programs print empty lines that will cause this to
                        # break and the output will be all goofed up.
                        # if not line :
                        #    break
                        # print output.rstrip()
                        output = output + line
                    elif proc.poll() is not None:
                        break
                os.close(master)
                os.close(slave)
                proc.wait()
                self.stdout = output
                self.stderr = str(proc.stderr)
                self.retcode = proc.returncode
            output = output.strip()
            if not silent:
                self.logger.log(lp.DEBUG, "retcode: " + str(self.stdout))
                self.logger.log(lp.DEBUG, "retcode: " + str(self.stderr))
                self.logger.log(lp.DEBUG, "retcode: " + str(self.retcode))

        self.command = None
        return self.stdout, self.stderr, self.retcode
Beispiel #55
0
class Tailer(threading.Thread):
    """Logic to tail follow a log file for a GUI viewer.

    logview -- A GUI view to display the content of the log file.
    filename -- The name of the log file.
    cmd_tmpl -- The command template use to follow the log file.
                (global cfg '[hosts][HOST]remote/local tail command template')
    pollable -- If specified, it must implement a pollable.poll() method,
                which is called at regular intervals.
    """

    READ_SIZE = 4096
    TAGS = {
        "CRITICAL": [re.compile(r"\b(?:CRITICAL|ERROR)\b"), "red"],
        "WARNING": [re.compile(r"\bWARNING\b"), "#a83fd3"]
    }

    def __init__(self,
                 logview,
                 filename,
                 cmd_tmpl=None,
                 pollable=None,
                 filters=None):
        super(Tailer, self).__init__()

        self.logview = logview
        self.filename = filename
        self.cmd_tmpl = cmd_tmpl
        self.pollable = pollable
        self.filters = filters

        self.logbuffer = logview.get_buffer()
        self.quit = False
        self.proc = None
        self.freeze = False
        self.has_warned_corrupt = False
        self.tags = {}

    def clear(self):
        """Clear the log buffer."""
        pos_start, pos_end = self.logbuffer.get_bounds()
        self.logbuffer.delete(pos_start, pos_end)

    def run(self):
        """Invoke the tailer."""
        command = []
        if ":" in self.filename:  # remote
            user_at_host, filename = self.filename.split(':')
            if "@" in user_at_host:
                owner, host = user_at_host.split("@", 1)
            else:
                owner, host = (None, user_at_host)
            ssh = str(
                GLOBAL_CFG.get_host_item("remote shell template", host, owner))
            command = shlex.split(ssh) + ["-n", user_at_host]
            cmd_tmpl = str(
                GLOBAL_CFG.get_host_item("remote tail command template", host,
                                         owner))
        else:
            filename = self.filename
            cmd_tmpl = str(
                GLOBAL_CFG.get_host_item("local tail command template"))

        if self.cmd_tmpl:
            cmd_tmpl = self.cmd_tmpl
        command += shlex.split(cmd_tmpl % {"filename": filename})
        try:
            self.proc = Popen(command,
                              stdout=PIPE,
                              stderr=STDOUT,
                              preexec_fn=os.setpgrp)
        except OSError as exc:
            # E.g. ssh command not found
            dialog = warning_dialog(
                "%s: %s" % (exc, " ".join([quote(item) for item in command])))
            gobject.idle_add(dialog.warn)
            return
        poller = select.poll()
        poller.register(self.proc.stdout.fileno())

        buf = ""
        while not self.quit and self.proc.poll() is None:
            try:
                self.pollable.poll()
            except (TypeError, AttributeError):
                pass
            if self.freeze or not poller.poll(100):  # 100 ms timeout
                sleep(1)
                continue
            # Both self.proc.stdout.read(SIZE) and self.proc.stdout.readline()
            # can block. However os.read(FILENO, SIZE) should be fine after a
            # poller.poll().
            try:
                data = os.read(self.proc.stdout.fileno(), self.READ_SIZE)
            except (IOError, OSError) as exc:
                dialog = warning_dialog(
                    "%s: %s" %
                    (exc, " ".join([quote(item) for item in command])))
                gobject.idle_add(dialog.warn)
                break
            if data:
                # Manage buffer, only add full lines to display to ensure
                # filtering and tagging work
                for line in data.splitlines(True):
                    if not line.endswith("\n"):
                        buf += line
                        continue
                    elif buf:
                        line = buf + line
                        buf = ""
                    if (not self.filters
                            or all([re.search(f, line)
                                    for f in self.filters])):
                        gobject.idle_add(self.update_gui, line)
            sleep(0.01)
        self.stop()

    def stop(self):
        """Stop the tailer."""
        self.quit = True
        try:
            # It is important that we kill processes like "tail -F", or it will
            # hang the GUI.
            os.killpg(self.proc.pid, signal.SIGTERM)
            self.proc.wait()
        except (AttributeError, OSError):
            pass

    def update_gui(self, line):
        """Update the GUI viewer."""
        try:
            line.decode('utf-8')
        except UnicodeDecodeError as exc:
            if self.has_warned_corrupt:
                return False
            self.has_warned_corrupt = True
            dialog = warning_dialog("Problem reading file:\n    %s: %s" %
                                    (type(exc).__name__, exc))
            gobject.idle_add(dialog.warn)
            return False
        for word, setting in self.TAGS.items():
            rec, colour = setting
            if rec.match(line):
                if word not in self.tags:
                    self.tags[word] = self.logbuffer.create_tag(
                        None, foreground=colour)
                self.logbuffer.insert_with_tags(self.logbuffer.get_end_iter(),
                                                line, self.tags[word])
                break
        else:
            self.logbuffer.insert(self.logbuffer.get_end_iter(), line)
        self.logview.scroll_to_iter(self.logbuffer.get_end_iter(), 0)
        return False
Beispiel #56
0
    def runWithSudo(self, password="", silent=True):
        """
        Use pty method to run "sudo" to run a command with elevated privilege.

        Required parameters: user, password, command

        @author: Roy Nielsen
        """
        self.logger.log(lp.DEBUG, "Starting runWithSudo: ")
        self.logger.log(lp.DEBUG, "\tcmd : " + str(self.command))
        if re.match("^\s+$", password) or \
           not password or \
           not self.command:
            self.logger.log(lp.WARNING, "Cannot pass in empty parameters...")
            self.logger.log(lp.WARNING, "check password...")
            if not silent:
                self.logger.log(lp.WARNING, "command: " + str(self.command))
            return (255)
        else:
            output = ""
            cmd = ["/usr/bin/sudo", "-S", "-s"]

            if isinstance(self.command, list):
                cmd = cmd + [" ".join(self.command)]

            elif isinstance(self.command, str):
                cmd = cmd + [self.command]

            try:
                (master, slave) = pty.openpty()
            except SubprocessError as err:
                self.logger.log(lp.WARNING,
                                "Error trying to open pty: " + str(err))
                self.logger.log(lp.WARNING, traceback.format_exc())
                self.logger.log(lp.WARNING, str(err))
                raise err
            else:
                try:
                    proc = Popen(cmd,
                                 stdin=slave,
                                 stdout=slave,
                                 stderr=slave,
                                 close_fds=True,
                                 text=self.text)
                except SubprocessError as err:
                    self.logger.log(
                        lp.WARNING,
                        "Error opening process to pty: " + str(err))
                    self.logger.log(lp.WARNING, traceback.format_exc())
                    self.logger.log(lp.WARNING, str(err))
                    raise err
                else:
                    #####
                    # Catch the sudo password prompt
                    # prompt = os.read(master, 512)
                    self.waitnoecho(master, 3)
                    self.prompt = os.read(master, 512)

                    #####
                    # Enter the sudo password
                    os.write(master, password + "\n")

                    #####
                    # Catch the password
                    os.read(master, 512)

                    # output = tmp + output
                    while True:
                        #####
                        # timeout of 0 means "poll"
                        ready, _, _ = select.select([master], [], [], 0)
                        if ready:
                            line = os.read(master, 512)
                            #####
                            # Warning, uncomment at your own risk - several
                            # programs print empty lines that will cause this
                            # to break and the output will be all goofed up.
                            # if not line :
                            #    break
                            # print output.rstrip()
                            output = output + line
                        elif proc.poll() is not None:
                            break
                        # print output.strip()
                    os.close(master)
                    os.close(slave)
                    self.libc.sync()
                    proc.wait()
                    self.libc.sync()
                    self.stdout = output
                    self.stderr = proc.stderr
                    self.retcode = proc.returncode
                    #print output.strip()
            #output = output.strip()
            if not silent:
                #####
                # ONLY USE WHEN IN DEVELOPMENT AND DEBUGGING OR YOU MAY
                # REVEAL MORE THAN YOU WANT TO IN THE LOGS!!!
                self.logger.log(lp.DEBUG, "\n\nLeaving runAs with Sudo: \"" + \
                                str(output) + "\"\n" + str(self.stdout) + "\n")
            return output
Beispiel #57
0
class DaemonProcess(GObject.GObject):
	__gsignals__ = {
		# line(text)	- emited when process outputs full line
		b"line"			: (GObject.SIGNAL_RUN_FIRST, None, (object,)),
		# exit(code)	- emited when process exits
		b"exit"			: (GObject.SIGNAL_RUN_FIRST, None, (int,)),
		# failed(exception) - emited if process fails to start
		b"failed"		: (GObject.SIGNAL_RUN_FIRST, None, (object,)),
	}
	SCROLLBACK_SIZE = 500	# Maximum number of output lines stored in memory
	PRIORITY_LOWEST		= 19
	PRIORITY_LOW		= 10
	PRIORITY_NORMAL		= 0
	PRIORITY_HIGH		= -10
	PRIORITY_HIGHEST	= -20
	
	def __init__(self, cmdline, priority=PRIORITY_NORMAL, max_cpus=0, env={}):
		""" cmdline should be list of arguments """
		GObject.GObject.__init__(self)
		self.cmdline = cmdline
		self.priority = priority
		self.env = { x:env[x] for x in env }
		self.env["STNORESTART"] = "1"	# see syncthing --help
		self.env["STNOUPGRADE"] = "1"
		if max_cpus > 0:
			self.env["GOMAXPROCS"] = str(max_cpus)
		self._proc = None
	
	def start(self):
		for x in self.env:
			os.environ[x] = self.env[x]
		try:
			self._cancel = Gio.Cancellable()
			if IS_WINDOWS:
				# Windows
				sinfo = STARTUPINFO()
				sinfo.dwFlags = STARTF_USESHOWWINDOW
				sinfo.wShowWindow = 0
				cflags = nice_to_priority_class(self.priority)
				self._proc = Popen(self.cmdline,
							stdin=PIPE, stdout=PIPE, stderr=PIPE,
							startupinfo=sinfo, creationflags=cflags)
				self._stdout = WinPopenReader(self._proc.stdout)
				self._check = GLib.timeout_add_seconds(1, self._cb_check_alive)
			elif HAS_SUBPROCESS:
				# New Gio
				flags = Gio.SubprocessFlags.STDOUT_PIPE | Gio.SubprocessFlags.STDERR_MERGE
				if self.priority == 0:
					self._proc = Gio.Subprocess.new(self.cmdline, flags)
				else:
					# I just really do hope that there is no distro w/out nice command
					self._proc = Gio.Subprocess.new([ "nice", "-n", "%s" % self.priority ] + self.cmdline, flags)
				self._proc.wait_check_async(None, self._cb_finished)
				self._stdout = self._proc.get_stdout_pipe()
			else:
				# Gio < 3.12 - Gio.Subprocess is missing :(
				if self.priority == 0:
					self._proc = Popen(self.cmdline, stdout=PIPE)
				else:
					# still hoping
					self._proc = Popen([ "nice", "-n", "%s" % self.priority ], stdout=PIPE)
				self._stdout = Gio.UnixInputStream.new(self._proc.stdout.fileno(), False)
				self._check = GLib.timeout_add_seconds(1, self._cb_check_alive)
		except Exception as e:
			# Startup failed
			self.emit("failed", e)
			return
		self._lines = deque([], DaemonProcess.SCROLLBACK_SIZE)
		self._buffer = ""
		self._stdout.read_bytes_async(256, 0, self._cancel, self._cb_read, ())
	
	def _cb_read(self, pipe, results, *a):
		""" Handler for read_bytes_async """
		try:
			response = pipe.read_bytes_finish(results)
		except Exception as e:
			if not self._cancel.is_cancelled():
				log.exception(e)
				GLib.idle_add(pipe.read_bytes_async, 256, 1, None, self._cb_read)
			return
		response = response.get_data().decode('utf-8')
		self._buffer = "%s%s" % (self._buffer, response)
		while "\n" in self._buffer:
			line, self._buffer = self._buffer.split("\n", 1)
			self._lines.append(line)
			self.emit('line', line)
		if not self._cancel.is_cancelled():
			GLib.idle_add(pipe.read_bytes_async, 256, 1, None, self._cb_read, ())
	
	def _cb_check_alive(self, *a):
		"""
		Repeatedly check if process is still alive.
		Called only on windows
		"""
		if self._proc == None:
			# Never started or killed really fast
			self.emit('exit', 1)
			self._cancel.cancel()
			if IS_WINDOWS: self._stdout.close()
			return False
		self._proc.poll()
		if self._proc.returncode is None:
			# Repeat until finished or canceled
			return (not self._cancel.is_cancelled())
		# Child just died :)
		self.emit('exit', self._proc.returncode)
		self._cancel.cancel()
		if IS_WINDOWS: self._stdout.close()
		return False
	
	def _cb_finished(self, proc, results):
		"""
		Callback for wait_check_async.
		With Gio < 3.12, timer and _cb_check_alive is used.
		"""
		try:
			proc.wait_check_finish(results)
			log.info("Subprocess finished with code %s", proc.get_exit_status())
		except GLib.GError:
			# Exited with exit code
			log.info("Subprocess exited with code %s", proc.get_exit_status())
		if proc.get_exit_status() == 127:
			# Command not found
			self.emit("failed", Exception("Command not found"))
		else:
			self.emit('exit', proc.get_exit_status())
		if IS_WINDOWS: self._stdout.close()
		self._cancel.cancel()
	
	def terminate(self):
		""" Terminates process (sends SIGTERM) """
		if not self._proc is None:
			if IS_WINDOWS:
				# Windows
				self._proc.terminate()
			elif HAS_SUBPROCESS:
				# Gio.Subprocess
				self._proc.send_signal(15)
			else:
				# subprocess.Popen
				self._proc.terminate()
			self._proc = None
			if IS_WINDOWS: self._stdout.close()
			self._cancel.cancel()
	
	def kill(self):
		""" Kills process (sends SIGTERM) """
		if not self._proc is None:
			if IS_WINDOWS:
				# Windows - can't actually kill
				self._proc.terminate()
			elif HAS_SUBPROCESS:
				# Gio.Subprocess
				self._proc.force_exit()
			else:
				# subprocess.Popen
				self._proc.kill()
			self._proc = None
			if IS_WINDOWS: self._stdout.close()
			self._cancel.cancel()
	
	def get_output(self):
		""" Returns process output as iterable list of lines """
		return self._lines
	
	def get_commandline(self):
		""" Returns commandline used to start process """
		return self.cmdline
Beispiel #58
0
def start_dbus(dbus_launch):
    if not dbus_launch or dbus_launch.lower() in FALSE_OPTIONS:
        log("start_dbus(%s) disabled", dbus_launch)
        return 0, {}
    bus_address = os.environ.get("DBUS_SESSION_BUS_ADDRESS")
    log("dbus_launch=%s, current DBUS_SESSION_BUS_ADDRESS=%s", dbus_launch,
        bus_address)
    if bus_address:
        log(
            "start_dbus(%s) disabled, found an existing DBUS_SESSION_BUS_ADDRESS=%s",
            dbus_launch, bus_address)
        return 0, {}
    assert POSIX
    try:

        def preexec():
            os.setsid()
            close_fds()

        env = dict((k, v) for k, v in os.environ.items() if k in (
            "PATH",
            "SSH_CLIENT",
            "SSH_CONNECTION",
            "XDG_CURRENT_DESKTOP",
            "XDG_SESSION_TYPE",
            "XDG_RUNTIME_DIR",
            "SHELL",
            "LANG",
            "USER",
            "LOGNAME",
            "HOME",
            "DISPLAY",
            "XAUTHORITY",
            "CKCON_X11_DISPLAY",
        ))
        import shlex
        cmd = shlex.split(dbus_launch)
        log("start_dbus(%s) env=%s", dbus_launch, env)
        proc = Popen(cmd,
                     stdin=PIPE,
                     stdout=PIPE,
                     shell=False,
                     env=env,
                     preexec_fn=preexec)
        out = proc.communicate()[0]
        assert proc.poll() == 0, "exit code is %s" % proc.poll()
        #parse and add to global env:
        dbus_env = {}
        log("out(%s)=%s", cmd, nonl(out))
        for l in bytestostr(out).splitlines():
            if l.startswith("export "):
                continue
            sep = "="
            if l.startswith("setenv "):
                l = l[len("setenv "):]
                sep = " "
            if l.startswith("set "):
                l = l[len("set "):]
            parts = l.split(sep, 1)
            if len(parts) != 2:
                continue
            k, v = parts
            if v.startswith("'") and v.endswith("';"):
                v = v[1:-2]
            elif v.endswith(";"):
                v = v[:-1]
            dbus_env[k] = v
        dbus_pid = int(dbus_env.get("DBUS_SESSION_BUS_PID", 0))
        log("dbus_pid=%i, dbus-env=%s", dbus_pid, dbus_env)
        return dbus_pid, dbus_env
    except Exception as e:
        log("start_dbus(%s)", dbus_launch, exc_info=True)
        log.error("dbus-launch failed to start using command '%s':\n" %
                  dbus_launch)
        log.error(" %s\n" % e)
        return 0, {}
Beispiel #59
0
class AcmeCapeInstrument(Instrument):

    mode = CONTINUOUS

    def __init__(self,
                 target,
                 iio_capture=which('iio-capture'),
                 host='baylibre-acme.local',
                 iio_device='iio:device0',
                 buffer_size=256):
        super(AcmeCapeInstrument, self).__init__(target)
        self.iio_capture = iio_capture
        self.host = host
        self.iio_device = iio_device
        self.buffer_size = buffer_size
        self.sample_rate_hz = 100
        if self.iio_capture is None:
            raise HostError('Missing iio-capture binary')
        self.command = None
        self.process = None

        self.add_channel('shunt', 'voltage')
        self.add_channel('bus', 'voltage')
        self.add_channel('device', 'power')
        self.add_channel('device', 'current')
        self.add_channel('timestamp', 'time_ms')

    def __del__(self):
        if self.process and self.process.pid:
            self.logger.warning('killing iio-capture process [{}]...'.format(
                self.process.pid))
            self.process.kill()

    def reset(self, sites=None, kinds=None, channels=None):
        super(AcmeCapeInstrument, self).reset(sites, kinds, channels)
        self.raw_data_file = tempfile.mkstemp('.csv')[1]
        params = dict(iio_capture=self.iio_capture,
                      host=self.host,
                      buffer_size=self.buffer_size,
                      iio_device=self.iio_device,
                      outfile=self.raw_data_file)
        self.command = IIOCAP_CMD_TEMPLATE.substitute(**params)
        self.logger.debug('ACME cape command: {}'.format(self.command))

    def start(self):
        self.process = Popen(self.command.split(), stdout=PIPE, stderr=STDOUT)

    def stop(self):
        self.process.terminate()
        timeout_secs = 10
        output = ''
        for _ in range(timeout_secs):
            if self.process.poll() is not None:
                break
            time.sleep(1)
        else:
            output += _read_nonblock(self.process.stdout)
            self.process.kill()
            self.logger.error('iio-capture did not terminate gracefully')
            if self.process.poll() is None:
                msg = 'Could not terminate iio-capture:\n{}'
                raise HostError(msg.format(output))
        if self.process.returncode != 15:  # iio-capture exits with 15 when killed
            if sys.version_info[0] == 3:
                output += self.process.stdout.read().decode(
                    sys.stdout.encoding, 'replace')
            else:
                output += self.process.stdout.read()
            self.logger.info('ACME instrument encountered an error, '
                             'you may want to try rebooting the ACME device:\n'
                             '  ssh root@{} reboot'.format(self.host))
            raise HostError(
                'iio-capture exited with an error ({}), output:\n{}'.format(
                    self.process.returncode, output))
        if not os.path.isfile(self.raw_data_file):
            raise HostError('Output CSV not generated.')
        self.process = None

    def get_data(self, outfile):
        if os.stat(self.raw_data_file).st_size == 0:
            self.logger.warning('"{}" appears to be empty'.format(
                self.raw_data_file))
            return

        all_channels = [c.label for c in self.list_channels()]
        active_channels = [c.label for c in self.active_channels]
        active_indexes = [all_channels.index(ac) for ac in active_channels]

        with csvreader(self.raw_data_file, skipinitialspace=True) as reader:
            with csvwriter(outfile) as writer:
                writer.writerow(active_channels)

                header = next(reader)
                ts_index = header.index('timestamp ms')

                for row in reader:
                    output_row = []
                    for i in active_indexes:
                        if i == ts_index:
                            # Leave time in ms
                            output_row.append(float(row[i]))
                        else:
                            # Convert rest into standard units.
                            output_row.append(float(row[i]) / 1000)
                    writer.writerow(output_row)
        return MeasurementsCsv(outfile, self.active_channels,
                               self.sample_rate_hz)

    def get_raw(self):
        return [self.raw_data_file]
Beispiel #60
0
    def send_heartbeats(self):
        if python_binary():
            heartbeat = self.build_heartbeat(**self.heartbeat)
            ua = 'sublime/%d sublime-wakatime/%s' % (ST_VERSION, __version__)
            cmd = [
                python_binary(),
                API_CLIENT,
                '--entity',
                heartbeat['entity'],
                '--time',
                str('%f' % heartbeat['timestamp']),
                '--plugin',
                ua,
            ]
            if self.api_key:
                cmd.extend(
                    ['--key',
                     str(bytes.decode(self.api_key.encode('utf8')))])
            if heartbeat['is_write']:
                cmd.append('--write')
            if heartbeat.get('alternate_project'):
                cmd.extend(
                    ['--alternate-project', heartbeat['alternate_project']])
            if heartbeat.get('cursorpos') is not None:
                cmd.extend(['--cursorpos', heartbeat['cursorpos']])
            for pattern in self.ignore:
                cmd.extend(['--exclude', pattern])
            if self.debug:
                cmd.append('--verbose')
            if self.hidefilenames:
                cmd.append('--hidefilenames')
            if self.proxy:
                cmd.extend(['--proxy', self.proxy])
            if self.has_extra_heartbeats:
                cmd.append('--extra-heartbeats')
                stdin = PIPE
                extra_heartbeats = [
                    self.build_heartbeat(**x) for x in self.extra_heartbeats
                ]
                extra_heartbeats = json.dumps(extra_heartbeats)
            else:
                extra_heartbeats = None
                stdin = None

            log(DEBUG, ' '.join(obfuscate_apikey(cmd)))
            try:
                process = Popen(cmd, stdin=stdin, stdout=PIPE, stderr=STDOUT)
                inp = None
                if self.has_extra_heartbeats:
                    inp = "{0}\n".format(extra_heartbeats)
                    inp = inp.encode('utf-8')
                output, err = process.communicate(input=inp)
                output = u(output)
                retcode = process.poll()
                if (not retcode or retcode == 102) and not output:
                    self.sent()
                else:
                    update_status_bar('Error')
                if retcode:
                    log(
                        DEBUG if retcode == 102 else ERROR,
                        'wakatime-core exited with status: {0}'.format(
                            retcode))
                if output:
                    log(ERROR, u('wakatime-core output: {0}').format(output))
            except:
                log(ERROR, u(sys.exc_info()[1]))
                update_status_bar('Error')

        else:
            log(ERROR, 'Unable to find python binary.')
            update_status_bar('Error')