def run(): args = _parse_args() try: tornado = Popen(['python', '../static-server.py']) node = Popen(['node', 'build/server/server.js']) sleep(1) for url, page_name in PAGE_LIST: out_file = os.path.join(BUILD_DIR, page_name + '.html') # useless, since template is cached by tornado os.unlink(out_file) url = TORNADO_SERVER[:-1] + url phantom = Popen(['phantomjs', 'ph.js', url], stdout=PIPE) styles = phantom.stdout.readline().rstrip() load_css_func = _process_js_func(LOAD_CSS_FUNC) template = TEMPLATE.format(style=styles, load_css_func=load_css_func) with open(out_file, 'w') as f: f.write(template) print("out file:", out_file) phantom.communicate() finally: try: if tornado.poll() is None: tornado.terminate() except Exception: pass try: if node.poll() is None: node.terminate() except Exception: pass
class GraphNode: def __init__(self, name, command): self.name = name self.command = command self.process = None self.stdin = None self.stdout = None self.outputs = [] def execute(self): if self.command is not None: self.process = Popen(self.command, shell=True, stdin=PIPE, stdout=PIPE, stderr=None) make_nonblocking(self.process.stdout.fileno()) make_nonblocking(self.process.stdin.fileno()) self.stdin = GraphNodeStream(self, self.process.stdin, 'stdin') self.stdout = GraphNodeStream(self, self.process.stdout, 'stdout') # Returns False iff this node will never produce more data def is_live(self): if self.process is not None: self.process.poll() return self.command is None or (self.process is not None and self.process.returncode is None) def is_readable(self): return self.stdout.is_live() def is_writable(self): return self.stdin.is_live() def __repr__(self): if self.command is None: return "(" + self.name + ")" return "(" + self.name + ": " + self.command + ")"
class Player: def __init__(self, name, cmd): self.name = name self.forts = set() self.marches = set() args = cmd.split(' ') self.process = Popen(list(args) + [name], stdin=PIPE, stdout=PIPE, universal_newlines=True) def capture(self, fort): if fort.owner: fort.owner.forts.remove(fort) self.forts.add(fort) fort.owner = self def is_defeated(self): return (not self.forts) and (not self.marches) def send_state(self): if not self.process.poll(): self.process.stdin.write(show_visible(self.forts)) self.process.stdin.write('\n') self.process.stdin.flush() def read_commands(self, game): if not self.process.poll(): try: read_commands(game, self, self.process.stdout) except StopIteration: self.process.kill()
def run(target, timeout=TIMEOUT, retry=RETRY, preserveChildren=False, verbose=False): """Run a process but kill it after timeout @param target: the command to launch (str) @param timeout: timeout in seconds before start killing process (int) @param retry: Number of time we will try to kill the process with SIGTERM and SIGKILL (int) @param preserveChildren: Do we need to also kill process children ? Default is True (bool) @param verbose: Print what happened on standard output (bool) @return: 0 if everything is ok. 1 if process was killed, 2 if something goes wrong. """ rc=0 # Some sanity checks if timeout<0 or retry<1: print "Timeout must be a positive integer and number of retry must be greater or equal than 1" return 2 if verbose: print "running %s" % target process=Popen(target, shell=True) endTime=time()+timeout while True: # Timeout if time()>endTime: rc=1 if verbose: print "Process timeout" break # Process finish if process.poll() is not None: if verbose: print "Process finish before timeout" break # Wait a little before looping sleep(0.05) if process.poll() is not None: if verbose: print "process correctly finished" else: if preserveChildren: pids=[] else: pids=getChildrenPid(process.pid, verbose) pids.append(process.pid) for i in xrange(retry): for signal in (15, 9): # SIGTERM then SIGKILL for pid in pids: if verbose: print "kill %s with signal %s" % (pid, signal) try: os.kill(pid, signal) except OSError, e: if e.errno==1: print "Not authorized to kill %s" % pid elif e.errno==3: # No such process - already dead pass else: print "Error while killing %s:\n%s" % (pid, e) sleep(i)
def testBasic(self): iface = self.tap.name record = PacketRecord() # start capture process = Popen([APP, iface, CAPTURE_FILE], stdout=DEV_NULL, stderr=DEV_NULL) # send packets for i in range(PACKET_COUNT): packet = IP(dst="www.google.com")/ICMP() sendp(packet, iface=iface, verbose=False) record.add_sent(packet) # wait for stragglers time.sleep(1) # stop capture process.terminate() # hack: send one more packet to make sure capture closes immediately sendp(IP(), iface=iface, verbose=False) process.poll() # verify capture file for packet in rdpcap(CAPTURE_FILE): record.add_received(packet) self.assertTrue(record.verify())
def exec_and_wait(self, cmd, timelife = 10): """ exec_and_wait is syncronous with a timelife given by parameter whether is reached the command request is returning and error. retruncode is an Operating System err code, != 0 are error codes """ proc = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) t_nought = time.time() seconds_passed = 0 self.pid = proc.pid self.stderr = proc.stderr self.stdout = proc.stdout.read().rstrip('\n') self.returncode = proc.poll() while(self.returncode != 0 and seconds_passed < timelife): seconds_passed = time.time() - t_nought self.returncode = proc.poll() if seconds_passed >= timelife: self.returncode = 1 self.stdout = "Timelife expired, connection aborted" return self.returncode = 0 return
def testLocal(self): # note: ignore_zero_padding necessary until '0' padding simulation bug is resolved record = PacketRecord(ignore_zero_padding=True) # start capture process = Popen([CLIENT_APP, DFE_IP, DFE_NETMASK, '-l', CAPTURE_FILE], env=self.env, \ stdout=DEV_NULL, stderr=DEV_NULL) self.processes.append(process) # send packets for i in range(PACKET_COUNT): packet = IP(dst='127.0.0.2')/ICMP() sendp(packet, iface=self.iface, verbose=False) record.add_sent(packet) # wait for stragglers time.sleep(1) # make sure still running process.poll() self.assertTrue(process.returncode == None) # stop capture process.terminate() # hack: send one more packet to make sure capture closes immediately sendp(IP(), iface=self.iface, verbose=False) process.wait() # verify capture CAPTURE_FILE for packet in rdpcap(CAPTURE_FILE): record.add_received(packet) self.assertTrue(record.verify())
def _launch_kernel(cmd): """start an embedded kernel in a subprocess, and wait for it to be ready Returns ------- kernel, kernel_manager: Popen instance and connected KernelManager """ kernel = Popen([sys.executable, '-c', cmd], stdout=PIPE, stderr=PIPE, env=env) connection_file = os.path.join(IPYTHONDIR, 'profile_default', 'security', 'kernel-%i.json' % kernel.pid ) # wait for connection file to exist, timeout after 5s tic = time.time() while not os.path.exists(connection_file) and kernel.poll() is None and time.time() < tic + 5: time.sleep(0.1) if not os.path.exists(connection_file): if kernel.poll() is None: kernel.terminate() raise IOError("Connection file %r never arrived" % connection_file) if kernel.poll() is not None: raise IOError("Kernel failed to start") km = BlockingKernelManager(connection_file=connection_file) km.load_connection_file() km.start_channels() return kernel, km
def run_and_capture(self): """ Run a command and capture exceptions. This is a blocking call :returns: tuple of exitcode, error (or None) :rtype: int, str | None """ subproc = Popen([self._executable, self._command] + self._args, stderr=PIPE) err = '' while subproc.poll() is None: line = subproc.stderr.readline().decode('utf-8') err += line sys.stderr.write(line) sys.stderr.flush() exitcode = subproc.poll() # We only want to catch exceptions, not other stderr messages # (such as "task does not exist", so we look for the 'Traceback' # string. This only works for python, so we'll need to revisit # this in the future when we support subcommands written in other # languages. err = ('Traceback' in err and err) or None return exitcode, err
class RunServer(object): def __init__(self, instance_name, port): self.instance_name = instance_name self.port = port self.lan_ip = get_lan_ip() self.process = None self.out = None def start(self): self.stop() cmd = [sys.executable, 'manage_%s.py' % self.instance_name, 'runserver', '--noreload', '--traceback', '0.0.0.0:%d' % self.port] self.process = Popen(cmd) sleep(3.0) if self.process.poll() is not None: self.stop() raise RunException(ugettext("Error to start!")) self.open_url() def open_url(self): webbrowser.open_new("http://%(ip)s:%(port)d" % {'ip': self.lan_ip, 'port': self.port}) def stop(self): if self.is_running(): self.process.terminate() self.process = None self.out = None def is_running(self): return (self.process is not None) and (self.process.poll() is None)
def run(cmd): handle = Popen(cmd, shell=True, stdout=PIPE, stderr=STDOUT) while handle.poll() is None: print(handle.stdout.read(1).decode('utf-8'), end='') print(handle.stdout.read(1).decode('utf-8'), end='') # Last byte, if there is any. handle.stdout.close() return handle.poll()
def run(self): try: if osflag: proc=Popen(self.cmd,shell=False,stdin=None,stdout=PIPE,\ stderr=STDOUT,bufsize=0) else: from subprocess import STARTUPINFO si=STARTUPINFO() si.dwFlags|=1 si.wShowWindow=0 proc=Popen(self.cmd,shell=False,stdin=None,stdout=PIPE,\ stderr=STDOUT,bufsize=0,startupinfo=si) while 1: if self.stop_flag: if osflag: proc.send_signal(signal.SIGKILL) else: proc.kill() break if osflag: if proc.stdout in select.select([proc.stdout],[],[],1)[0]: line=proc.stdout.readline() else: line=' \n' else: line=proc.stdout.readline() if not len(line): break else: if count(line,'ttl') or count(line,'TTL'): self.retries=0 else: self.retries=self.retries+1 line=' ' sleep(0.5) proc.poll() except: pass
def _popen(self, command, timeout, timeout_msg, rc_non_zero_msg, common_msg=""): """ Runs a command in background and returns its return_code, stdout and stderr. stdout and stderr will be None if return code = 0 """ stdout, stderr = None, None # Run the command p = Popen(command, stdout=PIPE, stderr=PIPE) # Sleep as long as requested and poll for results sleep(timeout) p.poll() if p.returncode is None: msg = timeout_msg + common_msg + "command:[{}]".format(command) raise Exception(msg.format(timeout)) else: if p.returncode != 0: stdout, stderr = p.communicate() msg = ( rc_non_zero_msg + common_msg + "command:[{}], return code:[{}], stdout:[{}], stderr:[{}] ".format( command, p.returncode, stdout, stderr ) ) raise Exception(msg) return p.returncode
def compile(self, filename): if (self.debug_output == 1): print("compile " + self.processor) else: chemin = os.path.dirname(filename) fichier = open(sys.path[0] + "/tmp/stdout", 'w+') sortie = Popen([sys.path[0] + "/tools/bin/sdcc", "-mpic16", "--denable-peeps", "--obanksel=9", "--opt-code-size", "--optimize-cmp", "--optimize-df", "-p" + self.processor, "-I" + sys.path[0] + "/include", "-I" + chemin + "/", "-c", "-c", "-o" + sys.path[0] + "/source/main.o", sys.path[0] + "/source/main.c"], stdout=fichier, stderr=STDOUT) sortie.communicate() if sortie.poll()!=0: fichier.seek(0) self.displaymsg(fichier.read(), 0) fichier.seek(0) line=fichier.readline() if line.find("error")!=-1: number=line[line.find(":")+1:line.find("error")-2] self.editor.highlightline(int(number)-1,'pink') self.displaymsg("error line"+" "+number+"\n",1) self.displaymsg(line[line.find("error")+10:len(line)],0) fichier.close() return sortie.poll()
def test_difftool(git_repo, request): nbdime.gitdifftool.main(['config', '--enable']) cmd = get_output('git config --get --local difftool.nbdime.cmd').strip() # pick a non-random port so we can connect later, and avoid opening a browser port = 62021 cmd = cmd + ' --port=%i --browser=disabled' % port call(['git', 'config', 'difftool.nbdime.cmd', cmd]) # avoid global diff driver config from disabling difftool: with open('.gitattributes', 'w') as f: f.write('*.ipynb\tdiff=notnbdime') p = Popen(['git', 'difftool', '--tool=nbdime', 'base']) def _term(): try: p.terminate() except OSError: pass request.addfinalizer(_term) # 3 is the number of notebooks in this diff url = 'http://127.0.0.1:%i' % port for i in range(3): _wait_up(url, check=lambda : p.poll() is None) # server started r = requests.get(url + '/difftool') r.raise_for_status() # close it r = requests.post(url + '/api/closetool', headers={'exit_code': '0'}) r.raise_for_status() time.sleep(0.25) # wait for exit p.wait() assert p.poll() == 0
def main(args=sys.argv[1:]): parser = argparse.ArgumentParser(description="Adapt socket communication.") parser.add_argument("--server") parser.add_argument("--port", type=int) parser.add_argument("--legacy", nargs=2) config = parser.parse_args(args) if config.legacy: address = (config.legacy[0], int(config.legacy[1])) else: address = (config.server, config.port) sock = socket.socket() sock.connect(address) proc = Popen("simple_engine", stdin=PIPE, stdout=PIPE, universal_newlines=True) com = _ProcCom(proc, sock) com.start() while proc.poll() is None: msg = sock.recv(4096) if len(msg) == 0: break proc.stdin.write(msg) proc.stdin.flush() if proc.poll() is None: proc.stdin.write("quit\n") proc.stdin.flush() com.stop.set() sock.close()
def worker(self, cmd_q, res_q): while True: name, cmd = cmd_q.get(True) self.log.debug("External : launch %s" % cmd) try: proc = Popen( cmd, stdin=None, stdout=PIPE, stderr=STDOUT, close_fds=True) output = "" while True: out, err = proc.communicate() proc.poll() output += out.decode('utf-8') if proc.returncode is not None: break except: self.log.debug(traceback.format_exc()) # Truncate multi-line output try: output = output[:output.index("\n")] except: pass res_q.put_nowait( (name, proc.returncode, output, int(time.time()))) cmd_q.task_done()
def upload_using_curl(self, source_directory, target_directory): logging.debug("Curl upload!") import os filesToUpload = os.listdir(source_directory) for file in filesToUpload: logging.warn(source_directory) logging.warn(file) filePath = os.path.join(source_directory, file) curlcommand = "curl --ftp-create-dirs -T {filepath} --keepalive-time 5 --user {user}:{password} ftp://{host}{remotedir}/".format( user=self.user, password=self.password, port=self.port, host=self.hostname, remotedir=target_directory, filepath=filePath, ) logging.debug(curlcommand) p = Popen(curlcommand) stdout, stderr = p.communicate() logging.debug(stdout) p.poll() curlReturnValue = p.returncode if curlReturnValue != 0: logging.error("CURL has encurred into an error! the program execution will stop!") raise Exception("CURL returned invalid status code")
def local_popen( f, cmdline ): if os.name != 'nt': cmdline="./" + cmdline if execution==0: print cmdline else: p=Popen( cmdline, shell=True, stdout=PIPE, stderr=STDOUT ) r=p.poll() while r == None: r=p.poll() pipe=p.stdout if r != 0: print "---- TESTING " + cmdline.split()[3] + "... FAILED(" + str(p.returncode) +") !" for line in pipe.readlines(): f.write(str(line)) else: found=0 for line in pipe.readlines(): f.write(str(line)) if "TESTING" in line : found = 1 print line, if found == 0: print cmdline.split()[0] + " " + cmdline.split()[3] + ": FAILED(Unexpected error)" f.flush(); return 0
def run_tsmuxer_2(msg, argv1, argv2): out = StringIO.StringIO() argv = ['tsMuxeR', argv1, argv2] p = Popen(argv, stdout=PIPE, stderr=PIPE) write("%s ...\n" % msg) while True: line = p.stdout.readline() if line == '': if p.poll() is not None: break #reached final line of output else: time.sleep(0.01) # sleep 10 milliseconds else: out.write(line) progress = extract_progress(line) if progress is not None: write("\r%5.1f%% completed" % progress) write("\n") try: assert p.poll() == 0 return True except AssertionError: print "ERROR: Muxing failed" print "Args:", argv out = out.read() print "STDOUT:" print out err = p.stderr.read() print "STDERR:" print err return False
class ExecuteHook(BaseHook): """ Hook that executes an arbitary command in the setup phase. If the command is still running during teardown, it will be terminated. """ def __init__(self, args): """ :param args: sequence of program arguments, see :class:`subprocess.Popen` for details :type args: string or sequence """ super(ExecuteHook, self).__init__() self.args = args self.proc = None def setup(self): self.proc = Popen(shlex.split(self.args) if isinstance(self.args, str) else self.args) def teardown(self): self.proc.poll() if self.proc.returncode is None: self.proc.terminate() self.proc.wait()
def setup_kernel(cmd): """start an embedded kernel in a subprocess, and wait for it to be ready Returns ------- kernel_manager: connected KernelManager instance """ kernel = Popen([sys.executable, "-c", cmd], stdout=PIPE, stderr=PIPE, env=env) connection_file = os.path.join(IPYTHONDIR, "profile_default", "security", "kernel-%i.json" % kernel.pid) # wait for connection file to exist, timeout after 5s tic = time.time() while not os.path.exists(connection_file) and kernel.poll() is None and time.time() < tic + SETUP_TIMEOUT: time.sleep(0.1) if kernel.poll() is not None: o, e = kernel.communicate() e = py3compat.cast_unicode(e) raise IOError("Kernel failed to start:\n%s" % e) if not os.path.exists(connection_file): if kernel.poll() is None: kernel.terminate() raise IOError("Connection file %r never arrived" % connection_file) client = BlockingKernelClient(connection_file=connection_file) client.load_connection_file() client.start_channels() try: yield client finally: client.stop_channels() kernel.terminate()
def start(self): """Start the process.""" import shlex from subprocess import Popen if self.process: raise RuntimeError("Process already started.") if self.host and self.network_ping(): raise RuntimeError("A process is already running on port %s" % self.port) if isinstance(self.cmd, basestring): cmd = shlex.split(self.cmd) else: cmd = self.cmd self.null = open('/dev/null', 'w') process = Popen(cmd, stdout=self.null, stderr=self.null, close_fds=True) if not self.host: time.sleep(0.35) if process.poll(): output = process.communicate()[0] raise RuntimeError("Did not start server! Woe!\n" + output) self.process = process return start = time.time() while process.poll() is None and time.time() - start < 15: if self.network_ping(): break else: output = process.communicate()[0] raise RuntimeError("Did not start server! Woe!\n" + output) self.process = process
def yes_proc(args, yes="yes"): proc = Popen(args, stdin=PIPE) while proc.returncode is None: proc.communicate(yes) proc.poll() return proc.returncode == 0
def wait_process_finish(scan, profile_fname): ''' Start w3af process in new process, and wait while it terminate ''' process = Popen([settings.W3AF_RUN, '--no-update', '-P', profile_fname], stdout=PIPE, stderr=PIPE) scan.pid = process.pid scan.save() while process.returncode is None: scan.last_updated = datetime.today() scan.save() sleep(W3AF_POLL_PERIOD) # wail process.poll() # get process status scan.set_task_status_free() finish_time = datetime.now() scan.finish = finish_time scan.save() target = scan.scan_task.target target.last_scan = finish_time target.save() logger.info('w3af Process return code %s' % process.returncode) return process.returncode
class TestServer(unittest.TestCase): server = 'WSGIRefServer' port = 12643 def setUp(self): # Start servertest.py in a subprocess cmd = [sys.executable, serverscript, self.server, str(self.port)] cmd += sys.argv[1:] # pass cmdline arguments to subprocesses self.p = Popen(cmd, stdout=PIPE, stderr=PIPE) # Wait for the socket to accept connections for i in xrange(100): time.sleep(0.1) # Check if the process has died for some reason if self.p.poll() != None: break if ping('127.0.0.1', self.port): break def tearDown(self): while self.p.poll() is None: os.kill(self.p.pid, signal.SIGINT) time.sleep(0.1) if self.p.poll() is None: os.kill(self.p.pid, signal.SIGTERM) for stream in (self.p.stdout, self.p.stderr): for line in stream: if tob('Warning') in line \ or tob('Error') in line: print line.strip().decode('utf8') def fetch(self, url): try: return urllib2.urlopen('http://127.0.0.1:%d/%s' % (self.port, url)).read() except Exception, e: return repr(e)
def start(self, timeout = 20): if self.is_running(): raise Exception('%s is already running' % self.name) if not self.run_cmd: raise Exception('No starting command registered for %s' % self.name) if type(self.run_cmd) is types.FunctionType: self.run_cmd() return with tempfile.TemporaryFile() as stdout_file, tempfile.TemporaryFile() as stderr_file: proc = Popen(shlex.split('%s -p %s' % (self.run_cmd, self.port)), cwd = self.dir, close_fds = True, stdout = stdout_file, stderr = stderr_file) if timeout > 0: poll_rate = 0.1 for i in range(int(timeout/poll_rate)): if self.is_running(): break sleep(poll_rate) if bool(proc.poll()): # process ended with error stdout_file.seek(0) stderr_file.seek(0) raise Exception('Run of %s ended unexpectfully: %s' % (self.name, [proc.returncode, stdout_file.read().decode(errors = 'replace'), stderr_file.read().decode(errors = 'replace')])) elif proc.poll() == 0: # process runs other process, and ended break if self.is_running(): if self.check_connectivity(): return True raise Exception('Daemon process is running, but no connectivity') raise Exception('%s failed to run.' % self.name)
class Inetd: def __init__(self, inq, outq): self.inq=inq self.outq=outq def start(self): self.proc=Popen(['fossil', 'http', '/home/blanu/fossil/'], stdin=PIPE, stdout=PIPE) # self.proc=Popen(['fossil', 'help'], stdin=PIPE, stdout=PIPE) t=Thread(target=self.processIn) t.setDaemon(True) t.start() t2=Thread(target=self.processOut) # t2.setDaemon(True) t2.start() def processIn(self): self.proc.poll() while not self.proc.returncode: data=self.inq.get() print('inetd -> '+str(data)) self.proc.stdin.write(data) self.proc.stdin.flush() self.proc.poll() def processOut(self): print('waiting to finish') # self.proc.wait() print('finished') print('waiting to read from inetd') data=self.proc.stdout.read() print('inetd <- '+str(data)) self.outq.put(data)
class Engine: def __init__(self, cmdline): self.proc = Popen(cmdline, stdin = PIPE, stdout = PIPE) self.proc_com = _ProcCom(self.proc) self.proc_com.start() def send(self, msg): self.proc.stdin.write(msg) self.proc.stdin.flush() def readline(self, timeout=None): if self.proc.poll(): return "" try: msg = self.proc_com.outq.get(timeout=timeout) except Empty: raise socket.timeout() return msg def cleanup(self): self.proc_com.stop.set() if self.proc.poll() is None: if sys.platform == 'win32': import ctypes handle = int(self.proc._handle) ctypes.windll.kernel32.TerminateProcess(handle, 0) else: os.kill(self.proc.pid, signal.SIGTERM)
class CMatProcess: def __init__(self, processPath='./cmatprocess/cmatprocess', nCores=4): assert processPath is not None self.cmd = [processPath, str(nCores)] self.p = Popen(self.cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE, bufsize=0) def exitHandler(): if self.p.poll() is None: self.p.kill() atexit.register(exitHandler) def processPaths(self, imgPaths, f): rc = self.p.poll() if rc is not None and rc != 0: raise Exception("Error from process!") self.p.stdin.write(f + ' ' + ' '.join(imgPaths) + '\n') try: n = int(self.p.stdout.readline()) result = [] for i in range(n): fileLocation = self.p.stdout.readline().strip() if len(fileLocation) > 3: result.append(cv2.imread(fileLocation)) return result except Exception as e: self.p.kill() stdout, stderr = self.p.communicate() print("Error from process output: " + stdout + '\n' + stderr) sys.exit(-1)
def run(self): self.started_event() mutex.lock() log_entry = EventLogModel(category='borg-run', subcommand=self.cmd[1], profile=self.params.get( 'profile_name', None)) log_entry.save() logger.info('Running command %s', ' '.join(self.cmd)) p = Popen(self.cmd, stdout=PIPE, stderr=PIPE, bufsize=1, universal_newlines=True, env=self.env, cwd=self.cwd, start_new_session=True) self.process = p # Prevent blocking of stdout/err. Via https://stackoverflow.com/a/7730201/3983708 os.set_blocking(p.stdout.fileno(), False) os.set_blocking(p.stderr.fileno(), False) def read_async(fd): try: return fd.read() except (IOError, TypeError): return '' stdout = [] while True: # Wait for new output select.select([p.stdout, p.stderr], [], [], 0.1) stdout.append(read_async(p.stdout)) stderr = read_async(p.stderr) if stderr: for line in stderr.split('\n'): try: parsed = json.loads(line) if parsed['type'] == 'log_message': self.app.backup_log_event.emit( f'{parsed["levelname"]}: {parsed["message"]}') level_int = getattr(logging, parsed["levelname"]) logger.log(level_int, parsed["message"]) elif parsed['type'] == 'file_status': self.app.backup_log_event.emit( f'{parsed["path"]} ({parsed["status"]})') elif parsed['type'] == 'archive_progress': msg = ( f"Files: {parsed['nfiles']}, " f"Original: {pretty_bytes(parsed['original_size'])}, " f"Deduplicated: {pretty_bytes(parsed['deduplicated_size'])}, " f"Compressed: {pretty_bytes(parsed['compressed_size'])}" ) self.app.backup_progress_event.emit(msg) except json.decoder.JSONDecodeError: msg = line.strip() if msg: # Log only if there is something to log. self.app.backup_log_event.emit(msg) logger.warning(msg) if p.poll() is not None: time.sleep(0.1) stdout.append(read_async(p.stdout)) break result = { 'params': self.params, 'returncode': self.process.returncode, 'cmd': self.cmd, } stdout = ''.join(stdout) try: result['data'] = json.loads(stdout) except ValueError: result['data'] = stdout log_entry.returncode = p.returncode log_entry.repo_url = self.params.get('repo_url', None) log_entry.save() self.process_result(result) self.finished_event(result) mutex.unlock()
def run_with_timeout(command, timeoutSecs, inputStream = "", combinedOutput=True): """ Runs the given process until it exits or the given time out is reached. Returns a tuple: (bool:timeout, str:stdout, str:stderr, int:exitcode) """ timePassed = 0.0 increment = 0.01 stderrFD, errFile = tempfile.mkstemp() if combinedOutput: stdoutFD, outFile = stderrFD, errFile else: stdoutFD, outFile = tempfile.mkstemp() process = Popen(command, shell=True, stdin=PIPE, stdout=stdoutFD, stderr=stderrFD, close_fds=False) if process == None: print "Could not create process" sys.exit(1) try: if inputStream != "": for line in inputStream: process.stdin.write(line.strip() + '\n') process.stdin.flush() process.stdin.close() while True: status = process.poll() if status != None: # Process terminated succesfully. stdoutSize = os.lseek(stdoutFD, 0, 2) stderrSize = os.lseek(stderrFD, 0, 2) os.lseek(stdoutFD, 0, 0) os.lseek(stderrFD, 0, 0) stdoutContents = os.read(stdoutFD, stdoutSize) os.close(stdoutFD) os.remove(outFile) if not combinedOutput: stderrContents = os.read(stderrFD, stderrSize) os.close(stderrFD) os.remove(errFile) else: stderrContents = stdoutContents return (False, stdoutContents, stderrContents, process.returncode) if timePassed < timeoutSecs: time.sleep(increment) timePassed = timePassed + increment else: # time out, kill the process. stdoutSize = os.lseek(stdoutFD, 0, 2) stderrSize = os.lseek(stderrFD, 0, 2) os.lseek(stdoutFD, 0, 0) stdoutContents = os.read(stdoutFD, stdoutSize) os.close(stdoutFD) os.remove(outFile) if not combinedOutput: os.lseek(stderrFD, 0, 0) stderrContents = os.read(stderrFD, stderrSize) os.close(stderrFD) os.remove(errFile) else: stderrContents = stdoutContents os.kill(process.pid, signal.SIGTSTP) return (True, stdoutContents, stderrContents, process.returncode) except Exception, e: # if something threw exception (e.g. ctrl-c) os.kill(process.pid, signal.SIGTSTP) try: # time out, kill the process. # time out, kill the process. stdoutSize = os.lseek(stdoutFD, 0, 2) stderrSize = os.lseek(stderrFD, 0, 2) os.lseek(stdoutFD, 0, 0) stdoutContents = os.read(stdoutFD, stdoutSize) os.close(stdoutFD) os.remove(outFile) if not combinedOutput: os.lseek(stderrFD, 0, 0) stderrContents = os.read(stderrFD, stderrSize) else: os.close(stderrFD) os.remove(errFile) stderrContents = stdoutContents os.kill(process.pid, signal.SIGTSTP) except: pass return (False, stdoutContents, stderrContents, process.returncode)
def gradeGroup(group_dir, question_ID): #student_question_ID = student_ID + question_ID os.chdir("./Groups/" + group_dir) #----------------------------------------------------- # Edit the .bochsrc-no-gui, replace the partport1 line to this question timestamped output file # We do this to make sure that Autograde doesn't mistakenly take the same file for the next question #----------------------------------------------------- # fraction of a second time stamp QTimestamp = str(time.time()).replace(".", "") # This question specific student_stdout file name student_stdout_default = "student_stdout" student_stdout_Q_specific = student_stdout_default + "_" + QTimestamp bochsrcDefault = open(os.getcwd() + "/.bochsrc-no-gui", 'r') bochsrcAutograde = open(os.getcwd() + "/.bochsrc_autograde_" + QTimestamp, 'w') for line in bochsrcDefault: bochsrcAutograde.write( line.replace(student_stdout_default, student_stdout_Q_specific)) bochsrcDefault.close() bochsrcAutograde.close() #----------------------------------------------------- # TODO: Improvment: If a compilation error occurs to a student folder, let the script save the output # of "make" into a disk file (just like it does for Bochs output) # According to the number of cores in your machine, you can change the -j number to compile faster # example: I use -j4 on my quad core machine makeProc = subprocess.Popen("make -j2", shell=True, stdout=PIPE, stderr=PIPE) #makeProc.wait() makeProc.communicate( ) #wait() caused the deadlock with stdout, err = PIPE, online documentation warns against that, and suggested the use of communicate() question_grade = -100 if (os.path.isfile(os.getcwd() + "/obj/kern/bochs.img") == True): myCommand = 'bochs-no-gui -q -f .bochsrc_autograde_' + QTimestamp #proc = Popen(myCommand, shell=True, stdout=PIPE, stderr=PIPE, stdin=PIPE, preexec_fn=os.setsid) #si = subprocess.STARTUPINFO() #si.dwFlags |= subprocess.STARTF_USESTDHANDLES #proc = Popen(myCommand, shell=True, startupinfo=si, stdout=PIPE, stderr=STDOUT) #shell=True, stdout=PIPE, stderr=PIPE, stdin=PIPE) proc = Popen("exec " + myCommand, shell=True, stdout=PIPE, stderr=PIPE, stdin=PIPE) #===== WARNING: Unix only work-around: non-blocking streams # default read() and readline() in Python block until a # valid output is available. # This makes it difficult to stop the bochs process # when we reach the FOS>, because we will not know when to check # as long as read() is working and doesn't return # # The workaround using fcntl non-blocking flag is Unix only solution # we should find an equivalent library to other platforms, in case # we migrate the course to windows or Mac #import fcntl #file_flags = fcntl.fcntl(proc.stdout.fileno(), fcntl.F_GETFL) #fcntl.fcntl(proc.stdout.fileno(), fcntl.F_SETFL, file_flags | os.O_NONBLOCK) # UPDATE: We didn't use this non-blocking work-around, and we returned # to using the default blocking read(1), but we made sure # to always check for the FOS> after every character read # from the stream. if for some unusual reason the FOS> # prompt isn't printed, we will have to manually close the # bochs window to allow the script to continue working. # - JUST MAKE SURE to run the script with python - u (unbuffered) #============= stream_data = "" question_grade = 0 #A closure (inner function) that I will use later to terminate any stucking threads or processes # and finialize remainig resources def finializeAndCleanUp(): try: if (proc.poll() is None): #kill the Bochs process if not already killed proc.kill() #os.kill(proc.pid, signal.SIGKILL) thefile.close() except Exception, msg: print msg try: startTimeOfQuestion = datetime.now() while os.path.isfile(os.getcwd() + "/" + student_stdout_Q_specific) == False: #keep waiting till we find the new student_stdout file generated by Bochs if ((datetime.now() - startTimeOfQuestion).seconds > 40): print "Timeout, no console output from Bochs" shutil.rmtree('./obj', ignore_errors=True) os.chdir("../../") if (proc.poll() is None ): #kill the Bochs process if not already killed proc.kill() return -100, (datetime.now() - startTimeOfQuestion).seconds time.sleep(0.001) #TODO: Consider removing the try catch statements as a running time optimization thefile = open(os.getcwd() + "/" + student_stdout_Q_specific, "r") #+ "/Students"+"/"+student_dir+ while True: timeNow = datetime.now() if ((timeNow - startTimeOfQuestion).seconds > 60): question_grade = -6 finializeAndCleanUp() print "Timeout" break try: #TODO: Revise this part to enable time out for project correction #def signal_handler(signum, frame): # raise Exception("Timed out!") #signal.signal(signal.SIGALRM, signal_handler) #signal.alarm(60*1) # 1 Min #try: # chunk = thefile.read(1) #except Exception, msg: # question_grade = -6 # finializeAndCleanUp() # print msg # break #time.sleep(0.00005) #this slowed down the loop consideralby, taking about 5 seconds chunk = thefile.read(1) if not chunk: continue #else: # sys.stdout.write(chunk) # sys.stdout.flush() stream_data += chunk #print stream_data if "InitialWSError1" in stream_data: question_grade = -4 finializeAndCleanUp() break elif "InitialWSError2" in stream_data: question_grade = -5 finializeAndCleanUp() break elif "[EVAL_FINAL]panic" in stream_data: if tests[ question_ID] == 'tia' and "kernel [EVAL_FINAL]panic" in stream_data: question_grade = 1 finializeAndCleanUp() break else: question_grade = -7 #Panic Case finializeAndCleanUp() break elif stream_data.count('!! FCIS says HELLO !!') > 1: question_grade = -3 #Restart Case finializeAndCleanUp() break elif outputs[ question_ID] == "TESTING MULTIPLE TIMES" and stream_data.count( '[AUTO_GR@DING]Congratulations!! The array is sorted correctly' ) == 3: question_grade = 1 finializeAndCleanUp() break elif outputs[ question_ID] == "TESTING MULTIPLE TIMES S2" and "[AUTO_GR@DING]Fibonacci #30 = 1346269" in stream_data and stream_data.count( '[AUTO_GR@DING]Congratulations!! The array is sorted correctly' ) == 4: question_grade = 1 finializeAndCleanUp() break elif outputs[ question_ID] == "TESTING MULTIPLE TIMES S3" and stream_data.count( '[AUTO_GR@DING]Congratulations!! The array is sorted correctly' ) == 2: question_grade = 1 finializeAndCleanUp() break elif outputs[ question_ID] == "TESTING MULTIPLE TIMES S4" and stream_data.count( '[AUTO_GR@DING]Congratulations!! The array is sorted correctly' ) == 1: question_grade = 1 finializeAndCleanUp() break elif outputs[ question_ID] == "TESTING ENVFREE SC 2" and stream_data.count( '[AUTO_GR@DING]Congratulations!! test Scenario 2 completed successfully.' ) == 1 and stream_data.count( '[AUTO_GR@DING]Congratulations!! The array is sorted correctly' ) == 2: question_grade = 1 finializeAndCleanUp() break elif outputs[question_ID] in stream_data: question_grade = 1 finializeAndCleanUp() break elif "FOS>" in stream_data: finializeAndCleanUp() break elif proc.poll( ) is not None: #if process is finished, break the loop finializeAndCleanUp() break #elif chunk is None: #break #elif chunk == "": #break except IOError: if proc.poll() is not None: #print "exiting loop, io error" finializeAndCleanUp() break except Exception, msg: #print msg if proc.poll() is not None: #print "exiting loop" finializeAndCleanUp() break except IOError, msg: #print "exiting, io error opening student_stdout_..." print msg finializeAndCleanUp()
def shebang(self, line, cell): """Run a cell via a shell command The `%%script` line is like the #! line of script, specifying a program (bash, perl, ruby, etc.) with which to run. The rest of the cell is run by that program. Examples -------- :: In [1]: %%script bash ...: for i in 1 2 3; do ...: echo $i ...: done 1 2 3 """ argv = arg_split(line, posix=not sys.platform.startswith('win')) args, cmd = self.shebang.parser.parse_known_args(argv) try: p = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE) except OSError as e: if e.errno == errno.ENOENT: print("Couldn't find program: %r" % cmd[0]) return else: raise if not cell.endswith('\n'): cell += '\n' cell = cell.encode('utf8', 'replace') if args.bg: self.bg_processes.append(p) self._gc_bg_processes() if args.out: self.shell.user_ns[args.out] = p.stdout if args.err: self.shell.user_ns[args.err] = p.stderr self.job_manager.new(self._run_script, p, cell, daemon=True) if args.proc: self.shell.user_ns[args.proc] = p return try: out, err = p.communicate(cell) except KeyboardInterrupt: try: p.send_signal(signal.SIGINT) time.sleep(0.1) if p.poll() is not None: print("Process is interrupted.") return p.terminate() time.sleep(0.1) if p.poll() is not None: print("Process is terminated.") return p.kill() print("Process is killed.") except OSError: pass except Exception as e: print("Error while terminating subprocess (pid=%i): %s" \ % (p.pid, e)) return out = py3compat.bytes_to_str(out) err = py3compat.bytes_to_str(err) if args.out: self.shell.user_ns[args.out] = out else: sys.stdout.write(out) sys.stdout.flush() if args.err: self.shell.user_ns[args.err] = err else: sys.stderr.write(err) sys.stderr.flush()
class FirefoxBinary(object): NO_FOCUS_LIBRARY_NAME = "x_ignore_nofocus.so" def __init__(self, firefox_path=None, log_file=None): """ Creates a new instance of Firefox binary. :Args: - firefox_path - Path to the Firefox executable. By default, it will be detected from the standard locations. - log_file - A file object to redirect the firefox process output to. It can be sys.stdout. Please note that with parallel run the output won't be synchronous. By default, it will be redirected to /dev/null. """ self._start_cmd = firefox_path # We used to default to subprocess.PIPE instead of /dev/null, but after # a while the pipe would fill up and Firefox would freeze. self._log_file = log_file or open(os.devnull, "wb") self.command_line = None if self._start_cmd is None: self._start_cmd = self._get_firefox_start_cmd() if not self._start_cmd.strip(): raise WebDriverException( "Failed to find firefox binary. You can set it by specifying " "the path to 'firefox_binary':\n\nfrom " "selenium.webdriver.firefox.firefox_binary import " "FirefoxBinary\n\nbinary = " "FirefoxBinary('/path/to/binary')\ndrive = " "webdriver.Firefox(firefox_binary=binary)") # Rather than modifying the environment of the calling Python process # copy it and modify as needed. self._firefox_env = os.environ.copy() self._firefox_env["MOZ_CRASHREPORTER_DISABLE"] = "1" self._firefox_env["MOZ_NO_REMOTE"] = "1" self._firefox_env["NO_EM_RESTART"] = "1" def add_command_line_options(self, *args): self.command_line = args def launch_browser(self, profile, timeout=30): """Launches the browser for the given profile name. It is assumed the profile already exists. """ self.profile = profile self._start_from_profile_path(self.profile.path) self._wait_until_connectable(timeout=timeout) def kill(self): """Kill the browser. This is useful when the browser is stuck. """ if self.process: self.process.kill() self.process.wait() def _start_from_profile_path(self, path): self._firefox_env["XRE_PROFILE_PATH"] = path if platform.system().lower() == 'linux': self._modify_link_library_path() command = [self._start_cmd, "-foreground"] if self.command_line is not None: for cli in self.command_line: command.append(cli) self.process = Popen( command, stdout=self._log_file, stderr=STDOUT, env=self._firefox_env) def _wait_until_connectable(self, timeout=30): """Blocks until the extension is connectable in the firefox.""" count = 0 while not utils.is_connectable(self.profile.port): if self.process.poll() is not None: # Browser has exited raise WebDriverException( "The browser appears to have exited " "before we could connect. If you specified a log_file in " "the FirefoxBinary constructor, check it for details.") if count >= timeout: self.kill() raise WebDriverException( "Can't load the profile. Possible firefox version mismatch. " "You must use GeckoDriver instead for Firefox 48+. Profile " "Dir: %s If you specified a log_file in the " "FirefoxBinary constructor, check it for details." % (self.profile.path)) count += 1 time.sleep(1) return True def _find_exe_in_registry(self): try: from _winreg import OpenKey, QueryValue, HKEY_LOCAL_MACHINE, HKEY_CURRENT_USER except ImportError: from winreg import OpenKey, QueryValue, HKEY_LOCAL_MACHINE, HKEY_CURRENT_USER import shlex keys = (r"SOFTWARE\Classes\FirefoxHTML\shell\open\command", r"SOFTWARE\Classes\Applications\firefox.exe\shell\open\command") command = "" for path in keys: try: key = OpenKey(HKEY_LOCAL_MACHINE, path) command = QueryValue(key, "") break except OSError: try: key = OpenKey(HKEY_CURRENT_USER, path) command = QueryValue(key, "") break except OSError: pass else: return "" if not command: return "" return shlex.split(command)[0] def _get_firefox_start_cmd(self): """Return the command to start firefox.""" start_cmd = "" if platform.system() == "Darwin": start_cmd = "/Applications/Firefox.app/Contents/MacOS/firefox-bin" # fallback to homebrew installation for mac users if not os.path.exists(start_cmd): start_cmd = os.path.expanduser("~") + start_cmd elif platform.system() == "Windows": start_cmd = (self._find_exe_in_registry() or self._default_windows_location()) elif platform.system() == 'Java' and os._name == 'nt': start_cmd = self._default_windows_location() else: for ffname in ["firefox", "iceweasel"]: start_cmd = self.which(ffname) if start_cmd is not None: break else: # couldn't find firefox on the system path raise RuntimeError( "Could not find firefox in your system PATH." + " Please specify the firefox binary location or install firefox") return start_cmd def _default_windows_location(self): program_files = [os.getenv("PROGRAMFILES", r"C:\Program Files"), os.getenv("PROGRAMFILES(X86)", r"C:\Program Files (x86)")] for path in program_files: binary_path = os.path.join(path, r"Mozilla Firefox\firefox.exe") if os.access(binary_path, os.X_OK): return binary_path return "" def _modify_link_library_path(self): existing_ld_lib_path = os.environ.get('LD_LIBRARY_PATH', '') new_ld_lib_path = self._extract_and_check( self.profile, self.NO_FOCUS_LIBRARY_NAME, "x86", "amd64") new_ld_lib_path += existing_ld_lib_path self._firefox_env["LD_LIBRARY_PATH"] = new_ld_lib_path self._firefox_env['LD_PRELOAD'] = self.NO_FOCUS_LIBRARY_NAME def _extract_and_check(self, profile, no_focus_so_name, x86, amd64): paths = [x86, amd64] built_path = "" for path in paths: library_path = os.path.join(profile.path, path) if not os.path.exists(library_path): os.makedirs(library_path) import shutil shutil.copy(os.path.join( os.path.dirname(__file__), path, self.NO_FOCUS_LIBRARY_NAME), library_path) built_path += library_path + ":" return built_path def which(self, fname): """Returns the fully qualified path by searching Path of the given name""" for pe in os.environ['PATH'].split(os.pathsep): checkname = os.path.join(pe, fname) if os.access(checkname, os.X_OK) and not os.path.isdir(checkname): return checkname return None
def network(s, **kwargs): """pysimm.apps.zeopp.network Perform 1. Pore diameters; 2. Channel identification and dimensionality; 3. Surface area; 4. Accessible volume; 5. Pore size distribution calculation using zeo++ v2.2 with options to do 6. Probe-occupiable volume; 7. Stochastic ray tracing; 8. Blocking spheres; 9. Distance grids; 10. Structure analysis Args: s: pysimm System object or filename of file in CSSR | CUC | V1 | CIF format atype_name: True to use atom type as atom name (usually need radii and mass info), False to use atom element radii: file name that contain atom radii data (rad.rad) mass: file name that contain atom mass data (mass.mass) probe_radius: radius of a probe used in sampling of surface (1.2 A) chan_radius: radius of a probe used to determine accessibility of void space (1.2 A) num_samples: number of Monte Carlo samples per unit cell (50000) option to include in the simulation: set True to activate ha: default=True, for using high accuracy, res: default=True, for diameters of the largest included sphere, the largest free sphere and the largest included sphere along free sphere path chan: default=True, for channel systems characterized by dimensionality as well as Di, Df and Dif sa: default=True, for surface area accessible to a spherical probe, characterized by accessible surface area (ASA) and non-accessible surface area (NASA) vol: default=True, for accessible volume (AV) and non-accessible volume (NAV) volpo: default=False, for accessible proce-occupiable volume (POAV) and non-accessible probe-occupiable volume (PONAV) psd: default=True, for the "deriviative distribution" (change of AV w.r.t probe size) reported in the histogram file with 1000 bins of size of 0.1 Ang ray_atom: default=False block: default=False extra: user provided options, such as -gridG, -gridBOV, -strinfo, -oms, etc. ZEOpp_EXEC: path to zeo++ executable (network) Returns: None """ global ZEOpp_EXEC if ZEOpp_EXEC is None: print('Please specify the environment variable ' 'ZEOpp_EXEC' ' that points to ' 'zeo++ executable (network)') exit(1) probe_radius = kwargs.get('probe_radius', 1.2) chan_radius = kwargs.get('chan_radius', 1.2) num_samples = kwargs.get('num_samples', 50000) atype_name = kwargs.get('atype_name', False) ha = kwargs.get('ha', True) res = kwargs.get('res', True) chan = kwargs.get('chan', True) sa = kwargs.get('sa', True) vol = kwargs.get('vol', True) psd = kwargs.get('psd', True) volpo = kwargs.get('volpo', False) ray_atom = kwargs.get('ray_atom', False) block = kwargs.get('block', False) extra = kwargs.get('extra') nanohub = kwargs.get('nanohub') if isinstance(s, system.System): if atype_name: s.write_cssr('zeopp_data.cssr', aname=1) else: s.write_cssr('zeopp_data.cssr') input_file = 'zeopp_data.cssr' elif isinstance(s, str): input_file = s args = ZEOpp_EXEC if 'radii' in kwargs.keys(): args += ' -r ' + kwargs.get('radii') if 'mass' in kwargs.keys(): args += ' -mass ' + kwargs.get('mass') if ha: args += ' -ha' if res: args += ' -res' if chan: args += ' -chan ' + str(probe_radius) if sa: args += ' -sa ' + str(chan_radius) + ' ' + str( probe_radius) + ' ' + str(num_samples) if vol: args += ' -vol ' + str(chan_radius) + ' ' + str( probe_radius) + ' ' + str(num_samples) if psd: args += ' -psd ' + str(chan_radius) + ' ' + str( probe_radius) + ' ' + str(num_samples) if volpo: args += ' -volpo ' + str(chan_radius) + ' ' + str( probe_radius) + ' ' + str(num_samples) if ray_atom: args += ' -ray_atom ' + str(chan_radius) + ' ' + str( probe_radius) + ' ' + str(num_samples) if block: args += ' -block ' + str(probe_radius) + ' ' + str(num_samples) if extra: args += ' ' + extra args += ' ' + input_file arg_list = shlex.split(args) print('%s: starting simulation using zeo++' % strftime('%H:%M:%S')) if nanohub: print('%s: sending zeo++ simulation to computer cluster' % strftime('%H:%M:%S')) sys.stdout.flush() cmd = ('submit -n 1 -w %s ' % (24 * 60)) + ZEOpp_EXEC + args cmd = shlex.split(cmd) exit_status, stdo, stde = RapptureExec(cmd) else: p = Popen(arg_list, stdin=PIPE, stdout=PIPE, stderr=PIPE) while True: stout = p.stdout.readline() if stout == '' and p.poll() is not None: break if stout: print(stout.strip()) # print(stout) sterr = p.stderr.readlines() print(sterr) print('%s: zeo++ simulation successful' % strftime('%H:%M:%S'))
def execute_code(self, event=None): if not getattr(self.shared, "src_image", None): self.execblink("No image", "#ffff00") self.status("No image given") return if event != "RUN": thr = Thread(target=self.execute_code, args=("RUN", )) thr.start() return self.status("Starting...") self.profiler_canvas.delete("top-text") self.profiler_canvas.delete("pieslice") self.update() self.shared.profiler_info = {} code = self.codeblock._text.get('1.0', 'end') tempcode = tempfile.mkstemp('.py', 'detcher-')[1] with open(tempcode, 'w') as fd: fd.write(code) self.status("Preparing...") tempimg = tempfile.mkstemp('.png', 'detcher-')[1] self.shared.src_image.save(tempimg) self.execbutton.config(text="Kill", fg="#ff0000", state="normal") self.status("Running...") proc = Popen([sys.executable, "-u", "-B", sys.argv[0], "--IS-HELPER"], stdin=PIPE, stdout=PIPE) self.shared.proc = proc proc.stdin.write(tempcode.encode() + b"\n") proc.stdin.write(tempimg.encode() + b"\n") proc.stdin.write(f"{self.shared.seed}\n".encode()) proc.stdin.flush() err = [None, None] out, tmpdir = None, None line = b'-- -- --' while proc.poll() is None or len(line): line = proc.stdout.readline() if not line: continue if proc.poll(): print("[INFO] Process finished but buffer is not empty") ln = line.decode().rstrip() if ln.startswith("\x1b"): cmd = ln.lstrip("\x1b") params = cmd.split(":") print(f">> {cmd}") name = params[0] if name == "OKDONE": out = ":".join(params[1:]) if name == "DR": ok, name, cpu, real = params[1:] cpu, real = map(float, [cpu, real]) self.after_call(name, [cpu, real]) if name == "ERR": err[0] = ":".join(params[1:]) if name == "ERRF": err[1] = ":".join(params[1:]) if name == "ERR_NOTB": err = [":".join(params[1:]), None] if name == "TMP": tmpdir = ":".join(params[1:]) if name == "DBG": self.status(":".join(params[1:]), "#7878fa") else: print(f"[SUB][STDOUT]: {ln}") self.status(f"[sub] {ln.rstrip()}", "#7878fa") proc.wait() exitcode = proc.poll() print(f"[SUB] -> {exitcode}") self.shared.proc = None if exitcode: short_e, e_file = err trace = "-- no trace --" if e_file: with open(e_file, "r") as fd: trace = fd.read() rm_file(e_file) self.execblink("Terminated" if exitcode < 0 else "Error", "#ff0000") self.status(f"{short_e or 'Terminated'}", "#ff0000") print(trace) self.shared.proc = None if tmpdir: shutil.rmtree(tmpdir) return self.status("Loading image...") self.shared.out_image = Image.open(out).copy() self.status("Cleaning files...") rm_file(tempcode) rm_file(tempimg) if tmpdir: shutil.rmtree(tmpdir) self.status("Updating...") self.update_image() self.execbutton.config(text="Execute", fg="#78fa78") self.status("Done.")
def lcall(cmd, logger, outlvl=logging.INFO, errlvl=logging.ERROR, timeout=None, **kwargs): """ Variant of subprocess.call that accepts a logger instead of stdout/stderr, and logs stdout messages via logger.debug and stderr messages via logger.error. """ start = time.time() if "close_fds" not in kwargs: kwargs["close_fds"] = close_fds os.environ["PYTHONIOENCODING"] = "UTF-8" rout, wout = os.pipe() rerr, werr = os.pipe() proc = Popen(cmd, stdout=wout, stderr=werr, **kwargs) log_level = {rout: outlvl, rerr: errlvl} pending = {rout: "", rerr: ""} terminated = False killed = False def check_io(): logged = 0 rlist, _, xlist = select.select([rout, rerr], [], [], 0.2) if xlist: return logged for io in rlist: buff = os.read(io, 32768) buff = bdecode(buff) if six.PY2: buff = buff.decode("utf8") if buff in ('', b''): continue buff = pending[io] + buff while True: l = buff.split("\n", 1) if len(l) == 1: pending[io] = l[0] break line, buff = l if logger: logger.log(log_level[io], "| " + line) elif log_level[io] < logging.ERROR: print(line) else: print(line, file=sys.stderr) logged += 1 return logged # keep checking stdout/stderr until the proc exits while proc.poll() is None: check_io() ellapsed = time.time() - start if timeout and ellapsed > timeout: if not terminated: if logger: logger.error( "execution timeout (%.1f seconds). send SIGTERM." % timeout) else: print("execution timeout (%.1f seconds). send SIGTERM." % timeout, file=sys.stderr) proc.terminate() terminated = True elif not killed and ellapsed > timeout * 2: if logger: logger.error( "SIGTERM handling timeout (%.1f seconds). send SIGKILL." % timeout) else: print( "SIGTERM handling timeout (%.1f seconds). send SIGKILL." % timeout, file=sys.stderr) proc.kill() killed = True while True: # check again to catch anything after the process exits logged = check_io() if logged == 0: break for io in rout, rerr: line = pending[io] if line: if logger: logger.log(log_level[io], "| " + line) elif log_level[io] < logging.ERROR: print(line) else: print(line, file=sys.stderr) os.close(rout) os.close(rerr) os.close(wout) os.close(werr) return proc.returncode
def start(self, cb=None): def which(program, paths): def is_exe(fpath): return os.path.exists(fpath) and os.access(fpath, os.X_OK) for path in paths: if not os.path.isdir(path): continue exe_file = os.path.join(u(path, fs_encoding), program) if is_exe(exe_file): return file_quote(exe_file) return None def convert_environment_variables(env): """ This function is use to convert environment variable to string because environment variable must be string in popen :param env: Dict of environment variable :return: Encoded environment variable as string """ encoding = sys.getdefaultencoding() if encoding is None or encoding == 'ascii': encoding = 'utf-8' temp_env = dict() for key, value in env.items(): if not isinstance(key, str): key = key.encode(encoding) if not isinstance(value, str): value = value.encode(encoding) temp_env[key] = value return temp_env if self.stime is not None: if self.etime is None: raise Exception(_('The process has already been started.')) raise Exception( _('The process has already finished and cannot be restarted.')) executor = file_quote( os.path.join(os.path.dirname(u(__file__)), u'process_executor.py')) paths = os.environ['PATH'].split(os.pathsep) interpreter = None if os.name == 'nt': paths.insert(0, os.path.join(u(sys.prefix), u'Scripts')) paths.insert(0, u(sys.prefix)) interpreter = which(u'pythonw.exe', paths) if interpreter is None: interpreter = which(u'python.exe', paths) if interpreter is None and current_app.PGADMIN_RUNTIME: # We've faced an issue with Windows 2008 R2 (x86) regarding, # not honouring the environment variables set under the Qt # (e.g. runtime), and also setting PYTHONHOME same as # sys.executable (i.e. pgAdmin4.exe). # # As we know, we're running it under the runtime, we can assume # that 'venv' directory will be available outside of 'bin' # directory. # # We would try out luck to find python executable based on that # assumptions. bin_path = os.path.dirname(sys.executable) venv = os.path.realpath(os.path.join(bin_path, u'..\\venv')) interpreter = which(u'pythonw.exe', [venv]) if interpreter is None: interpreter = which(u'pythonw.exe', [venv]) if interpreter is not None: # Our assumptions are proven right. # Let's append the 'bin' directory to the PATH environment # variable. And, also set PYTHONHOME environment variable # to 'venv' directory. os.environ['PATH'] = bin_path + ';' + os.environ['PATH'] os.environ['PYTHONHOME'] = venv else: # Let's not use sys.prefix in runtime. # 'sys.prefix' is not identified on *nix systems for some unknown # reason, while running under the runtime. # We're already adding '<installation path>/pgAdmin 4/venv/bin' # directory in the PATH environment variable. Hence - it will # anyway be the redundant value in paths. if not current_app.PGADMIN_RUNTIME: paths.insert(0, os.path.join(u(sys.prefix), u'bin')) python_binary_name = 'python{0}'.format(sys.version_info[0]) \ if sys.version_info[0] >= 3 else 'python' interpreter = which(u(python_binary_name), paths) p = None cmd = [ interpreter if interpreter is not None else 'python', executor, self.cmd ] cmd.extend(self.args) current_app.logger.info( u"Executing the process executor with the arguments: %s", str(cmd)) # Make a copy of environment, and add new variables to support env = os.environ.copy() env['PROCID'] = self.id env['OUTDIR'] = self.log_dir env['PGA_BGP_FOREGROUND'] = "1" if self.env: env.update(self.env) if cb is not None: cb(env) if os.name == 'nt': DETACHED_PROCESS = 0x00000008 from subprocess import CREATE_NEW_PROCESS_GROUP # We need to redirect the standard input, standard output, and # standard error to devnull in order to allow it start in detached # mode on stdout = os.devnull stderr = stdout stdin = open(os.devnull, "r") stdout = open(stdout, "a") stderr = open(stderr, "a") p = Popen(cmd, close_fds=False, env=env, stdout=stdout.fileno(), stderr=stderr.fileno(), stdin=stdin.fileno(), creationflags=(CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS)) else: def preexec_function(): import signal # Detaching from the parent process group os.setpgrp() # Explicitly ignoring signals in the child process signal.signal(signal.SIGINT, signal.SIG_IGN) # if in debug mode, wait for process to complete and # get the stdout and stderr of popen. if config.CONSOLE_LOG_LEVEL <= logging.DEBUG: p = Popen(cmd, close_fds=True, stdout=PIPE, stderr=PIPE, stdin=None, preexec_fn=preexec_function, env=env) output, errors = p.communicate() output = output.decode() \ if hasattr(output, 'decode') else output errors = errors.decode() \ if hasattr(errors, 'decode') else errors current_app.logger.debug( 'Process Watcher Out:{0}'.format(output)) current_app.logger.debug( 'Process Watcher Err:{0}'.format(errors)) else: p = Popen(cmd, close_fds=True, stdout=None, stderr=None, stdin=None, preexec_fn=preexec_function, env=env) self.ecode = p.poll() # Execution completed immediately. # Process executor cannot update the status, if it was not able to # start properly. if self.ecode is not None and self.ecode != 0: # There is no way to find out the error message from this process # as standard output, and standard error were redirected to # devnull. p = Process.query.filter_by(pid=self.id, user_id=current_user.id).first() p.start_time = p.end_time = get_current_time() if not p.exit_code: p.exit_code = self.ecode p.process_state = PROCESS_FINISHED db.session.commit() else: # Update the process state to "Started" p = Process.query.filter_by(pid=self.id, user_id=current_user.id).first() p.process_state = PROCESS_STARTED db.session.commit()
def check_coap_client(): p = Popen('coap-client', stdout=PIPE, stderr=PIPE, shell=True) time.sleep(0.05) if p.poll() == 127: print('coap-client not install. Please follow instructions from the README file') sys.exit(1)
def Server (args): # FIXME: this is convenience hack, one should decide either args # is a list or a string: if type (args) == type (""): args = shlex.split (args) # Root process such as naive backup can still enter this directory # and read/write to the fifo pipes: tmp = mkdtemp() # Just one possible choice of names: inp = os.path.join (tmp, "%inp") out = os.path.join (tmp, "%out") os.mkfifo (inp) os.mkfifo (out) # args[] is supposed to include the executable and flags, the # subcommand and two more positional arguments will be # appended. The "server" subcommand must be the first positional # argument in order to be interpreted as such: proc = Popen (args + ["server", inp, out]) # This function takes input text and returns output of the # subprocess, both communicated via the fifos: def server (x): try: with open (inp, "w") as f: f.write (x) with open (out, "r") as f: y = f.read () except KeyboardInterrupt as e: # # FIXME: If the IO is screwed, we will not be able to # regularly shut down the daemon. Kill it with SIGTERM. # Some minimal testing showed that proc.poll() will return # something trueish afterwards. Also if you catch a # generic Exception here the chances are slim that you # will catch an asynchrounous KeyboardInterrupt here and # now (see StackOverflow). # proc.terminate() proc.wait() raise e return y # This is a valid Func: func = protocol (server) # If an exception happens in the body of the "with" statment we # will note it here: try: # # The control returns to the # # with Server(...) as func: # # statement body: # yield func finally: # # When leaving the "with" statement body execute this. # Executed in any case, even if exceptions such as a # KeyboardInterrupt occur. We do not want zombie daemons and # stray files. # # Tell the daemon process to terminate. This will not work if # user hit C-c while the server() was already doing IO on the # fifos. In this case the process should have been terminated # there. # if not proc.poll(): # Regular shutdown sequence: with open (inp, "w") as f: f.write ("#f") # convention proc.wait () os.unlink (inp) os.unlink (out) os.rmdir (tmp)
class CheckUpdates(base.ThreadPoolText): """Shows number of pending updates in different unix systems""" orientations = base.ORIENTATION_HORIZONTAL defaults = [ ("distro", "Arch", "Name of your distribution"), ("custom_command", None, "Custom shell command for checking updates (counts the lines of the output)"), ("custom_command_modify", (lambda x: x), "Lambda function to modify line count from custom_command"), ("update_interval", 60, "Update interval in seconds."), ('execute', None, 'Command to execute on click'), ("display_format", "Updates: {updates}", "Display format if updates available"), ("colour_no_updates", "ffffff", "Colour when there's no updates."), ("colour_have_updates", "ffffff", "Colour when there are updates."), ("restart_indicator", "", "Indicator to represent reboot is required. (Ubuntu only)"), ("no_update_string", "", "String to display if no updates available") ] def __init__(self, **config): base.ThreadPoolText.__init__(self, "", **config) self.add_defaults(CheckUpdates.defaults) # Helpful to have this as a variable as we can shorten it for testing self.execute_polling_interval = 1 # format: "Distro": ("cmd", "number of lines to subtract from output") self.cmd_dict = {"Arch": ("pacman -Qu", 0), "Arch_checkupdates": ("checkupdates", 0), "Arch_Sup": ("pacman -Sup", 1), "Arch_yay": ("yay -Qu", 0), "Debian": ("apt-show-versions -u -b", 0), "Ubuntu": ("aptitude search ~U", 0), "Fedora": ("dnf list updates -q", 1), "FreeBSD": ("pkg_version -I -l '<'", 0), "Mandriva": ("urpmq --auto-select", 0) } if self.custom_command: # Use custom_command self.cmd = self.custom_command else: # Check if distro name is valid. try: self.cmd = self.cmd_dict[self.distro][0] self.custom_command_modify = (lambda x: x - self.cmd_dict[self.distro][1]) except KeyError: distros = sorted(self.cmd_dict.keys()) logger.error(self.distro + ' is not a valid distro name. ' + 'Use one of the list: ' + str(distros) + '.') self.cmd = None if self.execute: self.add_callbacks({'Button1': self.do_execute}) def _check_updates(self): # type: () -> str try: updates = self.call_process(self.cmd, shell=True) except CalledProcessError: updates = "" num_updates = self.custom_command_modify(len(updates.splitlines())) if num_updates < 0: num_updates = 0 if num_updates == 0: self.layout.colour = self.colour_no_updates return self.no_update_string num_updates = str(num_updates) if self.restart_indicator and os.path.exists('/var/run/reboot-required'): num_updates += self.restart_indicator self.layout.colour = self.colour_have_updates return self.display_format.format(**{"updates": num_updates}) def poll(self): # type: () -> str if not self.cmd: return "N/A" return self._check_updates() def do_execute(self): self._process = Popen(self.execute, shell=True) self.timeout_add(self.execute_polling_interval, self._refresh_count) def _refresh_count(self): if self._process.poll() is None: self.timeout_add(self.execute_polling_interval, self._refresh_count) else: self.timer_setup()
def _read_progress(self, proc: subprocess.Popen, pbar: List[Any], idx: int) -> bytes: """ Update tqdm progress bars according to CmdStan console progress msgs. Poll process to get CmdStan console outputs, check for output lines that start with 'Iteration: '. NOTE: if CmdStan output messages change, this will break. """ pattern = ( r'^Iteration\:\s*(\d+)\s*/\s*(\d+)\s*\[\s*\d+%\s*\]\s*\((\S*)\)$') pattern_compiled = re.compile(pattern, flags=re.IGNORECASE) pbar_warmup, pbar_sampling = pbar num_warmup = pbar_warmup.total num_sampling = pbar_sampling.total count_warmup = 0 count_sampling = 0 stdout = b'' try: # iterate while process is sampling while proc.poll() is None: output = proc.stdout.readline() stdout += output output = output.decode('utf-8').strip() refresh_warmup = True if output.startswith('Iteration'): match = re.search(pattern_compiled, output) if match: # check if pbars need reset if num_warmup is None or num_sampling is None: total_count = int(match.group(2)) if num_warmup is None and num_sampling is None: num_warmup = total_count // 2 num_sampling = total_count - num_warmup pbar_warmup.total = num_warmup pbar_sampling.total = num_sampling elif num_warmup is None: num_warmup = total_count - num_sampling pbar_warmup.total = num_warmup else: num_sampling = total_count - num_warmup pbar_sampling.total = num_sampling # raw_count = warmup + sampling raw_count = int(match.group(1)) if match.group(3).lower() == 'warmup': count, count_warmup = ( raw_count - count_warmup, raw_count, ) pbar_warmup.update(count) elif match.group(3).lower() == 'sampling': # refresh warmup and close the progress bar if refresh_warmup: pbar_warmup.update(num_warmup - count_warmup) pbar_warmup.refresh() pbar_warmup.close() refresh_warmup = False # update values to full count, count_sampling = ( raw_count - num_warmup - count_sampling, raw_count - num_warmup, ) pbar_sampling.update(count) # read and process rest of the stdout if needed warmup_cumulative_count = 0 sampling_cumulative_count = 0 for output in proc.stdout: stdout += output output = output.decode('utf-8').strip() if output.startswith('Iteration'): match = re.search(pattern_compiled, output) if match: # check if pbars need reset if num_warmup is None or num_sampling is None: total_count = int(match.group(2)) if num_warmup is None and num_sampling is None: num_warmup = total_count // 2 num_sampling = total_count - num_warmup pbar_warmup.total = num_warmup pbar_sampling.total = num_sampling elif num_warmup is None: num_warmup = total_count - num_sampling pbar_warmup.total = num_warmup else: num_sampling = total_count - num_warmup pbar_sampling.total = num_sampling # raw_count = warmup + sampling raw_count = int(match.group(1)) if match.group(3).lower() == 'warmup': count, count_warmup = ( raw_count - count_warmup, raw_count, ) warmup_cumulative_count += count elif match.group(3).lower() == 'sampling': count, count_sampling = ( raw_count - num_warmup - count_sampling, raw_count - num_warmup, ) sampling_cumulative_count += count # update warmup pbar if needed if warmup_cumulative_count: pbar_warmup.update(warmup_cumulative_count) pbar_warmup.refresh() # update sampling pbar if needed if sampling_cumulative_count: pbar_sampling.update(sampling_cumulative_count) pbar_sampling.refresh() except Exception as e: self._logger.warning( 'Chain %s: Failed to read the progress on the fly. Error: %s', idx, e, ) # close both pbar pbar_warmup.close() pbar_sampling.close() # return stdout return stdout
class ConfigurableHTTPProxy(Proxy): """Proxy implementation for the default configurable-http-proxy. This is the default proxy implementation for running the nodejs proxy `configurable-http-proxy`. If the proxy should not be run as a subprocess of the Hub, (e.g. in a separate container), set:: c.ConfigurableHTTPProxy.should_start = False """ proxy_process = Any() client = Instance(AsyncHTTPClient, ()) debug = Bool(False, help="Add debug-level logging to the Proxy.", config=True) auth_token = Unicode(help="""The Proxy auth token Loaded from the CONFIGPROXY_AUTH_TOKEN env variable by default. """, ).tag(config=True) check_running_interval = Integer(5, config=True) @default('auth_token') def _auth_token_default(self): token = os.environ.get('CONFIGPROXY_AUTH_TOKEN', None) if not token: self.log.warning('\n'.join([ "", "Generating CONFIGPROXY_AUTH_TOKEN. Restarting the Hub will require restarting the proxy.", "Set CONFIGPROXY_AUTH_TOKEN env or JupyterHub.proxy_auth_token config to avoid this message.", "", ])) token = utils.new_token() return token api_url = Unicode( 'http://127.0.0.1:8001', config=True, help="""The ip (or hostname) of the proxy's API endpoint""") command = Command('configurable-http-proxy', config=True, help="""The command to start the proxy""") @gen.coroutine def start(self): public_server = Server.from_url(self.public_url) api_server = Server.from_url(self.api_url) env = os.environ.copy() env['CONFIGPROXY_AUTH_TOKEN'] = self.auth_token cmd = self.command + [ '--ip', public_server.ip, '--port', str(public_server.port), '--api-ip', api_server.ip, '--api-port', str(api_server.port), '--error-target', url_path_join(self.hub.url, 'error'), ] if self.app.subdomain_host: cmd.append('--host-routing') if self.debug: cmd.extend(['--log-level', 'debug']) if self.ssl_key: cmd.extend(['--ssl-key', self.ssl_key]) if self.ssl_cert: cmd.extend(['--ssl-cert', self.ssl_cert]) if self.app.statsd_host: cmd.extend([ '--statsd-host', self.app.statsd_host, '--statsd-port', str(self.app.statsd_port), '--statsd-prefix', self.app.statsd_prefix + '.chp' ]) # Warn if SSL is not used if ' --ssl' not in ' '.join(cmd): self.log.warning( "Running JupyterHub without SSL." " I hope there is SSL termination happening somewhere else..." ) self.log.info("Starting proxy @ %s", public_server.bind_url) self.log.debug("Proxy cmd: %s", cmd) try: self.proxy_process = Popen(cmd, env=env, start_new_session=True) except FileNotFoundError as e: self.log.error( "Failed to find proxy %r\n" "The proxy can be installed with `npm install -g configurable-http-proxy`" % self.command) raise def _check_process(): status = self.proxy_process.poll() if status is not None: e = RuntimeError("Proxy failed to start with exit code %i" % status) # py2-compatible `raise e from None` e.__cause__ = None raise e for server in (public_server, api_server): for i in range(10): _check_process() try: yield server.wait_up(1) except TimeoutError: continue else: break yield server.wait_up(1) _check_process() self.log.debug("Proxy started and appears to be up") pc = PeriodicCallback(self.check_running, 1e3 * self.check_running_interval) pc.start() def stop(self): self.log.info("Cleaning up proxy[%i]...", self.proxy_process.pid) if self.proxy_process.poll() is None: try: self.proxy_process.terminate() except Exception as e: self.log.error("Failed to terminate proxy process: %s", e) @gen.coroutine def check_running(self): """Check if the proxy is still running""" if self.proxy_process.poll() is None: return self.log.error( "Proxy stopped with exit code %r", 'unknown' if self.proxy_process is None else self.proxy_process.poll()) yield self.start() yield self.restore_routes() def _routespec_to_chp_path(self, routespec): """Turn a routespec into a CHP API path For host-based routing, CHP uses the host as the first path segment. """ path = self.validate_routespec(routespec) # CHP always wants to start with / if not path.startswith('/'): path = '/' + path # BUG: CHP doesn't seem to like trailing slashes on some endpoints (DELETE) if path != '/' and path.endswith('/'): path = path.rstrip('/') return path def _routespec_from_chp_path(self, chp_path): """Turn a CHP route into a route spec In the JSON API, CHP route keys are unescaped, so re-escape them to raw URLs and ensure slashes are in the right places. """ # chp stores routes in unescaped form. # restore escaped-form we created it with. routespec = quote(chp_path, safe='@/') if self.host_routing: # host routes don't start with / routespec = routespec.lstrip('/') # all routes should end with / if not routespec.endswith('/'): routespec = routespec + '/' return routespec def api_request(self, path, method='GET', body=None, client=None): """Make an authenticated API request of the proxy.""" client = client or AsyncHTTPClient() url = url_path_join(self.api_url, 'api/routes', path) if isinstance(body, dict): body = json.dumps(body) self.log.debug("Proxy: Fetching %s %s", method, url) req = HTTPRequest( url, method=method, headers={'Authorization': 'token {}'.format(self.auth_token)}, body=body, ) return client.fetch(req) def add_route(self, routespec, target, data): body = data or {} body['target'] = target body['jupyterhub'] = True path = self._routespec_to_chp_path(routespec) return self.api_request( path, method='POST', body=body, ) def delete_route(self, routespec): path = self._routespec_to_chp_path(routespec) return self.api_request(path, method='DELETE') def _reformat_routespec(self, routespec, chp_data): """Reformat CHP data format to JupyterHub's proxy API.""" target = chp_data.pop('target') chp_data.pop('jupyterhub') return { 'routespec': routespec, 'target': target, 'data': chp_data, } @gen.coroutine def get_all_routes(self, client=None): """Fetch the proxy's routes.""" resp = yield self.api_request('', client=client) chp_routes = json.loads(resp.body.decode('utf8', 'replace')) all_routes = {} for chp_path, chp_data in chp_routes.items(): routespec = self._routespec_from_chp_path(chp_path) if 'jupyterhub' not in chp_data: # exclude routes not associated with JupyterHub self.log.debug("Omitting non-jupyterhub route %r", routespec) continue all_routes[routespec] = self._reformat_routespec( routespec, chp_data) return all_routes
def fastMerge(self): filelist = [] outputfile = self._outputFiles[0].value() for file in self._inputFiles: if file: value = file.value() if type(value).__name__ == 'list': filelist += value else: filelist.append(value) print "Files to Merge: %s" % filelist # First run mergePOOL.exe to get events.pool cmd = ['mergePOOL.exe', '-o', 'events.pool.root'] for file in filelist: cmd.extend(['-i', file]) cmd.extend([ '-e', 'MetaData', '-e', 'MetaDataHdrDataHeaderForm', '-e', 'MetaDataHdrDataHeader', '-e', 'MetaDataHdr' ]) print "Will execute hybrid merge step 1: %s" % cmd p = Popen(cmd, stdout=PIPE, stderr=STDOUT, close_fds=True) while p.poll() is None: line = p.stdout.readline() if line: print "mergePOOL.exe Report: %s" % line.strip() rc = p.returncode print "1st mergePOOL (event data) finished with code %s" % rc if rc == 1: print "mergePOOL.exe finished with unknown status (upgrade your RootFileTools to a newer version)" elif rc != 0: raise TransformError( "mergePOOL.exe (event merge) encountered a problem", error='TRF_MERGEERR') # Second merge with metadata.pool to produce final output cmd = ['mergePOOL.exe', '-o', 'events.pool.root', '-i', outputfile] print "Will execute hybrid merge step 2: %s" % cmd p = Popen(cmd, stdout=PIPE, stderr=STDOUT, close_fds=True) while p.poll() is None: line = p.stdout.readline() if line: print "mergePOOL.exe Report: %s" % line.strip() rc = p.returncode print "2nd mergePOOL (metadata) finished with code %s" % rc if rc == 1: print "mergePOOL.exe finished with unknown status (upgrade your RootFileTools to a newer version) - assuming all is ok" elif rc != 0: raise TransformError( "mergePOOL.exe (final merge) encountered a problem", error='TRF_MERGEERR') # Finish hybrid merge by moving the full file to the final output location shutil.move('events.pool.root', outputfile) # Now fix the metadata, which has been left by POOL as the _stub_ file's metadata # so it has the wrong GUID in the PFC print 'Now fixing metadata in PFC for %s' % outputfile try: check_call(['FCdeletePFN', '-p', outputfile]) correctGUID = None p = Popen(['pool_extractFileIdentifier.py', outputfile], stdout=PIPE, stderr=STDOUT, close_fds=True, bufsize=1) while p.poll() is None: line = p.stdout.readline() words = line.split() if len(words) >= 2 and outputfile in words[1]: correctGUID = words[0] if correctGUID == None or p.returncode != 0: raise TransformError( "pool_extractFileIdentifier.py failed to get merged file GUID", error='TRF_MERGEERR') print 'GUID is %s' % correctGUID check_call([ 'FCregisterPFN', '-p', outputfile, '-t', 'ROOT_All', '-g', correctGUID ]) except CalledProcessError, e: print 'Attempt to fix PFC with new merged file information failed: %s' % e
class SystemSSHTransport(Transport): def __init__( self, host: str = "", port: int = 22, auth_username: str = "", auth_private_key: str = "", auth_password: str = "", auth_strict_key: bool = True, timeout_socket: int = 5, timeout_transport: int = 5, timeout_ops: int = 10, timeout_exit: bool = True, keepalive: bool = False, keepalive_interval: int = 30, keepalive_type: str = "", keepalive_pattern: str = "\005", comms_prompt_pattern: str = r"^[a-z0-9.\-@()/:]{1,32}[#>$]$", comms_return_char: str = "\n", comms_ansi: bool = False, ssh_config_file: str = "", ssh_known_hosts_file: str = "", ) -> None: """ SystemSSHTransport Object Inherit from Transport ABC SSH2Transport <- Transport (ABC) If using this driver, and passing a ssh_config_file (or setting this argument to `True`), all settings in the ssh config file will be superseded by any arguments passed here! SystemSSHTransport *always* prefers public key auth if given the option! If auth_private_key is set in the provided arguments OR if ssh_config_file is passed/True and there is a key for ANY match (i.e. `*` has a key in ssh config file!!), we will use that key! If public key auth fails and a username and password is set (manually or by ssh config file), password auth will be attempted. Note that comms_prompt_pattern, comms_return_char and comms_ansi are only passed here to handle "in channel" authentication required by SystemSSH -- these are assigned to private attributes in this class and ignored after authentication. If you wish to modify these values on a "live" scrapli connection, modify them in the Channel object, i.e. `conn.channel.comms_prompt_pattern`. Additionally timeout_ops is passed and assigned to _timeout_ops to use the same timeout_ops that is used in Channel to decorate the authentication methods here. Args: host: host ip/name to connect to port: port to connect to auth_username: username for authentication auth_private_key: path to private key for authentication auth_password: password for authentication auth_strict_key: True/False to enforce strict key checking (default is True) timeout_socket: timeout for ssh session to start -- this directly maps to ConnectTimeout ssh argument; see `man ssh_config` timeout_transport: timeout for transport in seconds. since system ssh is using popen/pty we can't really set a timeout directly, so this value governs the time timeout decorator for the transport read and write methods timeout_ops: timeout for telnet channel operations in seconds -- this is also the timeout for finding and responding to username and password prompts at initial login. This is assigned to a private attribute and is ignored after authentication is completed. timeout_exit: True/False close transport if timeout encountered. If False and keepalives are in use, keepalives will prevent program from exiting so you should be sure to catch Timeout exceptions and handle them appropriately keepalive: whether or not to try to keep session alive keepalive_interval: interval to use for session keepalives keepalive_type: network|standard -- 'network' sends actual characters over the transport channel. This is useful for network-y type devices that may not support 'standard' keepalive mechanisms. 'standard' is not currently implemented for system ssh keepalive_pattern: pattern to send to keep network channel alive. Default is u'\005' which is equivalent to 'ctrl+e'. This pattern moves cursor to end of the line which should be an innocuous pattern. This will only be entered *if* a lock can be acquired. This is only applicable if using keepalives and if the keepalive type is 'network' comms_prompt_pattern: prompt pattern expected for device, same as the one provided to channel -- system ssh needs to know this to know how to decide if we are properly sending/receiving data -- i.e. we are not stuck at some password prompt or some other failure scenario. If using driver, this should be passed from driver (Scrape, or IOSXE, etc.) to this Transport class. This is assigned to a private attribute and is ignored after authentication is completed. comms_return_char: return character to use on the channel, same as the one provided to channel -- system ssh needs to know this to know what to send so that we can probe the channel to make sure we are authenticated and sending/receiving data. If using driver, this should be passed from driver (Scrape, or IOSXE, etc.) to this Transport class. This is assigned to a private attribute and is ignored after authentication is completed. comms_ansi: True/False strip comms_ansi characters from output; this value is assigned self._comms_ansi and is ignored after authentication. We only need it for transport on the off chance (maybe never?) that username/password prompts contain ansi characters, otherwise "comms_ansi" is really a channel attribute and is treated as such. This is assigned to a private attribute and is ignored after authentication is completed. ssh_config_file: string to path for ssh config file ssh_known_hosts_file: string to path for ssh known hosts file Returns: N/A # noqa: DAR202 Raises: N/A """ super().__init__( host, port, timeout_socket, timeout_transport, timeout_exit, keepalive, keepalive_interval, keepalive_type, keepalive_pattern, ) self.auth_username: str = auth_username self.auth_private_key: str = auth_private_key self.auth_password: str = auth_password self.auth_strict_key: bool = auth_strict_key self._timeout_ops: int = timeout_ops self._comms_prompt_pattern: str = comms_prompt_pattern self._comms_return_char: str = comms_return_char self._comms_ansi: bool = comms_ansi self._process_ssh_config(ssh_config_file) self.ssh_known_hosts_file: str = ssh_known_hosts_file self.session: Union[Popen[bytes], PtyProcess] # pylint: disable=E1136 self.lib_auth_exception = ScrapliAuthenticationFailed self._isauthenticated = False self.open_cmd = ["ssh", self.host] self._build_open_cmd() # create stdin/stdout fd in case we can use pipes for session self._stdin_fd = -1 self._stdout_fd = -1 def _process_ssh_config(self, ssh_config_file: str) -> None: """ Method to parse ssh config file Ensure ssh_config_file is valid (if providing a string path to config file), or resolve config file if passed True. Search config file for any private key, if ANY matching key is found and user has not provided a private key, set `auth_private_key` to the value of the found key. This is because we prefer to use `open_pipes` over `open_pty`! Args: ssh_config_file: string path to ssh config file; passed down from `Scrape`, or the `NetworkDriver` or subclasses of it, in most cases. Returns: N/A # noqa: DAR202 Raises: N/A """ ssh = SSHConfig(ssh_config_file) self.ssh_config_file = ssh.ssh_config_file host_config = ssh.lookup(self.host) if not self.auth_private_key and host_config.identity_file: self.auth_private_key = os.path.expanduser( host_config.identity_file.strip()) def _build_open_cmd(self) -> None: """ Method to craft command to open ssh session Args: N/A Returns: N/A # noqa: DAR202 Raises: N/A """ self.open_cmd.extend(["-p", str(self.port)]) self.open_cmd.extend(["-o", f"ConnectTimeout={self.timeout_socket}"]) if self.auth_private_key: self.open_cmd.extend(["-i", self.auth_private_key]) if self.auth_username: self.open_cmd.extend(["-l", self.auth_username]) if self.auth_strict_key is False: self.open_cmd.extend(["-o", "StrictHostKeyChecking=no"]) self.open_cmd.extend(["-o", "UserKnownHostsFile=/dev/null"]) else: self.open_cmd.extend(["-o", "StrictHostKeyChecking=yes"]) if self.ssh_known_hosts_file: self.open_cmd.extend( ["-o", f"UserKnownHostsFile={self.ssh_known_hosts_file}"]) if self.ssh_config_file: self.open_cmd.extend(["-F", self.ssh_config_file]) else: self.open_cmd.extend(["-F", "/dev/null"]) def open(self) -> None: """ Parent method to open session, authenticate and acquire shell If possible it is preferable to use the `_open_pipes` method, but we can only do this IF we can authenticate with public key authorization (because we don't have to spawn a PTY; if no public key we have to spawn PTY to deal w/ authentication prompts). IF we get a private key provided, use pipes method, otherwise we will just deal with `_open_pty`. `_open_pty` is less preferable because we have to spawn a PTY and cannot as easily tell if SSH authentication is successful. With `_open_pipes` we can read stderr which contains the output from the verbose flag for SSH -- this contains a message that indicates success of SSH auth. In the case of `_open_pty` we have to read from the channel directly like in the case of telnet... so it works, but its just a bit less desirable. Args: N/A Returns: N/A # noqa: DAR202 Raises: ScrapliAuthenticationFailed: if all authentication means fail """ self.session_lock.acquire() # If authenticating with private key prefer to use open pipes # _open_pipes uses subprocess Popen which is preferable to opening a pty # if _open_pipes fails and no password available, raise failure, otherwise try password auth if self.auth_private_key: open_pipes_result = self._open_pipes() if open_pipes_result: return if not self.auth_password or not self.auth_username: msg = ( f"Public key authentication to host {self.host} failed. Missing username or" " password unable to attempt password authentication.") LOG.critical(msg) raise ScrapliAuthenticationFailed(msg) # If public key auth fails or is not configured, open a pty session if not self._open_pty(): msg = f"Authentication to host {self.host} failed" LOG.critical(msg) raise ScrapliAuthenticationFailed(msg) if self.keepalive: self._session_keepalive() def _open_pipes(self) -> bool: """ Private method to open session with subprocess.Popen Args: N/A Returns: bool: True/False session was opened and authenticated Raises: N/A """ # copy the open_cmd as we don't want to update the objects open_cmd until we know we can # authenticate. add verbose output and disable batch mode (disables passphrase/password # queries). If auth is successful update the object open_cmd to represent what was used open_cmd = self.open_cmd.copy() open_cmd.append("-v") open_cmd.extend(["-o", "BatchMode=yes"]) stdout_master_pty, stdout_slave_pty = pty.openpty() stdin_master_pty, stdin_slave_pty = pty.openpty() self.session = Popen( open_cmd, bufsize=0, shell=False, stdin=stdin_slave_pty, stdout=stdout_slave_pty, stderr=PIPE, ) # close the slave fds, don't need them anymore os.close(stdin_slave_pty) os.close(stdout_slave_pty) LOG.debug(f"Session to host {self.host} spawned") try: self._pipes_isauthenticated(self.session) except TimeoutError: # If auth fails, kill the popen session, also need to manually close the stderr pipe # for some reason... unclear why, but w/out this it will hang open if self.session.stderr is not None: stderr_fd = self.session.stderr.fileno() os.close(stderr_fd) self.session.kill() return False LOG.debug(f"Authenticated to host {self.host} with public key") # set stdin/stdout to the new master pty fds self._stdin_fd = stdin_master_pty self._stdout_fd = stdout_master_pty self.open_cmd = open_cmd self.session_lock.release() return True @operation_timeout("_timeout_ops", "Timed out determining if session is authenticated") def _pipes_isauthenticated(self, pipes_session: "PopenBytes") -> bool: """ Private method to check initial authentication when using subprocess.Popen Since we always run ssh with `-v` we can simply check the stderr (where verbose output goes) to see if `Authenticated to [our host]` is in the output. Args: pipes_session: Popen pipes session object Returns: bool: True/False session was authenticated Raises: ScrapliTimeout: if `Operation timed out` in stderr output """ if pipes_session.stderr is None: raise ScrapliTimeout( f"Could not read stderr while connecting to host {self.host}") output = b"" while True: output += pipes_session.stderr.read(65535) if f"Authenticated to {self.host}".encode() in output: self._isauthenticated = True return True if "Operation timed out".encode() in output: raise ScrapliTimeout( f"Timed opening connection to host {self.host}") def _open_pty(self) -> bool: """ Private method to open session with PtyProcess Args: N/A Returns: bool: True/False session was opened and authenticated Raises: N/A """ self.session = PtyProcess.spawn(self.open_cmd) LOG.debug(f"Session to host {self.host} spawned") self.session_lock.release() self._pty_authenticate(self.session) if not self._pty_isauthenticated(self.session): return False LOG.debug(f"Authenticated to host {self.host} with password") return True @operation_timeout("_timeout_ops", "Timed out looking for SSH login password prompt") def _pty_authenticate(self, pty_session: PtyProcess) -> None: """ Private method to check initial authentication when using pty_session Args: pty_session: PtyProcess session object Returns: N/A # noqa: DAR202 Raises: ScrapliAuthenticationFailed: if we receive an EOFError -- this usually indicates that host key checking is enabled and failed. """ self.session_lock.acquire() output = b"" while True: try: output += pty_session.read() except EOFError: msg = f"Failed to open connection to host {self.host}" if b"Host key verification failed" in output: msg = f"Host key verification failed for host {self.host}" elif b"Operation timed out" in output: msg = f"Timed out connecting to host {self.host}" raise ScrapliAuthenticationFailed(msg) if self._comms_ansi: output = strip_ansi(output) if b"password" in output.lower(): LOG.debug("Found password prompt, sending password") pty_session.write(self.auth_password.encode()) pty_session.write(self._comms_return_char.encode()) self.session_lock.release() break @operation_timeout("_timeout_ops", "Timed out determining if session is authenticated") def _pty_isauthenticated(self, pty_session: PtyProcess) -> bool: """ Check if session is authenticated This is very naive -- it only knows if the sub process is alive and has not received an EOF. Beyond that we lock the session and send the return character and re-read the channel. Args: pty_session: PtyProcess session object Returns: bool: True if authenticated, else False Raises: N/A """ LOG.debug( "Attempting to determine if PTY authentication was successful") if pty_session.isalive() and not pty_session.eof(): prompt_pattern = get_prompt_pattern("", self._comms_prompt_pattern) self.session_lock.acquire() pty_session.write(self._comms_return_char.encode()) fd_ready, _, _ = select([pty_session.fd], [], [], 0) if pty_session.fd in fd_ready: output = b"" while True: output += pty_session.read() # we do not need to deal w/ line replacement for the actual output, only for # parsing if a prompt-like thing is at the end of the output output = re.sub(b"\r", b"", output) # always check to see if we should strip ansi here; if we don't handle this we # may raise auth failures for the wrong reason which would be confusing for # users if b"\x1B" in output: output = strip_ansi(output) channel_match = re.search(prompt_pattern, output) if channel_match: self.session_lock.release() self._isauthenticated = True return True if b"password" in output.lower(): # if we see "password" we know auth failed (hopefully in all scenarios!) return False if output: LOG.debug( f"Cannot determine if authenticated, \n\tRead: {repr(output)}" ) self.session_lock.release() return False def close(self) -> None: """ Close session and socket Args: N/A Returns: N/A # noqa: DAR202 Raises: N/A """ self.session_lock.acquire() if isinstance(self.session, Popen): self.session.kill() elif isinstance(self.session, PtyProcess): self.session.kill(1) LOG.debug(f"Channel to host {self.host} closed") self.session_lock.release() def isalive(self) -> bool: """ Check if session is alive and session is authenticated Args: N/A Returns: bool: True if session is alive and session authenticated, else False Raises: N/A """ if isinstance(self.session, Popen): if self.session.poll() is None and self._isauthenticated: return True elif isinstance(self.session, PtyProcess): if self.session.isalive( ) and self._isauthenticated and not self.session.eof(): return True return False @operation_timeout("timeout_transport", "Transport timeout during read operation.") def read(self) -> bytes: """ Read data from the channel Args: N/A Returns: bytes: bytes output as read from channel Raises: N/A """ read_bytes = 65535 if isinstance(self.session, Popen): return os.read(self._stdout_fd, read_bytes) if isinstance(self.session, PtyProcess): return self.session.read(read_bytes) return b"" @operation_timeout("timeout_transport", "Transport timeout during write operation.") def write(self, channel_input: str) -> None: """ Write data to the channel Args: channel_input: string to send to channel Returns: N/A # noqa: DAR202 Raises: N/A """ if isinstance(self.session, Popen): os.write(self._stdin_fd, channel_input.encode()) elif isinstance(self.session, PtyProcess): self.session.write(channel_input.encode()) def set_timeout(self, timeout: Optional[int] = None) -> None: """ Set session timeout Note that this modifies the objects `timeout_transport` value directly as this value is what controls the timeout decorator for read/write methods. This is slightly different behavior from ssh2/paramiko/telnet in that those transports modify the session value and leave the objects `timeout_transport` alone. Args: timeout: timeout in seconds Returns: N/A # noqa: DAR202 Raises: N/A """ if isinstance(timeout, int): set_timeout = timeout else: set_timeout = self.timeout_transport self.timeout_transport = set_timeout def _keepalive_standard(self) -> None: """ Send "out of band" (protocol level) keepalives to devices. Args: N/A Returns: N/A # noqa: DAR202 Raises: NotImplementedError: always, because this is not implemented for telnet """ raise NotImplementedError( "'standard' keepalive mechanism not yet implemented for system.")
class Local(Runner): """ Execute a command on the local system in a subprocess. .. note:: When Invoke itself is executed without a controlling terminal (e.g. when ``sys.stdin`` lacks a useful ``fileno``), it's not possible to present a handle on our PTY to local subprocesses. In such situations, `Local` will fallback to behaving as if ``pty=False`` (on the theory that degraded execution is better than none at all) as well as printing a warning to stderr. To disable this behavior, say ``fallback=False``. """ def __init__(self, context): super(Local, self).__init__(context) # Bookkeeping var for pty use case self.status = None def should_use_pty(self, pty=False, fallback=True): use_pty = False if pty: use_pty = True # TODO: pass in & test in_stream, not sys.stdin if not has_fileno(sys.stdin) and fallback: if not self.warned_about_pty_fallback: sys.stderr.write( "WARNING: stdin has no fileno; falling back to non-pty execution!\n" ) # noqa self.warned_about_pty_fallback = True use_pty = False return use_pty def read_proc_stdout(self, num_bytes): # Obtain useful read-some-bytes function if self.using_pty: # Need to handle spurious OSErrors on some Linux platforms. try: data = os.read(self.parent_fd, num_bytes) except OSError as e: # Only eat this specific OSError so we don't hide others if "Input/output error" not in str(e): raise # The bad OSErrors happen after all expected output has # appeared, so we return a falsey value, which triggers the # "end of output" logic in code using reader functions. data = None else: data = os.read(self.process.stdout.fileno(), num_bytes) return data def read_proc_stderr(self, num_bytes): # NOTE: when using a pty, this will never be called. # TODO: do we ever get those OSErrors on stderr? Feels like we could? return os.read(self.process.stderr.fileno(), num_bytes) def _write_proc_stdin(self, data): # NOTE: parent_fd from os.fork() is a read/write pipe attached to our # forked process' stdout/stdin, respectively. fd = self.parent_fd if self.using_pty else self.process.stdin.fileno() # Try to write, ignoring broken pipes if encountered (implies child # process exited before the process piping stdin to us finished; # there's nothing we can do about that!) try: return os.write(fd, data) except OSError as e: if 'Broken pipe' not in str(e): raise def start(self, command, shell, env): if self.using_pty: if pty is None: # Encountered ImportError sys.exit( "You indicated pty=True, but your platform doesn't support the 'pty' module!" ) # noqa cols, rows = pty_size() self.pid, self.parent_fd = pty.fork() # If we're the child process, load up the actual command in a # shell, just as subprocess does; this replaces our process - whose # pipes are all hooked up to the PTY - with the "real" one. if self.pid == 0: # TODO: both pty.spawn() and pexpect.spawn() do a lot of # setup/teardown involving tty.setraw, getrlimit, signal. # Ostensibly we'll want some of that eventually, but if # possible write tests - integration-level if necessary - # before adding it! # # Set pty window size based on what our own controlling # terminal's window size appears to be. # TODO: make subroutine? winsize = struct.pack('HHHH', rows, cols, 0, 0) fcntl.ioctl(sys.stdout.fileno(), termios.TIOCSWINSZ, winsize) # Use execve for bare-minimum "exec w/ variable # args + env" # behavior. No need for the 'p' (use PATH to find executable) # for now. # TODO: see if subprocess is using equivalent of execvp... os.execve(shell, [shell, '-c', command], env) else: self.process = Popen( command, shell=True, executable=shell, env=env, stdout=PIPE, stderr=PIPE, stdin=PIPE, ) @property def process_is_finished(self): if self.using_pty: # NOTE: # https://github.com/pexpect/ptyprocess/blob/4058faa05e2940662ab6da1330aa0586c6f9cd9c/ptyprocess/ptyprocess.py#L680-L687 # implies that Linux "requires" use of the blocking, non-WNOHANG # version of this call. Our testing doesn't verify this, however, # so... # NOTE: It does appear to be totally blocking on Windows, so our # issue #351 may be totally unsolvable there. Unclear. pid_val, self.status = os.waitpid(self.pid, os.WNOHANG) return pid_val != 0 else: return self.process.poll() is not None def returncode(self): if self.using_pty: # No subprocess.returncode available; use WIFEXITED/WIFSIGNALED to # determine whch of WEXITSTATUS / WTERMSIG to use. # TODO: is it safe to just say "call all WEXITSTATUS/WTERMSIG and # return whichever one of them is nondefault"? Probably not? # NOTE: doing this in an arbitrary order should be safe since only # one of the WIF* methods ought to ever return True. code = None if os.WIFEXITED(self.status): code = os.WEXITSTATUS(self.status) elif os.WIFSIGNALED(self.status): code = os.WTERMSIG(self.status) # Match subprocess.returncode by turning signals into negative # 'exit code' integers. code = -1 * code return code # TODO: do we care about WIFSTOPPED? Maybe someday? else: return self.process.returncode def stop(self): # No explicit close-out required (so far). pass
class Wasm3(): def __init__(self, exe): self.exe = exe self.p = None self.loaded = None self.timeout = args.timeout self.autorestart = True self.run() def run(self): if self.p: self.terminate() cmd = shlex.split(self.exe) #print(f"wasm3: Starting {' '.join(cmd)}") self.q = Queue() self.p = Popen(cmd, bufsize=0, stdin=PIPE, stdout=PIPE, stderr=STDOUT) def _read_output(out, queue): for data in iter(lambda: out.read(1024), b''): queue.put(data) queue.put(None) self.t = Thread(target=_read_output, args=(self.p.stdout, self.q)) self.t.daemon = True self.t.start() try: self._read_until("wasm3> ") except Exception as e: print(f"wasm3: Could not start: {e}") def restart(self): print(f"wasm3: Restarting") for i in range(10): try: self.run() try: if self.loaded: self.load(self.loaded) except Exception as e: pass break except Exception as e: print(f"wasm3: {e} => retry") time.sleep(0.1) def init(self): return self._run_cmd(f":init\n") def version(self): return self._run_cmd(f":version\n") def load(self, fn): self.loaded = None with open(fn, "rb") as f: wasm = f.read() res = self._run_cmd(f":load-hex {len(wasm)}\n{wasm.hex()}\n") self.loaded = fn return res def invoke(self, cmd): return self._run_cmd(":invoke " + " ".join(map(str, cmd)) + "\n") def _run_cmd(self, cmd): if self.autorestart and not self._is_running(): self.restart() self._flush_input() #print(f"wasm3: {cmd.strip()}") self._write(cmd) return self._read_until("wasm3> ") def _read_until(self, token): buff = "" tout = time.time() + self.timeout error = None while time.time() < tout: try: data = self.q.get(timeout=0.1) if data == None: error = "Crashed" break buff = buff + data.decode("utf-8") idx = buff.rfind(token) if idx >= 0: return buff[0:idx].strip() except Empty: pass else: error = "Timeout" self.terminate() raise Exception(error) def _write(self, data): self.p.stdin.write(data.encode("utf-8")) self.p.stdin.flush() def _is_running(self): return self.p and (self.p.poll() == None) def _flush_input(self): while not self.q.empty(): self.q.get() def terminate(self): self.p.stdin.close() self.p.terminate() self.p.wait(timeout=1.0) self.p = None
def runAsWithSudo(self, user="", password="", silent=True): """ Use pty method to run "su" to run a command as another user... Required parameters: user, password, command @author: Roy Nielsen """ self.logger.log(lp.DEBUG, "Starting runAsWithSudo: ") self.logger.log(lp.DEBUG, "\tuser: \"" + str(user) + "\"") self.logger.log(lp.DEBUG, "\tcmd : \"" + str(self.command) + "\"") if re.match("^\s+$", user) or re.match("^\s+$", password) or \ not user or not password or \ not self.command: self.logger.log(lp.WARNING, "Cannot pass in empty parameters...") self.logger.log(lp.WARNING, "user = \"" + str(user) + "\"") self.logger.log(lp.WARNING, "check password...") if not silent: self.logger.log(lp.WARNING, "command: " + str(self.command)) return 255 else: output = "" internal_command = [ "/usr/bin/su", str("-m"), str(user).strip(), str("-c") ] if isinstance(self.command, list): cmd = [] for i in range(len(self.command)): try: cmd.append(str(self.command[i]('utf-8'))) except UnicodeDecodeError: cmd.append(str(self.command[i])) internal_command.append( str("/usr/bin/sudo -S -s '" + " ".join(cmd) + "'")) elif isinstance(self.command, str): try: internal_command.append( str("/usr/bin/sudo -E -S -s " + "'" + str(self.command('utf-8')) + "'")) except UnicodeDecodeError: internal_command.append( str("/usr/bin/sudo -E -S -s " + "'" + str(self.command) + "'")) try: (master, slave) = pty.openpty() except SubprocessError as err: self.logger.log(lp.WARNING, "Error trying to open pty: " + str(err)) self.logger.log(lp.WARNING, traceback.format_exc()) self.logger.log(lp.WARNING, str(err)) raise err else: try: proc = Popen(internal_command, stdin=slave, stdout=slave, stderr=slave, close_fds=True, text=self.text) except SubprocessError as err: self.logger.log( lp.WARNING, "Error opening process to pty: " + str(err)) self.logger.log(lp.WARNING, traceback.format_exc()) self.logger.log(lp.WARNING, str(err)) raise err else: ##### # Catch the su password prompt # prompt = os.read(master, 512) self.waitnoecho(master, 3) self.prompt = os.read(master, 512) ##### # pass in the password os.write(master, password.strip() + "\n") ##### # catch the password self.prompt = os.read(master, 512) ##### # Wait for the next password prompt self.waitnoecho(master, 3) ##### # catch the password prompt self.prompt = os.read(master, 512) ##### # Enter the sudo password os.write(master, password + "\n") ##### # Catch the password os.read(master, 512) # output = tmp + output while True: ##### # timeout of 0 means "poll" ready, _, _ = select.select([master], [], [], 0) if ready: line = os.read(master, 512) ##### # Warning, uncomment at your own risk - several # programs print empty lines that will cause this # to break and the output will be all goofed up. # if not line : # break # print output.rstrip() output = output + line elif proc.poll() is not None: break # print output.strip() os.close(master) os.close(slave) self.libc.sync() proc.wait() self.libc.sync() self.stdout = proc.stdout self.stderr = proc.stderr self.retcode = proc.returncode # print output.strip() if not silent: self.logger.log( lp.DEBUG, "\n\nLeaving runAs with Sudo: \"" + str(self.stdout) + "\"\n\n") self.command = None return self.stdout, self.stderr, self.retcode
class Capture(object): default_port = 9900 # Default port to run CaptureJS at check_interval = 0.05 # Frequency (seconds) to check if self.started # Set engine configurations for PhantomJS and Puppeteer engines = AttrDict( phantomjs=AttrDict(cmd='phantomjs --ssl-protocol=any', script='capture.js', first_line=b'PhantomJS.*capture\\.js', name='Capture', version='1.0'), chrome=AttrDict(cmd='node', script='chromecapture.js', first_line=b'node\\.js.*chromecapture\\.js', name='ChromeCapture', version='1.1'), ) ''' Create a proxy for capture.js. Typical usage:: capture = Capture() with open('screenshot.png', 'wb') as handle: handle.write(capture.png('https://gramener.com/')) with open('screenshot.pdf', 'wb') as handle: handle.write(capture.pdf('https://gramener.com/')) The constructor accepts these optional parameters: :arg int port: port where capture.js is running. Default: 9900 :arg string url: URL:port where PhantomJS is running with capture.js. Default: ``http://localhost:<port>/`` :arg string cmd: Command to run PhantomJS with capture.js at the specified port. Default: ``phantomjs $GRAMEXPATH/apps/capture/capture.js --port=<port>`` :arg int timeout: Seconds to wait for PhantomJS to timeout. Default: 10 The constructor runs :meth:`Capture.start` in a new thread, which checks if capture.js is running at ``url``. If not, it runs ``cmd`` and checks again. Until capture.js is detected, all capture methods will fail. ''' def __init__(self, port=None, url=None, engine=None, cmd=None, timeout=DEFAULT_TIMEOUT): # Set default values for port, url and cmd self.engine = self.engines['phantomjs' if engine is None else engine] port = self.default_port if port is None else port if url is None: url = 'http://*****:*****@ {self.url}') thread.daemon = True thread.start() def _start(self): ''' Check if capture is already running at ``url``. If not, start ``cmd`` and check again. Print logs from ``cmd``. ''' self.started = False script = self.engine.script try: # Check if capture.js is at the url specified app_log.info('Pinging %s at %s', script, self.url) r = requests.get(self.url, timeout=self.timeout) self._validate_server(r) self.started = True except requests.ReadTimeout: # If capture.js doesn't respond immediately, we haven't started app_log.error('url: %s timed out', self.url) except requests.ConnectionError: # Try starting the process again app_log.info('Starting %s via %s', script, self.cmd) self.close() # self.cmd is taken from the YAML configuration. Safe to run self.proc = Popen(shlex.split(self.cmd), stdout=PIPE, stderr=STDOUT) # nosec self.proc.poll() atexit.register(self.close) # TODO: what if readline() does not return quickly? line = self.proc.stdout.readline().strip() if not self.first_line_re.search(line): return app_log.error('cmd: %s invalid. Returned "%s"', self.cmd, line) app_log.info('Pinging %s at %s', script, self.url) try: r = requests.get(self.url, timeout=self.timeout) self._validate_server(r) pid = self.proc.pid app_log.info(line.decode('utf-8') + ' live (pid=%s)', pid) self.started = True # Keep logging capture.js output until proc is killed by another thread while hasattr(self, 'proc'): line = self.proc.stdout.readline().strip() if len(line) == 0: app_log.info('%s terminated: pid=%d', script, pid) self.started = False break # Capture won't print anything, unless there's a problem, or if debug is on. # So log it at warning level not info. app_log.warning(line.decode('utf-8')) except Exception: app_log.exception('Ran %s. But %s not at %s', self.cmd, script, self.url) except Exception: app_log.exception('Cannot start Capture') def close(self): '''Stop capture.js if it has been started by this object''' if hasattr(self, 'proc'): try: process = psutil.Process(self.proc.pid) for proc in process.children(recursive=True): proc.kill() process.kill() except psutil.NoSuchProcess: app_log.info('%s PID %d already killed', self.engine.script, self.proc.pid) pass delattr(self, 'proc') def _validate_server(self, response): # Make sure that the response we got is from the right version of capture.js server = response.headers.get('Server', '') parts = server.split('/', 2) script = self.engine.script if not len(parts) == 2 or parts[0] != self.engine.name or parts[ 1] < self.engine.version: raise RuntimeError('Server: %s at %s is not %s' % (server, self.url, script)) @tornado.gen.coroutine def capture_async(self, headers=None, **kwargs): ''' Returns a screenshot of the URL. Runs asynchronously in Gramex. Arguments are same as :py:func:`capture` ''' # If ?start is provided, start server and wait until timeout if 'start' in kwargs: self.start() end_time = time.time() + self.timeout while not self.started and time.time() < end_time: yield tornado.gen.sleep(self.check_interval) if not self.started: raise RuntimeError('%s not started. See logs' % self.engine.script) r = yield self.browser.fetch(self.url, method='POST', body=urlencode(kwargs, doseq=True), raise_error=False, connect_timeout=self.timeout, request_timeout=self.timeout, headers=headers) if r.code == OK: self._validate_server(r) raise tornado.gen.Return(r) def capture(self, url, **kwargs): ''' Return a screenshot of the URL. :arg str url: URL to take a screenshot of :arg str ext: format of output. Can be pdf, png, gif or jpg :arg str selector: Restrict screenshot to (optional) CSS selector in URL :arg int delay: milliseconds (or expression) to wait for before taking a screenshot :arg str format: A3, A4, A5, Legal, Letter or Tabloid. Defaults to A4. For PDF :arg str layout: A3, A4, A5, Legal, 16x9, 16x10, 4x3. Defaults to 4x3. For PPTX :arg str orientation: portrait or landscape. Defaults to portrait. For PDF :arg str header: header for the page. For PDF :arg str footer: footer for the page. For PDF :arg int width: screen width. Default: 1200. For PNG/GIF/JPG :arg int height: screen height. Default: 768. For PNG/GIF/JPG :arg float scale: zooms the screen by a factor. For PNG/GIF/JPG :arg int dpi: dots (pixels) per inch. For PPTX :arg str title: slide title. For PPTX :arg int debug: sets log level for HTTP requests (2) and responses (1) :return: a bytestring with the binary contents of the screenshot :rtype: bytes :raises RuntimeError: if capture.js is not running or fails ''' # Ensure that we're connecting to the right version of capture.js if not self.started: end_time = time.time() + self.timeout while not self.started and time.time() < end_time: time.sleep(self.check_interval) if not self.started: raise RuntimeError('%s not started. See logs' % self.engine.script) kwargs['url'] = url r = requests.post(self.url, data=kwargs, timeout=self.timeout) if r.status_code == OK: self._validate_server(r) return r.content else: raise RuntimeError('%s error: %s' % (self.engine.script, r.content)) def pdf(self, url, **kwargs): '''An alias for :meth:`Capture.capture` with ``ext='pdf'``.''' kwargs['ext'] = 'pdf' return self.capture(url, **kwargs) def png(self, url, **kwargs): '''An alias for :meth:`Capture.capture` with ``ext='png'``.''' kwargs['ext'] = 'png' return self.capture(url, **kwargs) def pptx(self, url, **kwargs): '''An alias for :meth:`Capture.capture` with ``ext='pptx'``.''' kwargs['ext'] = 'pptx' return self.capture(url, **kwargs) def jpg(self, url, **kwargs): '''An alias for :meth:`Capture.capture` with ``ext='jpg'``.''' kwargs['ext'] = 'jpg' return self.capture(url, **kwargs) def gif(self, url, **kwargs): '''An alias for :meth:`Capture.capture` with ``ext='gif'``.''' kwargs['ext'] = 'gif' return self.capture(url, **kwargs)
def waitNpassThruStdout(self, chk_string=None, respawn=False, silent=True): """ Use the subprocess module to execute a command, returning the output of the command Author: Roy Nielsen """ self.stdout = '' self.stderr = '' self.retcode = 999 if self.command: try: proc = Popen(self.command, stdout=PIPE, stderr=PIPE, shell=self.myshell, env=self.environ, close_fds=self.cfds, text=self.text) if proc: while True: ##### # process stdout myout = proc.stdout.readline() if myout == '' and proc.poll() is not None: break tmpline = str(myout) tmpline = str(tmpline).strip() self.stdout += str(tmpline) + "\n" # if tmpline and not silent: # self.logger.log(lp.DEBUG, str(tmpline)) if isinstance(chk_string, str): if not chk_string: continue else: if re.search(chk_string, tmpline): try: proc.stdout.close() except SubprocessError as err: self.logger.log( lp.INFO, traceback.format_exc()) try: proc.stderr.close() except SubprocessError as err: self.logger.log( lp.INFO, traceback.format_exc()) if respawn: pass else: self.logger.log( lp.INFO, "chk_string found" + "... exiting process.") break if isinstance(chk_string, list): if not chk_string: continue else: found = False for mystring in chk_string: if chk_string(mystring, tmpline): try: proc.stdout.close() except SubprocessError as err: self.logger.log( lp.INFO, traceback.format_exc()) try: proc.stderr.close() except SubprocessError as err: self.logger.log( lp.INFO, traceback.format_exc()) proc.stderr.close() if respawn: pass else: self.logger.log( lp.INFO, "chk_string " + "found... " + "exiting process.") found = True break if found: break while True: myerr = proc.stderr.readline() if myerr == '' and proc.poll() is not None: break tmpline = str(myerr).strip() self.stderr += tmpline + "\n" if tmpline and not silent: self.logger.log(lp.DEBUG, str(tmpline)) if isinstance(chk_string, str): if not chk_string: continue else: if re.search(chk_string, tmpline): try: proc.stdout.close() except SubprocessError as err: self.logger.log( lp.INFO, traceback.format_exc()) try: proc.stderr.close() except SubprocessError as err: self.logger.log( lp.INFO, traceback.format_exc()) if respawn: pass else: self.logger.log( lp.INFO, "chk_string found" + "... exiting process.") break if isinstance(chk_string, list): if not chk_string: continue else: found = False for mystring in chk_string: if chk_string(mystring, tmpline): try: proc.stdout.close() except SubprocessError as err: self.logger.log( lp.INFO, traceback.format_exc()) try: proc.stderr.close() except SubprocessError as err: self.logger.log( lp.INFO, traceback.format_exc()) if respawn: pass else: self.logger.log( lp.INFO, "chk_string " + "found... " + "exiting process.") found = True break if found: break proc.wait() try: proc.stdout.close() except SubprocessError as err: self.logger.log(lp.INFO, traceback.format_exc()) try: proc.stderr.close() except SubprocessError as err: self.logger.log(lp.INFO, traceback.format_exc()) self.retcode = proc.returncode self.libc.sync() except SubprocessError as err: if not silent: self.logger.log(lp.WARNING, "command: " + str(self.printcmd)) self.logger.log(lp.WARNING, "stderr: " + str(self.stderr)) self.logger.log(lp.WARNING, traceback.format_exc()) self.logger.log(lp.WARNING, str(err)) raise err else: if not silent: self.logger.log(lp.DEBUG, "Done with: " + self.printcmd) finally: self.retcode = proc.returncode if not silent: self.logger.log(lp.DEBUG, "Done with command: " + self.printcmd) self.logger.log(lp.DEBUG, "stdout: " + str(self.stdout)) self.logger.log(lp.DEBUG, "stderr: " + str(self.stderr)) self.logger.log(lp.DEBUG, "retcode: " + str(self.retcode)) else: self.logger.log(lp.WARNING, "Cannot run a command that is empty...") self.stdout = None self.stderr = None self.retcode = None self.command = None return self.stdout, self.stderr, self.retcode
def launch_gateway(): if "PYSPARK_GATEWAY_PORT" in os.environ: gateway_port = int(os.environ["PYSPARK_GATEWAY_PORT"]) else: SPARK_HOME = os.environ["SPARK_HOME"] # Launch the Py4j gateway using Spark's run command so that we pick up the # proper classpath and settings from spark-env.sh on_windows = platform.system() == "Windows" script = "./bin/spark-submit.cmd" if on_windows else "./bin/spark-submit" submit_args = os.environ.get("PYSPARK_SUBMIT_ARGS", "pyspark-shell") if os.environ.get("SPARK_TESTING"): submit_args = "--conf spark.ui.enabled=false " + submit_args command = [os.path.join(SPARK_HOME, script)] + shlex.split(submit_args) # Start a socket that will be used by PythonGatewayServer to communicate its port to us callback_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) callback_socket.bind(('127.0.0.1', 0)) callback_socket.listen(1) callback_host, callback_port = callback_socket.getsockname() env = dict(os.environ) env['_PYSPARK_DRIVER_CALLBACK_HOST'] = callback_host env['_PYSPARK_DRIVER_CALLBACK_PORT'] = str(callback_port) # Launch the Java gateway. # We open a pipe to stdin so that the Java gateway can die when the pipe is broken if not on_windows: # Don't send ctrl-c / SIGINT to the Java gateway: def preexec_func(): signal.signal(signal.SIGINT, signal.SIG_IGN) proc = Popen(command, stdin=PIPE, preexec_fn=preexec_func, env=env) else: # preexec_fn not supported on Windows proc = Popen(command, stdin=PIPE, env=env) gateway_port = None # We use select() here in order to avoid blocking indefinitely if the subprocess dies # before connecting while gateway_port is None and proc.poll() is None: timeout = 1 # (seconds) readable, _, _ = select.select([callback_socket], [], [], timeout) if callback_socket in readable: gateway_connection = callback_socket.accept()[0] # Determine which ephemeral port the server started on: gateway_port = read_int(gateway_connection.makefile(mode="rb")) gateway_connection.close() callback_socket.close() if gateway_port is None: raise Exception( "Java gateway process exited before sending the driver its port number" ) # In Windows, ensure the Java child processes do not linger after Python has exited. # In UNIX-based systems, the child process can kill itself on broken pipe (i.e. when # the parent process' stdin sends an EOF). In Windows, however, this is not possible # because java.lang.Process reads directly from the parent process' stdin, contending # with any opportunity to read an EOF from the parent. Note that this is only best # effort and will not take effect if the python process is violently terminated. if on_windows: # In Windows, the child process here is "spark-submit.cmd", not the JVM itself # (because the UNIX "exec" command is not available). This means we cannot simply # call proc.kill(), which kills only the "spark-submit.cmd" process but not the # JVMs. Instead, we use "taskkill" with the tree-kill option "/t" to terminate all # child processes in the tree (http://technet.microsoft.com/en-us/library/bb491009.aspx) def killChild(): Popen([ "cmd", "/c", "taskkill", "/f", "/t", "/pid", str(proc.pid) ]) atexit.register(killChild) # Connect to the gateway gateway = JavaGateway(GatewayClient(port=gateway_port), auto_convert=True) # Import the classes used by PySpark java_import(gateway.jvm, "org.apache.spark.SparkConf") java_import(gateway.jvm, "org.apache.spark.api.java.*") java_import(gateway.jvm, "org.apache.spark.api.python.*") java_import(gateway.jvm, "org.apache.spark.mllib.api.python.*") # TODO(davies): move into sql java_import(gateway.jvm, "org.apache.spark.sql.*") java_import(gateway.jvm, "org.apache.spark.sql.hive.*") java_import(gateway.jvm, "scala.Tuple2") return gateway
class Process(object): ''' Represents a running/ran process ''' @staticmethod def devnull(): ''' Helper method for opening devnull ''' return open('/dev/null', 'w') @staticmethod def call(command, cwd=None, shell=False): ''' Calls a command (either string or list of args). Returns tuple: (stdout, stderr) ''' if type(command) is not str or ' ' in command or shell: shell = True if Configuration.verbose > 1: Color.pe("\n {C}[?] {W} Executing (Shell): {B}%s{W}" % command) else: shell = False if Configuration.verbose > 1: Color.pe("\n {C}[?]{W} Executing: {B}%s{W}" % command) pid = Popen(command, cwd=cwd, stdout=PIPE, stderr=PIPE, shell=shell) pid.wait() (stdout, stderr) = pid.communicate() # Python 3 compatibility if type(stdout) is bytes: stdout = stdout.decode('utf-8') if type(stderr) is bytes: stderr = stderr.decode('utf-8') if Configuration.verbose > 1 and stdout is not None and stdout.strip() != '': Color.pe("{P} [stdout] %s{W}" % '\n [stdout] '.join(stdout.strip().split('\n'))) if Configuration.verbose > 1 and stderr is not None and stderr.strip() != '': Color.pe("{P} [stderr] %s{W}" % '\n [stderr] '.join(stderr.strip().split('\n'))) return (stdout, stderr) @staticmethod def exists(program): ''' Checks if program is installed on this system ''' p = Process(['which', program]) stdout = p.stdout().strip() stderr = p.stderr().strip() if stdout == '' and stderr == '': return False return True def __init__(self, command, devnull=False, stdout=PIPE, stderr=PIPE, cwd=None, bufsize=0): ''' Starts executing command ''' if type(command) is str: # Commands have to be a list command = command.split(' ') self.command = command if Configuration.verbose > 1: Color.pe("\n {C}[?] {W} Executing: {B}%s{W}" % ' '.join(command)) self.out = None self.err = None if devnull: sout = Process.devnull() serr = Process.devnull() else: sout = stdout serr = stderr self.start_time = time.time() self.pid = Popen(command, stdout=sout, stderr=serr, cwd=cwd, bufsize=bufsize) def __del__(self): ''' Ran when object is GC'd. If process is still running at this point, it should die. ''' try: if self.pid and self.pid.poll() is None: self.interrupt() except AttributeError: pass def stdout(self): ''' Waits for process to finish, returns stdout output ''' self.get_output() if Configuration.verbose > 1 and self.out is not None and self.out.strip() != '': Color.pe("{P} [stdout] %s{W}" % '\n [stdout] '.join(self.out.strip().split('\n'))) return self.out def stderr(self): ''' Waits for process to finish, returns stderr output ''' self.get_output() if Configuration.verbose > 1 and self.err is not None and self.err.strip() != '': Color.pe("{P} [stderr] %s{W}" % '\n [stderr] '.join(self.err.strip().split('\n'))) return self.err def stdoutln(self): return self.pid.stdout.readline() def stderrln(self): return self.pid.stderr.readline() def get_output(self): ''' Waits for process to finish, sets stdout & stderr ''' if self.pid.poll() is None: self.pid.wait() if self.out is None: (self.out, self.err) = self.pid.communicate() if type(self.out) is bytes: self.out = self.out.decode('utf-8') if type(self.err) is bytes: self.err = self.err.decode('utf-8') return (self.out, self.err) def poll(self): ''' Returns exit code if process is dead, otherwise "None" ''' return self.pid.poll() def wait(self): self.pid.wait() def running_time(self): ''' Returns number of seconds since process was started ''' return int(time.time() - self.start_time) def interrupt(self, wait_time=2.0): ''' Send interrupt to current process. If process fails to exit within `wait_time` seconds, terminates it. ''' try: pid = self.pid.pid cmd = self.command if type(cmd) is list: cmd = ' '.join(cmd) if Configuration.verbose > 1: Color.pe('\n {C}[?] {W} sending interrupt to PID %d (%s)' % (pid, cmd)) os.kill(pid, signal.SIGINT) start_time = time.time() # Time since Interrupt was sent while self.pid.poll() is None: # Process is still running time.sleep(0.1) if time.time() - start_time > wait_time: # We waited too long for process to die, terminate it. if Configuration.verbose > 1: Color.pe('\n {C}[?] {W} Waited > %0.2f seconds for process to die, killing it' % wait_time) os.kill(pid, signal.SIGTERM) self.pid.terminate() break except OSError as e: if 'No such process' in e.__str__(): return raise e # process cannot be killed
def runAs(self, user="", password="", silent=True): """ Use pexpect to run "su" to run a command as another user... Required parameters: user, password, command @author: Roy Nielsen """ self.stdout = "" self.stderr = "" self.retcode = 999 # pretcode = 0 if re.match("^\s*$", user) or \ re.match("^\s*$", password) or \ not self.command: self.logger.log(lp.WARNING, "Cannot pass in empty parameters...") self.logger.log(lp.WARNING, "user = \"" + str(user) + "\"") self.logger.log(lp.WARNING, "check password...") if not silent: self.logger.log(lp.WARNING, "command = \"" + str(self.command) + "\"") return 255 else: output = "" internal_command = ["/usr/bin/su", "-", str(user.strip()), "-c"] if isinstance(self.command, list): internal_command.append(" ".join(self.command)) if not silent: self.logger.log( lp.DEBUG, "Trying to execute: \"" + " ".join(internal_command) + "\"") elif isinstance(self.command, str): internal_command += self.command if not silent: self.logger.log( lp.DEBUG, "Trying to execute: \"" + " ".join(internal_command) + "\"") (master, slave) = pty.openpty() proc = Popen(internal_command, stdin=slave, stdout=slave, stderr=slave, close_fds=True, text=self.text) prompt = os.read(master, 10) if re.match("^Password:"******"\n") line = os.read(master, 512) output = output + line while True: ##### # timeout of 0 means "poll" ready, _, _ = select.select([master], [], [], 0) if ready: line = os.read(master, 512) ##### # Warning, uncomment at your own risk - several # programs print empty lines that will cause this to # break and the output will be all goofed up. # if not line : # break # print output.rstrip() output = output + line elif proc.poll() is not None: break os.close(master) os.close(slave) self.libc.sync() proc.wait() self.libc.sync() self.stdout = proc.stdout self.stderr = proc.stderr self.retcode = proc.returncode else: output = prompt while True: ##### # timeout of 0 means "poll" ready, _, _ = select.select([master], [], [], 0) if ready: line = os.read(master, 512) ##### # Warning, uncomment at your own risk - several # programs print empty lines that will cause this to # break and the output will be all goofed up. # if not line : # break # print output.rstrip() output = output + line elif proc.poll() is not None: break os.close(master) os.close(slave) proc.wait() self.stdout = output self.stderr = str(proc.stderr) self.retcode = proc.returncode output = output.strip() if not silent: self.logger.log(lp.DEBUG, "retcode: " + str(self.stdout)) self.logger.log(lp.DEBUG, "retcode: " + str(self.stderr)) self.logger.log(lp.DEBUG, "retcode: " + str(self.retcode)) self.command = None return self.stdout, self.stderr, self.retcode
class Tailer(threading.Thread): """Logic to tail follow a log file for a GUI viewer. logview -- A GUI view to display the content of the log file. filename -- The name of the log file. cmd_tmpl -- The command template use to follow the log file. (global cfg '[hosts][HOST]remote/local tail command template') pollable -- If specified, it must implement a pollable.poll() method, which is called at regular intervals. """ READ_SIZE = 4096 TAGS = { "CRITICAL": [re.compile(r"\b(?:CRITICAL|ERROR)\b"), "red"], "WARNING": [re.compile(r"\bWARNING\b"), "#a83fd3"] } def __init__(self, logview, filename, cmd_tmpl=None, pollable=None, filters=None): super(Tailer, self).__init__() self.logview = logview self.filename = filename self.cmd_tmpl = cmd_tmpl self.pollable = pollable self.filters = filters self.logbuffer = logview.get_buffer() self.quit = False self.proc = None self.freeze = False self.has_warned_corrupt = False self.tags = {} def clear(self): """Clear the log buffer.""" pos_start, pos_end = self.logbuffer.get_bounds() self.logbuffer.delete(pos_start, pos_end) def run(self): """Invoke the tailer.""" command = [] if ":" in self.filename: # remote user_at_host, filename = self.filename.split(':') if "@" in user_at_host: owner, host = user_at_host.split("@", 1) else: owner, host = (None, user_at_host) ssh = str( GLOBAL_CFG.get_host_item("remote shell template", host, owner)) command = shlex.split(ssh) + ["-n", user_at_host] cmd_tmpl = str( GLOBAL_CFG.get_host_item("remote tail command template", host, owner)) else: filename = self.filename cmd_tmpl = str( GLOBAL_CFG.get_host_item("local tail command template")) if self.cmd_tmpl: cmd_tmpl = self.cmd_tmpl command += shlex.split(cmd_tmpl % {"filename": filename}) try: self.proc = Popen(command, stdout=PIPE, stderr=STDOUT, preexec_fn=os.setpgrp) except OSError as exc: # E.g. ssh command not found dialog = warning_dialog( "%s: %s" % (exc, " ".join([quote(item) for item in command]))) gobject.idle_add(dialog.warn) return poller = select.poll() poller.register(self.proc.stdout.fileno()) buf = "" while not self.quit and self.proc.poll() is None: try: self.pollable.poll() except (TypeError, AttributeError): pass if self.freeze or not poller.poll(100): # 100 ms timeout sleep(1) continue # Both self.proc.stdout.read(SIZE) and self.proc.stdout.readline() # can block. However os.read(FILENO, SIZE) should be fine after a # poller.poll(). try: data = os.read(self.proc.stdout.fileno(), self.READ_SIZE) except (IOError, OSError) as exc: dialog = warning_dialog( "%s: %s" % (exc, " ".join([quote(item) for item in command]))) gobject.idle_add(dialog.warn) break if data: # Manage buffer, only add full lines to display to ensure # filtering and tagging work for line in data.splitlines(True): if not line.endswith("\n"): buf += line continue elif buf: line = buf + line buf = "" if (not self.filters or all([re.search(f, line) for f in self.filters])): gobject.idle_add(self.update_gui, line) sleep(0.01) self.stop() def stop(self): """Stop the tailer.""" self.quit = True try: # It is important that we kill processes like "tail -F", or it will # hang the GUI. os.killpg(self.proc.pid, signal.SIGTERM) self.proc.wait() except (AttributeError, OSError): pass def update_gui(self, line): """Update the GUI viewer.""" try: line.decode('utf-8') except UnicodeDecodeError as exc: if self.has_warned_corrupt: return False self.has_warned_corrupt = True dialog = warning_dialog("Problem reading file:\n %s: %s" % (type(exc).__name__, exc)) gobject.idle_add(dialog.warn) return False for word, setting in self.TAGS.items(): rec, colour = setting if rec.match(line): if word not in self.tags: self.tags[word] = self.logbuffer.create_tag( None, foreground=colour) self.logbuffer.insert_with_tags(self.logbuffer.get_end_iter(), line, self.tags[word]) break else: self.logbuffer.insert(self.logbuffer.get_end_iter(), line) self.logview.scroll_to_iter(self.logbuffer.get_end_iter(), 0) return False
def runWithSudo(self, password="", silent=True): """ Use pty method to run "sudo" to run a command with elevated privilege. Required parameters: user, password, command @author: Roy Nielsen """ self.logger.log(lp.DEBUG, "Starting runWithSudo: ") self.logger.log(lp.DEBUG, "\tcmd : " + str(self.command)) if re.match("^\s+$", password) or \ not password or \ not self.command: self.logger.log(lp.WARNING, "Cannot pass in empty parameters...") self.logger.log(lp.WARNING, "check password...") if not silent: self.logger.log(lp.WARNING, "command: " + str(self.command)) return (255) else: output = "" cmd = ["/usr/bin/sudo", "-S", "-s"] if isinstance(self.command, list): cmd = cmd + [" ".join(self.command)] elif isinstance(self.command, str): cmd = cmd + [self.command] try: (master, slave) = pty.openpty() except SubprocessError as err: self.logger.log(lp.WARNING, "Error trying to open pty: " + str(err)) self.logger.log(lp.WARNING, traceback.format_exc()) self.logger.log(lp.WARNING, str(err)) raise err else: try: proc = Popen(cmd, stdin=slave, stdout=slave, stderr=slave, close_fds=True, text=self.text) except SubprocessError as err: self.logger.log( lp.WARNING, "Error opening process to pty: " + str(err)) self.logger.log(lp.WARNING, traceback.format_exc()) self.logger.log(lp.WARNING, str(err)) raise err else: ##### # Catch the sudo password prompt # prompt = os.read(master, 512) self.waitnoecho(master, 3) self.prompt = os.read(master, 512) ##### # Enter the sudo password os.write(master, password + "\n") ##### # Catch the password os.read(master, 512) # output = tmp + output while True: ##### # timeout of 0 means "poll" ready, _, _ = select.select([master], [], [], 0) if ready: line = os.read(master, 512) ##### # Warning, uncomment at your own risk - several # programs print empty lines that will cause this # to break and the output will be all goofed up. # if not line : # break # print output.rstrip() output = output + line elif proc.poll() is not None: break # print output.strip() os.close(master) os.close(slave) self.libc.sync() proc.wait() self.libc.sync() self.stdout = output self.stderr = proc.stderr self.retcode = proc.returncode #print output.strip() #output = output.strip() if not silent: ##### # ONLY USE WHEN IN DEVELOPMENT AND DEBUGGING OR YOU MAY # REVEAL MORE THAN YOU WANT TO IN THE LOGS!!! self.logger.log(lp.DEBUG, "\n\nLeaving runAs with Sudo: \"" + \ str(output) + "\"\n" + str(self.stdout) + "\n") return output
class DaemonProcess(GObject.GObject): __gsignals__ = { # line(text) - emited when process outputs full line b"line" : (GObject.SIGNAL_RUN_FIRST, None, (object,)), # exit(code) - emited when process exits b"exit" : (GObject.SIGNAL_RUN_FIRST, None, (int,)), # failed(exception) - emited if process fails to start b"failed" : (GObject.SIGNAL_RUN_FIRST, None, (object,)), } SCROLLBACK_SIZE = 500 # Maximum number of output lines stored in memory PRIORITY_LOWEST = 19 PRIORITY_LOW = 10 PRIORITY_NORMAL = 0 PRIORITY_HIGH = -10 PRIORITY_HIGHEST = -20 def __init__(self, cmdline, priority=PRIORITY_NORMAL, max_cpus=0, env={}): """ cmdline should be list of arguments """ GObject.GObject.__init__(self) self.cmdline = cmdline self.priority = priority self.env = { x:env[x] for x in env } self.env["STNORESTART"] = "1" # see syncthing --help self.env["STNOUPGRADE"] = "1" if max_cpus > 0: self.env["GOMAXPROCS"] = str(max_cpus) self._proc = None def start(self): for x in self.env: os.environ[x] = self.env[x] try: self._cancel = Gio.Cancellable() if IS_WINDOWS: # Windows sinfo = STARTUPINFO() sinfo.dwFlags = STARTF_USESHOWWINDOW sinfo.wShowWindow = 0 cflags = nice_to_priority_class(self.priority) self._proc = Popen(self.cmdline, stdin=PIPE, stdout=PIPE, stderr=PIPE, startupinfo=sinfo, creationflags=cflags) self._stdout = WinPopenReader(self._proc.stdout) self._check = GLib.timeout_add_seconds(1, self._cb_check_alive) elif HAS_SUBPROCESS: # New Gio flags = Gio.SubprocessFlags.STDOUT_PIPE | Gio.SubprocessFlags.STDERR_MERGE if self.priority == 0: self._proc = Gio.Subprocess.new(self.cmdline, flags) else: # I just really do hope that there is no distro w/out nice command self._proc = Gio.Subprocess.new([ "nice", "-n", "%s" % self.priority ] + self.cmdline, flags) self._proc.wait_check_async(None, self._cb_finished) self._stdout = self._proc.get_stdout_pipe() else: # Gio < 3.12 - Gio.Subprocess is missing :( if self.priority == 0: self._proc = Popen(self.cmdline, stdout=PIPE) else: # still hoping self._proc = Popen([ "nice", "-n", "%s" % self.priority ], stdout=PIPE) self._stdout = Gio.UnixInputStream.new(self._proc.stdout.fileno(), False) self._check = GLib.timeout_add_seconds(1, self._cb_check_alive) except Exception as e: # Startup failed self.emit("failed", e) return self._lines = deque([], DaemonProcess.SCROLLBACK_SIZE) self._buffer = "" self._stdout.read_bytes_async(256, 0, self._cancel, self._cb_read, ()) def _cb_read(self, pipe, results, *a): """ Handler for read_bytes_async """ try: response = pipe.read_bytes_finish(results) except Exception as e: if not self._cancel.is_cancelled(): log.exception(e) GLib.idle_add(pipe.read_bytes_async, 256, 1, None, self._cb_read) return response = response.get_data().decode('utf-8') self._buffer = "%s%s" % (self._buffer, response) while "\n" in self._buffer: line, self._buffer = self._buffer.split("\n", 1) self._lines.append(line) self.emit('line', line) if not self._cancel.is_cancelled(): GLib.idle_add(pipe.read_bytes_async, 256, 1, None, self._cb_read, ()) def _cb_check_alive(self, *a): """ Repeatedly check if process is still alive. Called only on windows """ if self._proc == None: # Never started or killed really fast self.emit('exit', 1) self._cancel.cancel() if IS_WINDOWS: self._stdout.close() return False self._proc.poll() if self._proc.returncode is None: # Repeat until finished or canceled return (not self._cancel.is_cancelled()) # Child just died :) self.emit('exit', self._proc.returncode) self._cancel.cancel() if IS_WINDOWS: self._stdout.close() return False def _cb_finished(self, proc, results): """ Callback for wait_check_async. With Gio < 3.12, timer and _cb_check_alive is used. """ try: proc.wait_check_finish(results) log.info("Subprocess finished with code %s", proc.get_exit_status()) except GLib.GError: # Exited with exit code log.info("Subprocess exited with code %s", proc.get_exit_status()) if proc.get_exit_status() == 127: # Command not found self.emit("failed", Exception("Command not found")) else: self.emit('exit', proc.get_exit_status()) if IS_WINDOWS: self._stdout.close() self._cancel.cancel() def terminate(self): """ Terminates process (sends SIGTERM) """ if not self._proc is None: if IS_WINDOWS: # Windows self._proc.terminate() elif HAS_SUBPROCESS: # Gio.Subprocess self._proc.send_signal(15) else: # subprocess.Popen self._proc.terminate() self._proc = None if IS_WINDOWS: self._stdout.close() self._cancel.cancel() def kill(self): """ Kills process (sends SIGTERM) """ if not self._proc is None: if IS_WINDOWS: # Windows - can't actually kill self._proc.terminate() elif HAS_SUBPROCESS: # Gio.Subprocess self._proc.force_exit() else: # subprocess.Popen self._proc.kill() self._proc = None if IS_WINDOWS: self._stdout.close() self._cancel.cancel() def get_output(self): """ Returns process output as iterable list of lines """ return self._lines def get_commandline(self): """ Returns commandline used to start process """ return self.cmdline
def start_dbus(dbus_launch): if not dbus_launch or dbus_launch.lower() in FALSE_OPTIONS: log("start_dbus(%s) disabled", dbus_launch) return 0, {} bus_address = os.environ.get("DBUS_SESSION_BUS_ADDRESS") log("dbus_launch=%s, current DBUS_SESSION_BUS_ADDRESS=%s", dbus_launch, bus_address) if bus_address: log( "start_dbus(%s) disabled, found an existing DBUS_SESSION_BUS_ADDRESS=%s", dbus_launch, bus_address) return 0, {} assert POSIX try: def preexec(): os.setsid() close_fds() env = dict((k, v) for k, v in os.environ.items() if k in ( "PATH", "SSH_CLIENT", "SSH_CONNECTION", "XDG_CURRENT_DESKTOP", "XDG_SESSION_TYPE", "XDG_RUNTIME_DIR", "SHELL", "LANG", "USER", "LOGNAME", "HOME", "DISPLAY", "XAUTHORITY", "CKCON_X11_DISPLAY", )) import shlex cmd = shlex.split(dbus_launch) log("start_dbus(%s) env=%s", dbus_launch, env) proc = Popen(cmd, stdin=PIPE, stdout=PIPE, shell=False, env=env, preexec_fn=preexec) out = proc.communicate()[0] assert proc.poll() == 0, "exit code is %s" % proc.poll() #parse and add to global env: dbus_env = {} log("out(%s)=%s", cmd, nonl(out)) for l in bytestostr(out).splitlines(): if l.startswith("export "): continue sep = "=" if l.startswith("setenv "): l = l[len("setenv "):] sep = " " if l.startswith("set "): l = l[len("set "):] parts = l.split(sep, 1) if len(parts) != 2: continue k, v = parts if v.startswith("'") and v.endswith("';"): v = v[1:-2] elif v.endswith(";"): v = v[:-1] dbus_env[k] = v dbus_pid = int(dbus_env.get("DBUS_SESSION_BUS_PID", 0)) log("dbus_pid=%i, dbus-env=%s", dbus_pid, dbus_env) return dbus_pid, dbus_env except Exception as e: log("start_dbus(%s)", dbus_launch, exc_info=True) log.error("dbus-launch failed to start using command '%s':\n" % dbus_launch) log.error(" %s\n" % e) return 0, {}
class AcmeCapeInstrument(Instrument): mode = CONTINUOUS def __init__(self, target, iio_capture=which('iio-capture'), host='baylibre-acme.local', iio_device='iio:device0', buffer_size=256): super(AcmeCapeInstrument, self).__init__(target) self.iio_capture = iio_capture self.host = host self.iio_device = iio_device self.buffer_size = buffer_size self.sample_rate_hz = 100 if self.iio_capture is None: raise HostError('Missing iio-capture binary') self.command = None self.process = None self.add_channel('shunt', 'voltage') self.add_channel('bus', 'voltage') self.add_channel('device', 'power') self.add_channel('device', 'current') self.add_channel('timestamp', 'time_ms') def __del__(self): if self.process and self.process.pid: self.logger.warning('killing iio-capture process [{}]...'.format( self.process.pid)) self.process.kill() def reset(self, sites=None, kinds=None, channels=None): super(AcmeCapeInstrument, self).reset(sites, kinds, channels) self.raw_data_file = tempfile.mkstemp('.csv')[1] params = dict(iio_capture=self.iio_capture, host=self.host, buffer_size=self.buffer_size, iio_device=self.iio_device, outfile=self.raw_data_file) self.command = IIOCAP_CMD_TEMPLATE.substitute(**params) self.logger.debug('ACME cape command: {}'.format(self.command)) def start(self): self.process = Popen(self.command.split(), stdout=PIPE, stderr=STDOUT) def stop(self): self.process.terminate() timeout_secs = 10 output = '' for _ in range(timeout_secs): if self.process.poll() is not None: break time.sleep(1) else: output += _read_nonblock(self.process.stdout) self.process.kill() self.logger.error('iio-capture did not terminate gracefully') if self.process.poll() is None: msg = 'Could not terminate iio-capture:\n{}' raise HostError(msg.format(output)) if self.process.returncode != 15: # iio-capture exits with 15 when killed if sys.version_info[0] == 3: output += self.process.stdout.read().decode( sys.stdout.encoding, 'replace') else: output += self.process.stdout.read() self.logger.info('ACME instrument encountered an error, ' 'you may want to try rebooting the ACME device:\n' ' ssh root@{} reboot'.format(self.host)) raise HostError( 'iio-capture exited with an error ({}), output:\n{}'.format( self.process.returncode, output)) if not os.path.isfile(self.raw_data_file): raise HostError('Output CSV not generated.') self.process = None def get_data(self, outfile): if os.stat(self.raw_data_file).st_size == 0: self.logger.warning('"{}" appears to be empty'.format( self.raw_data_file)) return all_channels = [c.label for c in self.list_channels()] active_channels = [c.label for c in self.active_channels] active_indexes = [all_channels.index(ac) for ac in active_channels] with csvreader(self.raw_data_file, skipinitialspace=True) as reader: with csvwriter(outfile) as writer: writer.writerow(active_channels) header = next(reader) ts_index = header.index('timestamp ms') for row in reader: output_row = [] for i in active_indexes: if i == ts_index: # Leave time in ms output_row.append(float(row[i])) else: # Convert rest into standard units. output_row.append(float(row[i]) / 1000) writer.writerow(output_row) return MeasurementsCsv(outfile, self.active_channels, self.sample_rate_hz) def get_raw(self): return [self.raw_data_file]
def send_heartbeats(self): if python_binary(): heartbeat = self.build_heartbeat(**self.heartbeat) ua = 'sublime/%d sublime-wakatime/%s' % (ST_VERSION, __version__) cmd = [ python_binary(), API_CLIENT, '--entity', heartbeat['entity'], '--time', str('%f' % heartbeat['timestamp']), '--plugin', ua, ] if self.api_key: cmd.extend( ['--key', str(bytes.decode(self.api_key.encode('utf8')))]) if heartbeat['is_write']: cmd.append('--write') if heartbeat.get('alternate_project'): cmd.extend( ['--alternate-project', heartbeat['alternate_project']]) if heartbeat.get('cursorpos') is not None: cmd.extend(['--cursorpos', heartbeat['cursorpos']]) for pattern in self.ignore: cmd.extend(['--exclude', pattern]) if self.debug: cmd.append('--verbose') if self.hidefilenames: cmd.append('--hidefilenames') if self.proxy: cmd.extend(['--proxy', self.proxy]) if self.has_extra_heartbeats: cmd.append('--extra-heartbeats') stdin = PIPE extra_heartbeats = [ self.build_heartbeat(**x) for x in self.extra_heartbeats ] extra_heartbeats = json.dumps(extra_heartbeats) else: extra_heartbeats = None stdin = None log(DEBUG, ' '.join(obfuscate_apikey(cmd))) try: process = Popen(cmd, stdin=stdin, stdout=PIPE, stderr=STDOUT) inp = None if self.has_extra_heartbeats: inp = "{0}\n".format(extra_heartbeats) inp = inp.encode('utf-8') output, err = process.communicate(input=inp) output = u(output) retcode = process.poll() if (not retcode or retcode == 102) and not output: self.sent() else: update_status_bar('Error') if retcode: log( DEBUG if retcode == 102 else ERROR, 'wakatime-core exited with status: {0}'.format( retcode)) if output: log(ERROR, u('wakatime-core output: {0}').format(output)) except: log(ERROR, u(sys.exc_info()[1])) update_status_bar('Error') else: log(ERROR, 'Unable to find python binary.') update_status_bar('Error')