def create(self, usernames, passwords, target_iqn, target_lun, size, initiator_iqn_list): # NB: initiator_iqn_list needs to be a comma separated list of initiator iqn strings self.logger.debug("Preparing to execute create()") timeout = Timeout(self.script_timeout) process = Popen(self.scriptfile_path + " -c -q" + " -u " + usernames + " -p " + passwords + " -s " + size + " -m " + target_lun + " -t " + target_iqn + " -i " + initiator_iqn_list, stdout=PIPE, shell=True) output = "Create operation exceeded execution timeout.\n" returncode = 1 timeout.start() try: output = process.communicate()[0] returncode = process.returncode except Timeout: process.kill() self.logger.warn( "Process %s servicing create() " + "exceeded execution timeout and was terminated.", process.pid) if process.returncode is not None: returncode = process.returncode finally: timeout.cancel() return [output, returncode]
def proc(input, output, args, env=None, stderr=None): """Run a subprocess, passing input to its stdin and sending its stdout to output, with each item newline-seperated. Args is either a string to be shell-interpreted, or a list of args. stderr is either not redirected (default), mixed in with stdout (pass subprocess.STDOUT), or redirected to a given file. """ from gevent.subprocess import Popen, PIPE, STDOUT if isinstance(args, unicode): args = args.encode('utf8') if isinstance(args, str): shell = True else: shell = False group = gevent.pool.Group() proc = None try: proc = Popen(args, shell=shell, env=env, stdin=PIPE, stdout=PIPE, stderr=stderr) @group.spawn def do_input(): for item in input: item = item.encode('utf8') if isinstance(item, unicode) else str(item) proc.stdin.write('{}\n'.format(item)) proc.stdin.flush() proc.stdin.close() @group.spawn def do_output(): for line in proc.stdout: output.put(line.rstrip('\n')) output.close() proc.wait() group.join() finally: if proc and proc.poll() is None: try: proc.kill() except OSError as e: if e.errno != errno.ESRCH: raise
def get_git_refs(): if DISABLE_NEW_EXTENSIONS: return 'Disabled', 403 git_endpoint = request.values.get('ep', None) if git_endpoint is None: return jsonify(error={'message': 'Missing endpoint'}), 400 if not git_endpoint.endswith('.git'): return jsonify(error={'message': 'Invalid git endpoint'}), 400 git_path = config.get('MINEMELD_GIT_PATH', None) if git_path is None: return jsonify(error={'message': 'MINEMELD_GIT_PATH not set'}), 500 git_args = [git_path, 'ls-remote', '-t', '-h', git_endpoint] git_process = Popen( args=git_args, close_fds=True, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) timeout = Timeout(20.0) timeout.start() try: git_stdout, git_stderr = git_process.communicate() except Timeout: git_process.kill() return jsonify(error={'message': 'Timeout accessing git repo'}), 400 finally: timeout.cancel() if git_process.returncode != 0: LOG.error('Error running {}: {}'.format(git_args, git_stderr)) return jsonify(error={'message': 'Error running git: {}'.format(git_stderr)}), 400 return jsonify(result=[line.rsplit('/', 1)[-1] for line in git_stdout.splitlines()])
def get_git_refs(): if DISABLE_NEW_EXTENSIONS: return 'Disabled', 403 git_endpoint = request.values.get('ep', None) if git_endpoint is None: return jsonify(error={'message': 'Missing endpoint'}), 400 if not git_endpoint.endswith('.git'): return jsonify(error={'message': 'Invalid git endpoint'}), 400 git_path = config.get('MINEMELD_GIT_PATH', None) if git_path is None: return jsonify(error={'message': 'MINEMELD_GIT_PATH not set'}), 500 git_args = [git_path, 'ls-remote', '-t', '-h', git_endpoint] git_process = Popen(args=git_args, close_fds=True, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) timeout = Timeout(20.0) timeout.start() try: git_stdout, git_stderr = git_process.communicate() except Timeout: git_process.kill() return jsonify(error={'message': 'Timeout accessing git repo'}), 400 finally: timeout.cancel() if git_process.returncode != 0: LOG.error('Error running {}: {}'.format(git_args, git_stderr)) return jsonify( error={'message': 'Error running git: {}'.format(git_stderr)}), 400 return jsonify( result=[line.rsplit('/', 1)[-1] for line in git_stdout.splitlines()])
def delete(self, name): self.logger.debug("Preparing to execute delete()") timeout = Timeout(self.script_timeout) process = Popen(self.scriptfile_path + " -d -q" + " -n " + name, stdout=PIPE, shell=True) output = "Delete operation exceeded execution timeout.\n" returncode = 1 timeout.start() try: output = process.communicate()[0] returncode = process.returncode except Timeout: process.kill() self.logger.warn( "Process %s servicing delete() " + "exceeded execution timeout and was terminated.", process.pid) if process.returncode is not None: returncode = process.returncode finally: timeout.cancel() return [output, returncode]
def startStreamReader(self, url, cid, counter, req_headers=None): logger = logging.getLogger('StreamReader') logger.debug('Open video stream: %s' % url) self._streamReaderState = 1 transcoder = None if 'range' in req_headers: del req_headers['range'] logger.debug('Get headers from client: %s' % req_headers) with requests.get(url, headers=req_headers, stream=True, timeout=(5, None)) as self._streamReaderConnection: try: if self._streamReaderConnection.status_code not in (200, 206): logger.error('Failed to open video stream %s' % url) return None if url.endswith('.m3u8'): self._streamReaderConnection.headers = { 'Content-Type': 'application/octet-stream', 'Connection': 'Keep-Alive', 'Keep-Alive': 'timeout=15, max=100' } popen_params = { "bufsize": AceConfig.readchunksize, "stdout": PIPE, "stderr": None, "shell": False } if AceConfig.osplatform == 'Windows': ffmpeg_cmd = 'ffmpeg.exe ' CREATE_NO_WINDOW = 0x08000000 CREATE_NEW_PROCESS_GROUP = 0x00000200 DETACHED_PROCESS = 0x00000008 popen_params.update(creationflags=CREATE_NO_WINDOW | DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP) else: ffmpeg_cmd = 'ffmpeg ' ffmpeg_cmd += '-hwaccel auto -hide_banner -loglevel fatal -re -i %s -c copy -f mpegts -' % url transcoder = Popen(ffmpeg_cmd.split(), **popen_params) out = transcoder.stdout logger.warning( 'HLS stream detected. Ffmpeg transcoding started') else: out = self._streamReaderConnection.raw except requests.exceptions.RequestException: logger.error('Failed to open video stream %s' % url) logger.error(traceback.format_exc()) except: logger.error(traceback.format_exc()) else: with self._lock: self._streamReaderState = 2 self._lock.notifyAll() self.play_event() while 1: self.getPlayEvent( ) # Wait for PlayEvent (stop/resume sending data from AceEngine to streamReaderQueue) clients = counter.getClients(cid) try: data = out.read(AceConfig.readchunksize) except: data = None if data is not None and clients: if self._streamReaderQueue.full(): self._streamReaderQueue.get() self._streamReaderQueue.put(data) for c in clients: try: c.queue.put(data, timeout=5) except gevent.queue.Full: #Queue.Full client does not read data from buffer until 5sec - disconnect it if len(clients) > 1: logger.debug('Disconnecting client: %s' % c.handler.clientip) c.destroy() elif counter.count(cid) == 0: logger.debug( 'All clients disconnected - broadcast stoped') break else: logger.warning('No data received - broadcast stoped') counter.deleteAll(cid) break finally: with self._lock: self._streamReaderState = None self._lock.notifyAll() if transcoder: try: transcoder.kill() logger.warning('Ffmpeg transcoding stoped') except: pass
def run(command, use_sudo=False, user='', group='', freturn=False, err_to_out=False, input=None, force=False, **kwargs): """Execute command on host via ssh or subprocess. TODO: check on windows - maybe it will not work on it Factory uses pipes for communication with subprocess. So, there is no way to use popen and automatically write passwords for ssh and sudo on localhost, because "smart" programs like ssh and sudo uses tty directly. Also active tty required (needed check it) and for sudo uses "sudo -S". Alternatives: 1) Use paramico like fabric = no ssh sockets. 2) Use pty.fork, waitpid, execv as pexcpect and sh = only unix, no separated stderr, hard to communicate. 3) Use ssh-copy-id like sh module recommended = ask passwords only one first time. 4) Use sshpass like ansible = external dependencies. 5) Use local ssh server and run all commands through it instead of popen. Args: command (str): command for executing use_sudo (bool): running with sudo prefix if True and current user not root, default is False user (str): username for sudo -u prefix group (str): group for sudo -g prefix freturn (bool): return tuple if True, else return str, default is False err_to_out (bool): redirect stderr to stdout if True, default is False input (str or tuple of str): str will be flushed to stdin after executed command, default is None force (bool): executing full operations even if envs.common.dry_run is True **kwargs (dict): add only for supporting dry-run replacing Return: str if freturn is False: string that contained all stdout messages tuple if freturn is True: string that contained all stdout messages string that contained all stderr int that mean return code of command """ # hack for dry-run if envs.common.dry_run and not force: from dry_operations import run return run(command, use_sudo, user, group, freturn, err_to_out, input, **kwargs) logger = envs.connect.logger interactive = envs.common.interactive parallel = envs.common.parallel host_string = ''.join((envs.connect.user, '@', envs.connect.host)) logger.debug('executing run function') logger.debug('arguments for executing and another locals: %s', locals()) command = command_patching_for_sudo(command, use_sudo, user, group) # logging write_message_to_log(command, 'in: ') stderr = PIPE if err_to_out: stderr = STDOUT logger.debug('stderr: %s', stderr) # open new connect if envs.connect.host in envs.common.localhost: logger.debug('executing command %s with shell=True', command) p = Popen(command, stdout=PIPE, stderr=stderr, stdin=PIPE, shell=True) else: scommand = [ envs.common.ssh_binary, envs.common.ssh_port_option, str(envs.connect.port), host_string, ] scommand += envs.common.ssh_args.split() scommand += envs.connect.con_args.split() scommand += [command] logger.debug('executing command %s', scommand) p = Popen(scommand, stdout=PIPE, stderr=stderr, stdin=PIPE) # flush input if input: if type(input) is str: input = [input] for s in input: s = str(s) if s[-1] not in ('\n', '\r'): s += '\n' logger.debug('flushing input %s', s) p.stdin.write(s) p.stdin.flush() # run another command if parallel: gevent.sleep(0) logger.debug('run another command with gevent.sleep(0)') # processing std loop threads = [] if interactive: args = (p, copy(envs.common), copy(envs.connect)) gin = gevent.spawn(in_loop, *args) logger.debug('executing in_loop with args %s', args) threads.append(gin) args = (p, copy(envs.common), copy(envs.connect)) gout = gevent.spawn(out_loop, *args) logger.debug('executing out_loop with args %s', args) threads.append(gout) if not err_to_out: args = (p, copy(envs.common), copy(envs.connect), True) gerr = gevent.spawn(out_loop, *args) logger.debug('executing err_loop with args %s', args) threads.append(gerr) gevent.joinall(threads) logger.debug('child process has terminated with status %s', p.returncode) #TODO: check returncode if returncode==None sumout = gout.value sumerr = gerr.value if not err_to_out else '' status = p.returncode if p.poll() is None: p.terminate() p.kill() if freturn: logger.debug('return sumout %s, sumerr %s, status %s', sumout, sumerr, status) return (sumout, sumerr, status) logger.debug('return sumout %s', sumout) return sumout
def handle(self, fmt=None): logger = logging.getLogger("ClientHandler") self.connectionTime = time.time() with self.ace._lock: start = time.time() while self.handler.connection and self.ace._streamReaderState == 1: remaining = start + 5.0 - time.time() if remaining > 0: self.ace._lock.wait(remaining) else: self.handler.dieWithError( 500, 'Video stream not opened in 5 seconds - disconnecting') return if self.handler.connection and self.ace._streamReaderState != 2: self.handler.dieWithError( 500, 'No video stream received from AceEngine', logging.WARNING) return # Sending client headers to videostream if self.handler.connection: self.handler.send_response( self.ace._streamReaderConnection.status_code) FORWARD_HEADERS = [ 'Connection', 'Keep-Alive', 'Content-Range', 'Content-Type', 'X-Content-Duration', 'Content-Length' ] SKIP_HEADERS = [ 'Server', 'Date', 'Transfer-Encoding', 'Accept-Ranges' ] new_headers = { k: v for (k, v) in self.ace._streamReaderConnection.headers.items() if k not in (FORWARD_HEADERS + SKIP_HEADERS) } if new_headers: logger.error('NEW HEADERS FOUND: %s' % new_headers) response_headers = { k: v for (k, v) in self.ace._streamReaderConnection.headers.items() if k not in SKIP_HEADERS } for h in response_headers: self.handler.send_header(h, response_headers[h]) self.handler.end_headers() logger.debug('Sending HTTPAceProxy headers to client: %s' % response_headers) transcoder = None out = self.handler.wfile if AceConfig.transcode and fmt and AceConfig.osplatform != 'Windows': if fmt in AceConfig.transcodecmd: stderr = None if AceConfig.loglevel == logging.DEBUG else DEVNULL popen_params = { "bufsize": AceConfig.readchunksize, "stdin": PIPE, "stdout": self.handler.wfile, "stderr": stderr, "shell": False } transcoder = Popen(AceConfig.transcodecmd[fmt], **popen_params) gevent.sleep() out = transcoder.stdin logger.warning('Ffmpeg transcoding started') else: logger.error( "Can't found fmt key. Ffmpeg transcoding not started !") try: while self.handler.connection and self.ace._streamReaderState == 2: try: data = self.queue.get(timeout=60) try: out.write(data) except: break except gevent.queue.Empty: logger.warning( 'No data received in 60 seconds - disconnecting "%s"' % self.channelName) break finally: if transcoder: try: transcoder.kill() logger.warning('Ffmpeg transcoding stoped') except: pass return