def create(self, usernames, passwords, target_iqn, target_lun, size,
initiator_iqn_list):
# NB: initiator_iqn_list needs to be a comma separated list of initiator iqn strings
self.logger.debug("Preparing to execute create()")
timeout = Timeout(self.script_timeout)
process = Popen(self.scriptfile_path + " -c -q" + " -u " + usernames +
" -p " + passwords + " -s " + size + " -m " +
target_lun + " -t " + target_iqn + " -i " +
initiator_iqn_list,
stdout=PIPE,
shell=True)
output = "Create operation exceeded execution timeout.\n"
returncode = 1
timeout.start()
try:
output = process.communicate()[0]
returncode = process.returncode
except Timeout:
process.kill()
self.logger.warn(
"Process %s servicing create() " +
"exceeded execution timeout and was terminated.", process.pid)
if process.returncode is not None:
returncode = process.returncode
finally:
timeout.cancel()
return [output, returncode]
def proc(input, output, args, env=None, stderr=None):
"""Run a subprocess, passing input to its stdin and sending its stdout to output,
with each item newline-seperated.
Args is either a string to be shell-interpreted, or a list of args.
stderr is either not redirected (default), mixed in with stdout (pass subprocess.STDOUT),
or redirected to a given file.
"""
from gevent.subprocess import Popen, PIPE, STDOUT
if isinstance(args, unicode):
args = args.encode('utf8')
if isinstance(args, str):
shell = True
else:
shell = False
group = gevent.pool.Group()
proc = None
try:
proc = Popen(args, shell=shell, env=env, stdin=PIPE, stdout=PIPE, stderr=stderr)
@group.spawn
def do_input():
for item in input:
item = item.encode('utf8') if isinstance(item, unicode) else str(item)
proc.stdin.write('{}\n'.format(item))
proc.stdin.flush()
proc.stdin.close()
@group.spawn
def do_output():
for line in proc.stdout:
output.put(line.rstrip('\n'))
output.close()
proc.wait()
group.join()
finally:
if proc and proc.poll() is None:
try:
proc.kill()
except OSError as e:
if e.errno != errno.ESRCH:
raise
def get_git_refs():
if DISABLE_NEW_EXTENSIONS:
return 'Disabled', 403
git_endpoint = request.values.get('ep', None)
if git_endpoint is None:
return jsonify(error={'message': 'Missing endpoint'}), 400
if not git_endpoint.endswith('.git'):
return jsonify(error={'message': 'Invalid git endpoint'}), 400
git_path = config.get('MINEMELD_GIT_PATH', None)
if git_path is None:
return jsonify(error={'message': 'MINEMELD_GIT_PATH not set'}), 500
git_args = [git_path, 'ls-remote', '-t', '-h', git_endpoint]
git_process = Popen(
args=git_args,
close_fds=True,
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)
timeout = Timeout(20.0)
timeout.start()
try:
git_stdout, git_stderr = git_process.communicate()
except Timeout:
git_process.kill()
return jsonify(error={'message': 'Timeout accessing git repo'}), 400
finally:
timeout.cancel()
if git_process.returncode != 0:
LOG.error('Error running {}: {}'.format(git_args, git_stderr))
return jsonify(error={'message': 'Error running git: {}'.format(git_stderr)}), 400
return jsonify(result=[line.rsplit('/', 1)[-1] for line in git_stdout.splitlines()])
def get_git_refs():
if DISABLE_NEW_EXTENSIONS:
return 'Disabled', 403
git_endpoint = request.values.get('ep', None)
if git_endpoint is None:
return jsonify(error={'message': 'Missing endpoint'}), 400
if not git_endpoint.endswith('.git'):
return jsonify(error={'message': 'Invalid git endpoint'}), 400
git_path = config.get('MINEMELD_GIT_PATH', None)
if git_path is None:
return jsonify(error={'message': 'MINEMELD_GIT_PATH not set'}), 500
git_args = [git_path, 'ls-remote', '-t', '-h', git_endpoint]
git_process = Popen(args=git_args,
close_fds=True,
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
timeout = Timeout(20.0)
timeout.start()
try:
git_stdout, git_stderr = git_process.communicate()
except Timeout:
git_process.kill()
return jsonify(error={'message': 'Timeout accessing git repo'}), 400
finally:
timeout.cancel()
if git_process.returncode != 0:
LOG.error('Error running {}: {}'.format(git_args, git_stderr))
return jsonify(
error={'message': 'Error running git: {}'.format(git_stderr)}), 400
return jsonify(
result=[line.rsplit('/', 1)[-1] for line in git_stdout.splitlines()])
def delete(self, name):
self.logger.debug("Preparing to execute delete()")
timeout = Timeout(self.script_timeout)
process = Popen(self.scriptfile_path + " -d -q" + " -n " + name,
stdout=PIPE,
shell=True)
output = "Delete operation exceeded execution timeout.\n"
returncode = 1
timeout.start()
try:
output = process.communicate()[0]
returncode = process.returncode
except Timeout:
process.kill()
self.logger.warn(
"Process %s servicing delete() " +
"exceeded execution timeout and was terminated.", process.pid)
if process.returncode is not None:
returncode = process.returncode
finally:
timeout.cancel()
return [output, returncode]
def startStreamReader(self, url, cid, counter, req_headers=None):
logger = logging.getLogger('StreamReader')
logger.debug('Open video stream: %s' % url)
self._streamReaderState = 1
transcoder = None
if 'range' in req_headers: del req_headers['range']
logger.debug('Get headers from client: %s' % req_headers)
with requests.get(url,
headers=req_headers,
stream=True,
timeout=(5, None)) as self._streamReaderConnection:
try:
if self._streamReaderConnection.status_code not in (200, 206):
logger.error('Failed to open video stream %s' % url)
return None
if url.endswith('.m3u8'):
self._streamReaderConnection.headers = {
'Content-Type': 'application/octet-stream',
'Connection': 'Keep-Alive',
'Keep-Alive': 'timeout=15, max=100'
}
popen_params = {
"bufsize": AceConfig.readchunksize,
"stdout": PIPE,
"stderr": None,
"shell": False
}
if AceConfig.osplatform == 'Windows':
ffmpeg_cmd = 'ffmpeg.exe '
CREATE_NO_WINDOW = 0x08000000
CREATE_NEW_PROCESS_GROUP = 0x00000200
DETACHED_PROCESS = 0x00000008
popen_params.update(creationflags=CREATE_NO_WINDOW
| DETACHED_PROCESS
| CREATE_NEW_PROCESS_GROUP)
else:
ffmpeg_cmd = 'ffmpeg '
ffmpeg_cmd += '-hwaccel auto -hide_banner -loglevel fatal -re -i %s -c copy -f mpegts -' % url
transcoder = Popen(ffmpeg_cmd.split(), **popen_params)
out = transcoder.stdout
logger.warning(
'HLS stream detected. Ffmpeg transcoding started')
else:
out = self._streamReaderConnection.raw
except requests.exceptions.RequestException:
logger.error('Failed to open video stream %s' % url)
logger.error(traceback.format_exc())
except:
logger.error(traceback.format_exc())
else:
with self._lock:
self._streamReaderState = 2
self._lock.notifyAll()
self.play_event()
while 1:
self.getPlayEvent(
) # Wait for PlayEvent (stop/resume sending data from AceEngine to streamReaderQueue)
clients = counter.getClients(cid)
try:
data = out.read(AceConfig.readchunksize)
except:
data = None
if data is not None and clients:
if self._streamReaderQueue.full():
self._streamReaderQueue.get()
self._streamReaderQueue.put(data)
for c in clients:
try:
c.queue.put(data, timeout=5)
except gevent.queue.Full: #Queue.Full client does not read data from buffer until 5sec - disconnect it
if len(clients) > 1:
logger.debug('Disconnecting client: %s' %
c.handler.clientip)
c.destroy()
elif counter.count(cid) == 0:
logger.debug(
'All clients disconnected - broadcast stoped')
break
else:
logger.warning('No data received - broadcast stoped')
counter.deleteAll(cid)
break
finally:
with self._lock:
self._streamReaderState = None
self._lock.notifyAll()
if transcoder:
try:
transcoder.kill()
logger.warning('Ffmpeg transcoding stoped')
except:
pass
def run(command, use_sudo=False, user='', group='', freturn=False, err_to_out=False, input=None, force=False, **kwargs):
"""Execute command on host via ssh or subprocess.
TODO: check on windows - maybe it will not work on it
Factory uses pipes for communication with subprocess.
So, there is no way to use popen and automatically write passwords for ssh and sudo on localhost,
because "smart" programs like ssh and sudo uses tty directly.
Also active tty required (needed check it) and for sudo uses "sudo -S".
Alternatives:
1) Use paramico like fabric = no ssh sockets.
2) Use pty.fork, waitpid, execv as pexcpect and sh = only unix, no separated stderr, hard to communicate.
3) Use ssh-copy-id like sh module recommended = ask passwords only one first time.
4) Use sshpass like ansible = external dependencies.
5) Use local ssh server and run all commands through it instead of popen.
Args:
command (str): command for executing
use_sudo (bool): running with sudo prefix if True and current user not root, default is False
user (str): username for sudo -u prefix
group (str): group for sudo -g prefix
freturn (bool): return tuple if True, else return str, default is False
err_to_out (bool): redirect stderr to stdout if True, default is False
input (str or tuple of str): str will be flushed to stdin after executed command, default is None
force (bool): executing full operations even if envs.common.dry_run is True
**kwargs (dict): add only for supporting dry-run replacing
Return:
str if freturn is False: string that contained all stdout messages
tuple if freturn is True:
string that contained all stdout messages
string that contained all stderr
int that mean return code of command
"""
# hack for dry-run
if envs.common.dry_run and not force:
from dry_operations import run
return run(command, use_sudo, user, group, freturn, err_to_out, input, **kwargs)
logger = envs.connect.logger
interactive = envs.common.interactive
parallel = envs.common.parallel
host_string = ''.join((envs.connect.user,
'@',
envs.connect.host))
logger.debug('executing run function')
logger.debug('arguments for executing and another locals: %s', locals())
command = command_patching_for_sudo(command, use_sudo, user, group)
# logging
write_message_to_log(command, 'in: ')
stderr = PIPE
if err_to_out:
stderr = STDOUT
logger.debug('stderr: %s', stderr)
# open new connect
if envs.connect.host in envs.common.localhost:
logger.debug('executing command %s with shell=True', command)
p = Popen(command, stdout=PIPE, stderr=stderr, stdin=PIPE, shell=True)
else:
scommand = [
envs.common.ssh_binary,
envs.common.ssh_port_option,
str(envs.connect.port),
host_string,
]
scommand += envs.common.ssh_args.split()
scommand += envs.connect.con_args.split()
scommand += [command]
logger.debug('executing command %s', scommand)
p = Popen(scommand, stdout=PIPE, stderr=stderr, stdin=PIPE)
# flush input
if input:
if type(input) is str:
input = [input]
for s in input:
s = str(s)
if s[-1] not in ('\n', '\r'):
s += '\n'
logger.debug('flushing input %s', s)
p.stdin.write(s)
p.stdin.flush()
# run another command
if parallel:
gevent.sleep(0)
logger.debug('run another command with gevent.sleep(0)')
# processing std loop
threads = []
if interactive:
args = (p, copy(envs.common), copy(envs.connect))
gin = gevent.spawn(in_loop, *args)
logger.debug('executing in_loop with args %s', args)
threads.append(gin)
args = (p, copy(envs.common), copy(envs.connect))
gout = gevent.spawn(out_loop, *args)
logger.debug('executing out_loop with args %s', args)
threads.append(gout)
if not err_to_out:
args = (p, copy(envs.common), copy(envs.connect), True)
gerr = gevent.spawn(out_loop, *args)
logger.debug('executing err_loop with args %s', args)
threads.append(gerr)
gevent.joinall(threads)
logger.debug('child process has terminated with status %s', p.returncode)
#TODO: check returncode if returncode==None
sumout = gout.value
sumerr = gerr.value if not err_to_out else ''
status = p.returncode
if p.poll() is None:
p.terminate()
p.kill()
if freturn:
logger.debug('return sumout %s, sumerr %s, status %s', sumout, sumerr, status)
return (sumout, sumerr, status)
logger.debug('return sumout %s', sumout)
return sumout
def handle(self, fmt=None):
logger = logging.getLogger("ClientHandler")
self.connectionTime = time.time()
with self.ace._lock:
start = time.time()
while self.handler.connection and self.ace._streamReaderState == 1:
remaining = start + 5.0 - time.time()
if remaining > 0: self.ace._lock.wait(remaining)
else:
self.handler.dieWithError(
500,
'Video stream not opened in 5 seconds - disconnecting')
return
if self.handler.connection and self.ace._streamReaderState != 2:
self.handler.dieWithError(
500, 'No video stream received from AceEngine',
logging.WARNING)
return
# Sending client headers to videostream
if self.handler.connection:
self.handler.send_response(
self.ace._streamReaderConnection.status_code)
FORWARD_HEADERS = [
'Connection', 'Keep-Alive', 'Content-Range', 'Content-Type',
'X-Content-Duration', 'Content-Length'
]
SKIP_HEADERS = [
'Server', 'Date', 'Transfer-Encoding', 'Accept-Ranges'
]
new_headers = {
k: v
for (k, v) in self.ace._streamReaderConnection.headers.items()
if k not in (FORWARD_HEADERS + SKIP_HEADERS)
}
if new_headers: logger.error('NEW HEADERS FOUND: %s' % new_headers)
response_headers = {
k: v
for (k, v) in self.ace._streamReaderConnection.headers.items()
if k not in SKIP_HEADERS
}
for h in response_headers:
self.handler.send_header(h, response_headers[h])
self.handler.end_headers()
logger.debug('Sending HTTPAceProxy headers to client: %s' %
response_headers)
transcoder = None
out = self.handler.wfile
if AceConfig.transcode and fmt and AceConfig.osplatform != 'Windows':
if fmt in AceConfig.transcodecmd:
stderr = None if AceConfig.loglevel == logging.DEBUG else DEVNULL
popen_params = {
"bufsize": AceConfig.readchunksize,
"stdin": PIPE,
"stdout": self.handler.wfile,
"stderr": stderr,
"shell": False
}
transcoder = Popen(AceConfig.transcodecmd[fmt], **popen_params)
gevent.sleep()
out = transcoder.stdin
logger.warning('Ffmpeg transcoding started')
else:
logger.error(
"Can't found fmt key. Ffmpeg transcoding not started !")
try:
while self.handler.connection and self.ace._streamReaderState == 2:
try:
data = self.queue.get(timeout=60)
try:
out.write(data)
except:
break
except gevent.queue.Empty:
logger.warning(
'No data received in 60 seconds - disconnecting "%s"' %
self.channelName)
break
finally:
if transcoder:
try:
transcoder.kill()
logger.warning('Ffmpeg transcoding stoped')
except:
pass
return
