args = (binName + ' -sharable ' + fileHash + ' -progress')
#function to pash json
def parseJosnOutput(stringObject):
jsonObject = json.dumps(stringObject)
jsonObject1 = json.loads(jsonObject)
return jsonObject
#function to convert non json output to json.
def pasreNonJsonOutput(stringObject):
if stringObject[0] == "P":
stringArr = stringObject.split(" ")
return {
"percentage": stringArr[1][0],
"downloaded": stringArr[2][1:],
"size": stringArr[4][:-1]
}
else:
return stringObject
# //executing the command and assign to p
with Popen(args, stdout=PIPE, bufsize=1, universal_newlines=True) as p:
for b in p.stdout: #reading the stdout and printing
print(pasreNonJsonOutput(b))
if p.returncode != 0:
raise CalledProcessError(p.returncode, p.args)
def call_effect(*args, **kwargs):
if args[0][-3:] == ["snap", "watch", "--last=auto-refresh"]:
raise CalledProcessError(returncode=1, cmd=args[0])
return self.fake_lxd.check_output_side_effect()(*args, **kwargs)
def call_effect(*args, **kwargs):
if args[0][:2] == ["lxc", "start"]:
raise CalledProcessError(returncode=255, cmd=args[0])
return d(*args, **kwargs)
def shutdown(self):
self.process.stdin.close()
if self.process.wait() != 0:
raise CalledProcessError(self.process.returncode, self.command)
def call_effect(*args, **kwargs):
if "snapcraft snap --output snap.snap" in " ".join(args[0]):
raise CalledProcessError(returncode=255, cmd=args[0])
return self.fake_lxd.check_output_side_effect()(*args, **kwargs)
def execute(self):
if self.static:
self.disabled('login')
# Collect the list of running hosts.
try:
status = self.molecule._provisioner.status()
except Exception as e:
status = []
# make sure vagrant knows about this host
try:
# Nowhere to log into if there is no running host.
if len(status) == 0:
raise InvalidHost("There is no running host.")
# Check whether a host was specified.
if self.molecule._args['<host>'] is None:
# One running host is perfect. Log into it.
if len(status) == 1:
hostname = status[0].name
# But too many hosts is trouble as well.
else:
raise InvalidHost(
"There are {} running hosts. You can only log into one at a time."
.format(len(status)))
else:
# If the host was specified, try to use it.
hostname = self.molecule._args['<host>']
match = [x.name for x in status if x.name.startswith(hostname)]
if len(match) == 0:
raise CalledProcessError(1, None)
elif len(match) != 1:
raise InvalidHost(
"There are {} hosts that match '{}'. You can only log into one at a time.\n"
"Try {}molecule status{} to see available hosts.".
format(len(match), hostname, Fore.YELLOW, Fore.RED))
hostname = match[0]
# Try to retrieve the SSH configuration of the host.
conf = self.molecule._provisioner.conf(vm_name=hostname)
# Prepare the command to SSH into the host.
ssh_args = [
conf['HostName'], conf['User'], conf['Port'],
conf['IdentityFile'], ' '.join(
self.molecule._config.config['molecule']['raw_ssh_args'])
]
ssh_cmd = 'ssh {} -l {} -p {} -i {} {}'
except CalledProcessError:
# gets appended to python-vagrant's error message
conf_format = [
Fore.RED, self.molecule._args['<host>'], Fore.YELLOW,
Fore.RESET
]
conf_errmsg = '\n{0}Unknown host {1}. Try {2}molecule status{0} to see available hosts.{3}'
print(conf_errmsg.format(*conf_format))
sys.exit(1)
except InvalidHost as e:
conf_format = [Fore.RED, e.message, Fore.RESET]
conf_errmsg = '{}{}{}'
print(conf_errmsg.format(*conf_format))
sys.exit(1)
lines, columns = os.popen('stty size', 'r').read().split()
dimensions = (int(lines), int(columns))
self.molecule._pt = pexpect.spawn('/usr/bin/env ' +
ssh_cmd.format(*ssh_args),
dimensions=dimensions)
signal.signal(signal.SIGWINCH, self.molecule._sigwinch_passthrough)
self.molecule._pt.interact()
return None, None
def oscap(profile=None):
# If FIFO exists, delete it
try:
unlink(FIFO_PATH)
except OSError:
pass
# Create an unique FIFO file
mkfifo(FIFO_PATH, 0666)
try:
cmd = [OSCAP_BIN, arg_module, 'eval', '--results', FIFO_PATH]
if profile:
cmd.extend(["--profile", profile])
if arg_xccdfid:
for arg_id in arg_xccdfid:
cmd.extend(["--xccdf-id", arg_id])
if arg_ovalid:
for arg_id in arg_ovalid:
cmd.extend(["--oval-id", arg_id])
if arg_dsid:
for arg_id in arg_dsid:
cmd.extend(["--datastream-id", arg_id])
if arg_cpe:
cmd.extend(["--cpe", arg_cpe])
cmd.append(arg_file)
if debug:
print("\nCMD: '{0}'".format(' '.join(cmd)))
DEVNULL = open(devnull, 'wb')
ps = Popen(cmd, shell=False, stdout=DEVNULL, stderr=None)
except CalledProcessError as error:
# return code 2 means that some checks failed
if error.returncode != 2:
# output = error.output
print(
"{0} Executing profile \"{1}\" of file \"{2}\": Return Code: \"{3}\" Error: \"{4}\"."
.format(OSCAP_LOG_ERROR, profile, arg_file, error.returncode,
error.output.replace('\r', '').split("\n")[0]))
unlink(FIFO_PATH)
return
try:
content_filename = arg_file.split('/')[-1]
# Generate scan ID: agent_id + epoch
try:
if isdir('ruleset'):
agent_id = '000'
else:
with open('etc/client.keys', 'r') as f:
first_line = f.readline()
agent_id = first_line.split(' ')[0]
except:
agent_id = randrange(1, 9999)
scan_id = "{0}{1}".format(agent_id, int(time()))
if arg_module == 'xccdf':
ps_xsltproc = Popen([XSLT_BIN, TEMPLATE_XCCDF, FIFO_PATH],
stdin=None,
stdout=PIPE,
stderr=STDOUT)
ps.wait()
output = ps_xsltproc.communicate()[0]
ps_xsltproc.wait()
returncode = ps_xsltproc.returncode
if returncode != 0:
error_cmd = CalledProcessError(
returncode, [XSLT_BIN, TEMPLATE_XCCDF, FIFO_PATH])
error_cmd.output = output
raise error_cmd
else:
if version_info[0] >= 3:
output = output.decode('utf-8', 'backslashreplace')
for line in output.split("\n"):
if not line:
continue
# Adding file
if 'msg: "xccdf-overview"' in line:
new_line = line.replace(
'oscap: msg: "xccdf-overview",',
'oscap: msg: "xccdf-overview", scan-id: "{0}", content: "{1}",'
.format(scan_id, content_filename))
else:
new_line = line.replace(
'oscap: msg: "xccdf-result",',
'oscap: msg: "xccdf-result", scan-id: "{0}", content: "{1}",'
.format(scan_id, content_filename))
print(new_line)
else:
ps_xsltproc = Popen([XSLT_BIN, TEMPLATE_OVAL, FIFO_PATH],
stdin=None,
stdout=PIPE,
stderr=STDOUT)
ps.wait()
output = ps_xsltproc.communicate()[0]
ps_xsltproc.wait()
returncode = ps_xsltproc.returncode
if version_info[0] >= 3:
output = output.decode('utf-8', 'backslashreplace')
total = 0
total_KO = 0
for line in output.split("\n"):
if not line:
continue
total += 1
# Adding file
new_line = line.replace(
'oscap: msg: "oval-result"',
'oscap: msg: "oval-result", scan-id: "{0}", content: "{1}"'
.format(scan_id, content_filename))
class1 = ['class: "compliance"', 'class: "inventory"']
class2 = ['class: "vulnerability"', 'class: "patch"']
if any(x in line for x in class1):
if 'result: "false"' in line:
total_KO += 1
new_line = new_line.replace('result: "false"',
'result: "fail"')
elif 'result: "true"' in line:
new_line = new_line.replace('result: "true"',
'result: "pass"')
elif any(x in line for x in class2):
if 'result: "true"' in line:
total_KO += 1
new_line = new_line.replace('result: "true"',
'result: "fail"')
elif 'result: "false"' in line:
new_line = new_line.replace('result: "false"',
'result: "pass"')
new_line = new_line.replace('", class: "',
'", profile-title: "')
print(new_line)
score = (float((total - total_KO)) / float(total)) * 100
# summary
print(
'oscap: msg: "oval-overview", scan-id: "{0}", content: "{1}", score: "{2:.2f}".'
.format(scan_id, content_filename, score))
except CalledProcessError as error:
print(
"{0} Formatting data for profile \"{1}\" of file \"{2}\": Return Code: \"{3}\" Error: \"{4}\"."
.format(OSCAP_LOG_ERROR, profile, arg_file, error.returncode,
error.output.replace('\r', '').split("\n")[0]))
unlink(FIFO_PATH)
def test_terminate_vm_on_error(self, mc_run_ans):
mc_run_ans.side_effect = CalledProcessError(0, cmd=["ls"])
# doesn't raise an error
terminate_vm(self.opts, self.terminate_pb_path, 0, self.vm_name, self.vm_ip)
def test_raises_if_token_file_not_present(self):
client = Mock()
client.get_juju_output.side_effect = CalledProcessError(-1, None, '')
with self.assertRaises(JujuAssertionError):
aps.get_stored_token_content(client)
def mock_check_output(call, shell=True, stderr=None):
raise CalledProcessError(1, call, b'', stderr)
def fake_run(process_args, *args, **kwargs):
if process_args == ["terraform", "init", "-no-color", "-input=false"]:
raise CalledProcessError(1, "terraform init")
def run(self,
command,
debug=False,
suppress_logging=False,
allow_nonzero=False,
allow_insecure=False):
"""
Executes a shell command
:param suppress_logging: Do not log anything
:param command: Command to execute
:param debug: Extended logging and stderr output returned
:param allow_nonzero: Allow non-zero exit code
:param allow_insecure: Allow string commands (which might be inproper escaped)
"""
if self._unittest_mode is True:
_command = command
if isinstance(command, list):
_command = ' '.join(command)
SSHClient._logger.debug('Executing: {0}'.format(_command))
SSHClient._run_recordings.append(_command)
if _command in SSHClient._run_returns:
SSHClient._logger.debug('Emulating return value')
return SSHClient._run_returns[_command]
if not isinstance(command, list) and not allow_insecure:
raise RuntimeError(
'The given command must be a list, or the allow_insecure flag must be set'
)
if isinstance(command, list):
command = ' '.join(
[self.shell_safe(str(entry)) for entry in command])
if self.is_local is True:
stderr = None
try:
try:
if not hasattr(select, 'poll'):
import subprocess
subprocess._has_poll = False # Damn 'monkey patching'
channel = Popen(command,
stdout=PIPE,
stderr=PIPE,
shell=not isinstance(command, list))
except OSError as ose:
raise CalledProcessError(1, command, str(ose))
stdout, stderr = channel.communicate()
stdout = self._clean_text(stdout)
stderr = self._clean_text(stderr)
exit_code = channel.returncode
if exit_code != 0 and allow_nonzero is False: # Raise same error as check_output
raise CalledProcessError(exit_code, command, stdout)
if debug:
SSHClient._logger.debug('stdout: {0}'.format(stdout))
SSHClient._logger.debug('stderr: {0}'.format(stderr))
return stdout, stderr
else:
return stdout
except CalledProcessError as cpe:
if suppress_logging is False:
SSHClient._logger.error(
'Command "{0}" failed with output "{1}"{2}'.format(
command, cpe.output, '' if stderr is None else
' and error "{0}"'.format(stderr)))
raise
else:
_, stdout, stderr = self._client.exec_command(
command) # stdin, stdout, stderr
output = self._clean_text(stdout.readlines())
error = self._clean_text(stderr.readlines())
exit_code = stdout.channel.recv_exit_status()
if exit_code != 0 and allow_nonzero is False: # Raise same error as check_output
if suppress_logging is False:
SSHClient._logger.error(
'Command "{0}" failed with output "{1}" and error "{2}"'
.format(command, output, error))
raise CalledProcessError(exit_code, command, output)
if debug:
return output, error
else:
return output
def check_process(proc):
wait_for_process(proc)
if proc.returncode != 0:
raise CalledProcessError(proc.returncode, proc.command_string)
@patch('subprocess.check_output')
def test_run_delegates_to_shell(check_output):
command_runner.run('ls -l -a')
check_output.assert_called_with(['ls', '-l', '-a'],
universal_newlines=True)
@patch('subprocess.check_output', return_value='test\n')
def test_run_returns_command_output_without_trailing_newline(check_output):
assert command_runner.run('ls') == 'test'
@patch('subprocess.check_output')
@patch('ticket.cli.output.show')
def test_run_prints_the_command_to_run(show, check_output):
command_runner.run('ls')
show.assert_called_with('ls')
@patch('subprocess.check_output', side_effect=OSError())
def test_run_raises_an_exception_if_the_command_does_not_exist(check_output):
with pytest.raises(CommandNotFoundError):
command_runner.run('asdfg')
@patch('subprocess.check_output',
side_effect=CalledProcessError(2, ['git', 'unknownCommand']))
def test_run_raises_an_exception_if_the_command_fails(check_output):
with pytest.raises(CommandFailedError):
command_runner.run('git unknownCommand')
def mock_check_output_error(args_list, **kwargs):
"""
mock for subprocess.check_output, returns a dummy error output of pqos_wrapper
"""
raise CalledProcessError(1, "cmd", json.dumps(mock_pqos_wrapper_error).encode())
def raise_process_exception(msg, *_a, **_kw):
raise CalledProcessError(1, msg)
def test_raises_calledprocesserror(self):
mock_run_smartctl = self.patch(smartctl, "run_smartctl")
mock_run_smartctl.side_effect = CalledProcessError(42, "smartctl")
blockdevice = factory.make_name("blockdevice")
self.assertRaises(CalledProcessError, smartctl.check_smartctl,
blockdevice)
def rkd_mock(*args, **kwargs):
if len(recorded_calls) == 0:
recorded_calls.append(args)
raise CalledProcessError(1, 'bash')
recorded_calls.append(args)
def fake_run(*args, **kwargs):
raise CalledProcessError(42, "bad eyaml")
def _run_job_in_hadoop(self):
for step_num in range(self._num_steps()):
step_args = self._args_for_step(step_num)
# log this *after* _args_for_step(), which can start a search
# for the Hadoop streaming jar
log.info('Running step %d of %d...' %
(step_num + 1, self._num_steps()))
log.debug('> %s' % cmd_line(step_args))
log_interpretation = {}
self._log_interpretations.append(log_interpretation)
# try to use a PTY if it's available
try:
pid, master_fd = pty.fork()
except (AttributeError, OSError):
# no PTYs, just use Popen
# user won't get much feedback for a while, so tell them
# Hadoop is running
log.debug('No PTY available, using Popen() to invoke Hadoop')
step_proc = Popen(step_args, stdout=PIPE, stderr=PIPE)
step_interpretation = _interpret_hadoop_jar_command_stderr(
step_proc.stderr,
record_callback=_log_record_from_hadoop)
# there shouldn't be much output to STDOUT
for line in step_proc.stdout:
_log_line_from_hadoop(to_string(line).strip('\r\n'))
step_proc.stdout.close()
step_proc.stderr.close()
returncode = step_proc.wait()
else:
# we have PTYs
if pid == 0: # we are the child process
os.execvp(step_args[0], step_args)
else:
log.debug('Invoking Hadoop via PTY')
with os.fdopen(master_fd, 'rb') as master:
# reading from master gives us the subprocess's
# stderr and stdout (it's a fake terminal)
step_interpretation = (
_interpret_hadoop_jar_command_stderr(
master,
record_callback=_log_record_from_hadoop))
_, returncode = os.waitpid(pid, 0)
# make sure output_dir is filled
if 'output_dir' not in step_interpretation:
step_interpretation['output_dir'] = (
self._hdfs_step_output_dir(step_num))
log_interpretation['step'] = step_interpretation
counters = self._pick_counters(log_interpretation)
if counters:
log.info(_format_counters(counters))
else:
log.warning('No counters found')
if returncode:
error = self._pick_error(log_interpretation)
if error:
log.error('Probable cause of failure:\n\n%s\n' %
_format_error(error))
# use CalledProcessError's well-known message format
reason = str(CalledProcessError(returncode, step_args))
raise StepFailedException(
reason=reason, step_num=step_num,
num_steps=self._num_steps())
def test_get_host_ips_fail_check_output(self, m_check_output):
m_check_output.side_effect = CalledProcessError(
returncode=1, cmd=check_output(["ip", "-4", "addr"]))
with self.assertRaises(SystemExit):
addrs = get_host_ips(version=4)
def raise_error(*args, **kwargs):
raise CalledProcessError(-1, "move")
def run_program(commands,
executable_path,
stdout,
run_dir,
write_input_file=None):
"""
Run
Parameters
----------
commands : str
Input commands for executable
executable_path : str
Path to executable
stdout : bool, optional
Whether to display output when running NJOY
run_dir : str
Directory to run in
write_input_file : str, optional
File name to write out input commands
Raises
------
subprocess.CalledProcessError
If the executable process returns with a non-zero status
"""
if write_input_file is not None:
with open(write_input_file, 'a') as f:
f.write(commands)
f.write("\n")
# Start up process
executable = Popen([executable_path],
cwd=run_dir,
stdin=PIPE,
stdout=PIPE,
stderr=STDOUT,
universal_newlines=True)
executable.stdin.write(commands)
executable.stdin.flush()
lines = []
while True:
# If process is finished, break loop
line = executable.stdout.readline()
if not line and executable.poll() is not None:
break
lines.append(line)
if stdout:
# If user requested output, print to screen
print(line, end='')
# Check for error
if executable.returncode != 0:
raise CalledProcessError(executable.returncode, executable_path,
''.join(lines))
executable.stdin.close()
if stdout:
executable.stdout.close()
def raise_gs_fail(*args, **kwargs):
raise CalledProcessError(
1, 'gs', output=b"", stderr=b"ERROR: Casper is not a friendly ghost"
)
def run_effect(*args, **kwargs):
call_list.append(args[0])
if args[0][:4] == ["snapcraft", "snap", "--output", "snap.snap"]:
raise CalledProcessError(returncode=255, cmd=args[0])
def systemCall(cmd, sh=True, log=None):
if log is None:
log = conduct.app.log
log.debug('System call [sh:%s]: %s' \
% (sh, cmd))
out = []
proc = None
poller = None
outBuf = ['']
errBuf = ['']
def pollOutput():
'''
Read, log and store output (if any) from processes pipes.
'''
removeChars = '\r\n'
# collect fds with new output
fds = [entry[0] for entry in poller.poll()]
if proc.stdout.fileno() in fds:
while True:
try:
tmp = proc.stdout.read(100)
except IOError:
break
outBuf[0] += tmp
while '\n' in outBuf[0]:
line, _, outBuf[0] = outBuf[0].partition('\n')
log.debug(line)
out.append(line + '\n')
if not tmp:
break
if proc.stderr.fileno() in fds:
while True:
try:
tmp = proc.stderr.read(100)
except IOError:
break
errBuf[0] += tmp
while '\n' in errBuf[0]:
line, _, errBuf[0] = errBuf[0].partition('\n')
log.warning(line)
if not tmp:
break
while True:
if proc is None:
# create and start process
proc = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE, shell=sh)
# create poll select
poller = select.poll()
flags = fcntl.fcntl(proc.stdout, fcntl.F_GETFL)
fcntl.fcntl(proc.stdout, fcntl.F_SETFL, flags | os.O_NONBLOCK)
flags = fcntl.fcntl(proc.stderr, fcntl.F_GETFL)
fcntl.fcntl(proc.stderr, fcntl.F_SETFL, flags | os.O_NONBLOCK)
# register pipes to polling
poller.register(proc.stdout, select.POLLIN)
poller.register(proc.stderr, select.POLLIN)
pollOutput()
if proc.poll() is not None: # proc finished
break
# poll once after the process ended to collect all the missing output
pollOutput()
# check return code
if proc.returncode != 0:
raise RuntimeError(
CalledProcessError(proc.returncode, cmd, ''.join(out)))
return ''.join(out)
def call_effect(*args, **kwargs):
if args[0][:2] == ["lxc", "init"]:
raise CalledProcessError(returncode=255, cmd=args[0])
return self.fake_lxd.check_output_side_effect()(*args, **kwargs)
async def solve_challenge(self, body, domain):
try:
js = re.search(
r"setTimeout\(function\(\){\s+(var "
"s,t,o,p,b,r,e,a,k,i,n,g,f.+?\r?\n[\s\S]+?a\.value =.+?)\r?\n",
body).group(1)
except Exception:
raise ValueError(
"Unable to identify Cloudflare IUAM Javascript on website. %s"
% BUG_REPORT)
js = re.sub(r"a\.value = (.+ \+ t\.length).+", r"\1", js)
js = re.sub(r"\s{3,}[a-z](?: = |\.).+", "",
js).replace("t.length", str(len(domain)))
# Strip characters that could be used to exit the string context
# These characters are not currently used in Cloudflare's arithmetic snippet
js = re.sub(r"[\n\\']", "", js)
if "toFixed" not in js:
raise ValueError(
"Error parsing Cloudflare IUAM Javascript challenge. %s" %
BUG_REPORT)
# Use vm.runInNewContext to safely evaluate code
# The sandboxed code cannot use the Node.js standard library
js = "console.log(require('vm').runInNewContext('%s', Object.create(null), {timeout: 5000}));" % js
try:
process = await asyncio.create_subprocess_exec("node",
"-e",
js,
stdout=PIPE,
stderr=PIPE)
result, result_err = await process.communicate()
if process.returncode != 0:
raise CalledProcessError(process.returncode,
["node", "-e", js],
stderr=result_err)
except OSError as e:
if e.errno == 2:
raise EnvironmentError(
"Missing Node.js runtime. Node is required and must be in the PATH (check with `node -v`). Your Node binary may be called `nodejs` rather than `node`, in which case you may need to run `apt-get install nodejs-legacy` on some Debian-based systems. (Please read the cfscrape"
" README's Dependencies section: https://github.com/Anorov/cloudflare-scrape#dependencies."
)
raise
except Exception:
logging.error("Error executing Cloudflare IUAM Javascript. %s" %
BUG_REPORT)
raise
try:
float(result)
except Exception:
raise ValueError(
"Cloudflare IUAM challenge returned unexpected answer. %s" %
BUG_REPORT)
# Not sure why but aiohttp errors if we dont convert to str here
return result.decode().strip()
):
hostname = "github.com"
mock_path_read.side_effect = OSError("Test Exception")
with raises(
RuntimeError,
match=f"Error updating known hosts with public key from {hostname}",
):
setup_ssh_for_git_host(hostname)
@patch("bodywork.git.run")
def test_get_ssh_public_key_from_domain_throws_exception_if_ssh_fingerprints_do_not_match(
mock_run: MagicMock,
):
hostname = "github.com"
with raises(
ConnectionAbortedError,
match=f"SECURITY ALERT! SSH Fingerprint received "
f"from server does not match the fingerprint for {hostname}.",
):
get_ssh_public_key_from_domain(hostname)
@patch("bodywork.git.run", side_effect=CalledProcessError(999, "git rev-parse"))
def test_get_git_commit_hash_throws_exception_on_fail(
mock_run: MagicMock,
):
with raises(BodyworkGitError, match=f"Unable to retrieve git commit hash:"):
get_git_commit_hash()
def run_effect(*args, **kwargs):
call_list.append(args[0])
if args[0] == ['snapcraft', 'snap', '--output', 'snap.snap']:
raise CalledProcessError(returncode=255, cmd=args[0])