def exec_command(comando_str, directorio="."):
"""Executa o comando pasdo no string 'command_str' no directorio pasado como argumento, captura o resultado e
devolveo como un array de bytes.
Argumentos:
(str) comando_str : o literal do commando a executar
(str) directorio : indica se hai que cambiar de directorio para executar o comando
"""
resultado = {}
# Para volver o directorio actual
dir_actual = os.getcwd()
# Cambia ó directorio no que se quere executar o comando
os.chdir(directorio)
# Executa o comando
resultado["saida"], resultado["status"], = pexpect.runu(comando_str,
withexitstatus=1)
# Volve ó directorio actual
os.chdir(dir_actual)
return resultado
def run(self):
timeout_sec = cfg.get("encode_timeout_mins", None)
if timeout_sec is not None:
timeout_sec *= 60
cmd = self._get_command()
logger.info("Executing: {0}".format(cmd))
# stop now if dry run set
if self.dry_run:
logger.warning("Execution skipping (dry-run)!")
return
try:
output, code = pexpect.runu(cmd,
timeout=timeout_sec,
withexitstatus=True)
except Exception as exc:
logger.exception("Failed to encode file: {0}".format(
self.file_path))
self._cleanup_failed_run()
raise exc
else:
if code != 0:
logger.error("ffmpeg returned an error: {0}".format(output))
self._cleanup_failed_run()
return
else:
logger.debug(output)
os.rename(str(self.temp_file_name), str(self.file_path))
logger.info("Successfully re-encoded: {0}".format(self.file_path))
def run(self):
timeout_sec = cfg.get("encode_timeout_mins", None)
if timeout_sec is not None:
timeout_sec *= 60
cmd = self._get_command()
logger.info("Executing: {0}".format(cmd))
# stop now if dry run set
if self.dry_run:
logger.warning("Execution skipping (dry-run)!")
return
try:
output, code = pexpect.runu(cmd, timeout=timeout_sec, withexitstatus=True)
except Exception as exc:
logger.exception("Failed to encode file: {0}".format(self.file_path))
self._cleanup_failed_run()
raise exc
else:
if code != 0:
logger.error("ffmpeg returned an error: {0}".format(output))
self._cleanup_failed_run()
return
else:
logger.debug(output)
os.rename(str(self.temp_file_name), str(self.file_path))
logger.info("Successfully re-encoded: {0}".format(self.file_path))
def run_test(testName, buildConfig):
log.info("Running test %s...", testName)
success = False
unexpectedRc = False
startTime = time.time()
while True:
testConfig = get_test_config(testName, buildConfig)
testFolder = os.path.join(testPath, testName)
testBinFolder = os.path.join(binPath, testName)
testExe = os.path.join(testBinFolder, testName)
if not os.path.exists(testFolder):
log.error("Test folder does not exists [%s]", testFolder)
break
if not os.path.exists(testBinFolder):
log.error("Test bin folder does not exists [%s]", testBinFolder)
break
if not os.path.exists(testExe):
log.error("Test exe does not exists [%s]", testExe)
break
testCmd = "{} {}".format(testExe, testConfig["cmdline"])
testOutput, testRc = pexpect.runu(testCmd,
withexitstatus=True,
timeout=-1,
cwd=testBinFolder)
testOutput = testOutput.replace("\r\n", "\n")
if testRc != testConfig["returnCode"]:
log.error(
"Unexpected return code. Test = %s, Expected = %d, Actual = %d",
testName, testConfig["returnCode"], testRc)
unexpectedRc = True
if not check_test_output(testName, testOutput, testConfig):
break
if unexpectedRc:
break
success = True
break
endTime = time.time()
duration = endTime - startTime
if success:
log.info("Test %s%s%s passed (%.6fs)", COLOR_GREEN, testName,
COLOR_RESET, duration)
else:
log.error("Test %s%s%s failed (%.6fs)", COLOR_RED, testName,
COLOR_RESET, duration)
return success
def _read_file_info(self):
try:
ffprobe_cmd = "ffprobe -v quiet -print_format json -show_format -show_streams \"{0}\"".format(
self.file_path)
ffprobe_json, code = pexpect.runu(ffprobe_cmd, timeout=30, withexitstatus=True)
return json.loads(ffprobe_json)
except Exception:
logger.exception("Error reading file info")
return {}
def _read_file_info(self):
try:
ffprobe_cmd = "ffprobe -v quiet -print_format json -show_format -show_streams \"{0}\"".format(
self.file_path)
ffprobe_json, code = pexpect.runu(ffprobe_cmd,
timeout=30,
withexitstatus=True)
return json.loads(ffprobe_json)
except Exception:
logger.exception("Error reading file info")
return {}
def _ansible_debug(hostkeys):
"""Debug Ansible"""
cmd = [
'ansible', '-e', "'{}'".format(hostkeys), '-m', 'debug', '-a',
'var=vars', 'all'
]
output, exitstatus = pexpect.runu(" ".join([arg for arg in cmd]),
withexitstatus=1)
print(output, file=sys.stdout, flush=True)
if exitstatus != 0:
raise RuntimeError('[-] Ansible error ' +
'(see stdout and stderr above)')
def send_coverage_report():
if not enableCoverage:
return True
coverageFilesParam = ""
coverageFiles = glob.iglob(os.path.join(coveragePath, "coverage_*.info"))
for f in coverageFiles:
coverageFilesParam += " -a " + f
coverageReportFile = os.path.join(coveragePath, "coverage.info")
lcovCmd = "lcov {} -o {}".format(coverageFilesParam, coverageReportFile)
output, rc = pexpect.runu(lcovCmd, timeout=-1, withexitstatus=1)
if rc != 0:
log.error("lcov failed: rc = %d, cmdline = %s", rc, lcovCmd)
print_process_output(output, True, "lcov")
return False
print_process_output(output, False, "lcov")
if not reportCoverage:
return True
log.info("Reporting coverage to coveralls...")
coverallsCmd = "coveralls-lcov {}".format(coverageReportFile)
output, rc = pexpect.runu(coverallsCmd, timeout=-1, withexitstatus=1)
if rc != 0:
log.error("coveralls failed: rc = %d, cmdline = %s", rc, coverallsCmd)
print_process_output(output, True, "coveralls")
return False
print_process_output(output, False, "coveralls")
log.info("Coverage reported to coveralls successfully")
return True
def test_cli():
run('mypass add foo.example.com', [('New passphrase: ', 'masterpw'),
('Verify passphrase: ', 'masterpw'),
('Password: '******'password1')])
run('mypass alias foo.example.com same-as-foo.example.com')
run('mypass add bar.example.com jane', [('Password: '******'password2')])
run('mypass add bar.example.com john password3')
output, status = pexpect.runu('mypass new bar.example.com jack',
withexitstatus=True,
timeout=TIMEOUT)
assert status == 0
assert output.endswith('\r\n')
passwd = output[:-2]
assert len(passwd) == 16
chars = set(passwd)
assert chars.intersection(string.ascii_lowercase)
assert chars.intersection(string.ascii_uppercase)
assert chars.intersection(string.digits)
assert chars.intersection(string.punctuation)
assert chars.issubset(string.ascii_letters + string.digits +
string.punctuation)
run('mypass lock')
run('mypass list', [('Unlock database: ', 'masterpw'),
('bar.example.com', None), ('foo.example.com', None),
('same-as-foo.example.com', None)])
run('mypass get foo.example.com', [('password1', None)])
run('mypass get bar.example.com', [('jack ' + passwd, None),
('jane password2', None),
('john password3', None)])
run('mypass remove foo.example.com')
run('mypass remove bar.example.com jack')
run('mypass list', [('bar.example.com', None),
('same-as-foo.example.com', None)])
run('mypass get bar.example.com', [('jane password2', None),
('john password3', None)])
run('mypass get same-as-foo.example.com', [('password1', None)])
run('mypass rename bar.example.com --new-context=new.example.com')
run('mypass rename new.example.com jane --new-context=bar.example.com')
run('mypass rename new.example.com john --new-username=jeff')
run('mypass list', [('bar.example.com', None), ('new.example.com', None),
('same-as-foo.example.com', None)])
run('mypass get new.example.com', [('jeff password3', None)])
run('mypass get bar.example.com', [('password2', None)])
run('mypass changepw', [('New passphrase: ', 'masterpw2'),
('Verify passphrase: ', 'masterpw2')])
run('mypass lock')
run('mypass list', [('Unlock database: ', 'masterpw2'),
('bar.example.com', None), ('new.example.com', None),
('same-as-foo.example.com', None)])
def test_cli():
run('mypass add foo.example.com', [('New passphrase: ', 'masterpw'),
('Verify passphrase: ', 'masterpw'),
('Password: '******'password1')])
run('mypass add bar.example.com jane', [('Password: '******'password2')])
run('mypass add bar.example.com john password3')
output, status = pexpect.runu('mypass new bar.example.com jack',
withexitstatus=True, timeout=TIMEOUT)
assert status == 0
assert output.endswith('\r\n')
passwd = output[:-2]
assert len(passwd) == 16
chars = set(passwd)
assert chars.intersection(string.ascii_lowercase)
assert chars.intersection(string.ascii_uppercase)
assert chars.intersection(string.digits)
assert chars.intersection(string.punctuation)
assert chars.issubset(string.ascii_letters +
string.digits +
string.punctuation)
run('mypass lock')
run('mypass list', [('Unlock database: ', 'masterpw'),
('bar.example.com', None),
('foo.example.com', None)])
run('mypass get foo.example.com', [('password1', None)])
run('mypass get bar.example.com', [('jack ' + passwd, None),
('jane password2', None),
('john password3', None)])
run('mypass remove foo.example.com')
run('mypass remove bar.example.com jack')
run('mypass list', [('bar.example.com', None)])
run('mypass get bar.example.com', [('jane password2', None),
('john password3', None)])
run('mypass rename bar.example.com --new-context=foo.example.com')
run('mypass rename foo.example.com jane --new-context=bar.example.com')
run('mypass rename foo.example.com john --new-username=jeff')
run('mypass list', [('bar.example.com', None),
('foo.example.com', None)])
run('mypass get foo.example.com', [('jeff password3', None)])
run('mypass get bar.example.com', [('password2', None)])
run('mypass changepw', [('New passphrase: ', 'masterpw2'),
('Verify passphrase: ', 'masterpw2')])
run('mypass lock')
run('mypass list', [('Unlock database: ', 'masterpw2'),
('bar.example.com', None),
('foo.example.com', None)])
def take_action(self, parsed_args):
self.log.debug('[*] creating SSH configuration snippet(s)')
self.app.secrets.requires_environment()
self.app.secrets.read_secrets_and_descriptions()
# if parsed_args.public_ip is None or parsed_args.public_dns is None:
# raise RuntimeError(
# '[-] must specify --public-ip and --public-dns')
# TODO(dittrich): Need to pass key name explicitly...
# _aws_privatekey_path = \
# self.app.secrets.get_secret('aws_privatekey_path')
home = os.path.expanduser('~')
ssh_config = os.path.join(home, '.ssh', 'config')
snippet_prefix = 'psec.{}'.format(self.app.secrets._environment)
if parsed_args.clean:
files = glob.glob(
os.path.join(ssh_config + '.d', snippet_prefix + ".*"))
for f in files:
if self.app_args.verbose_level > 1:
self.log.info('[+] deleting "{}"'.format(f))
os.remove(f)
host_info = _parse_known_hosts(root=parsed_args.known_hosts_root)
private_key_file = self._get_private_key_file()
if private_key_file is None:
raise RuntimeError('[-] no SSH private key specified')
for host, info in host_info.items():
snippet = 'psec.{}.{}'.format(self.app.secrets._environment, host)
if parsed_args.show_config:
# TODO(dittrich): finish this...
print()
_write_ssh_configd(ssh_config=ssh_config,
shortname=host,
name=snippet,
user=parsed_args.ssh_user,
identity_file=private_key_file,
public_ip=info['public_ip'],
public_dns=info['public_dns'])
output, exitstatus = pexpect.runu(
'update-dotdee {}'.format(ssh_config), withexitstatus=1)
if exitstatus == 0:
if self.app_args.verbose_level >= 1:
print(output, file=sys.stdout, flush=True)
else:
print(output, file=sys.stdout, flush=True)
raise RuntimeError(
'[-] update-dotdee error (see stdout and stderr above)')
def test_run_unicode(self):
if pexpect.PY3:
c = chr(254) # þ
pattern = '<in >'
else:
c = unichr(254) # analysis:ignore
pattern = '<in >'.decode('ascii')
def callback(d):
if d['event_count'] == 0:
return c + '\n'
else:
return True # Stop the child process
output = pexpect.runu(sys.executable + ' echo_w_prompt.py',
env={'PYTHONIOENCODING':'utf-8'},
events={pattern:callback})
assert isinstance(output, unicode_type), type(output)
assert '<out>'+c in output, output
def test_run_unicode(self):
if pexpect.PY3:
char = chr(254) # þ
pattern = "<in >"
else:
char = unichr(254) # analysis:ignore
pattern = "<in >".decode("ascii")
def callback(values):
if values["event_count"] == 0:
return char + "\n"
else:
return True # Stop the child process
output = pexpect.runu(
self.PYTHONBIN + " echo_w_prompt.py",
env={"PYTHONIOENCODING": "utf-8"},
events={pattern: callback},
)
assert isinstance(output, unicode_type), type(output)
assert ("<out>" + char) in output, output
def cifs_mount(mount):
if mount.settings["Linux_CIFS_method"] == "gvfs":
# 1) Remove broken symlink or empty dir
if mount.settings["Linux_gvfs_symlink"]:
if (os.path.lexists(mount.settings["local_path"])
and not os.path.exists(mount.settings["local_path"])):
os.unlink(mount.settings["local_path"])
if (os.path.isdir(mount.settings["local_path"])
and os.listdir(mount.settings["local_path"]) == []):
os.rmdir(mount.settings["local_path"])
if os.path.exists(mount.settings["local_path"]):
mount.ui.notify_user("Error : Path {} already exists".format(
mount.settings["local_path"]))
return False
# 2) Mount
share = re.sub(r" ", r"%20", mount.settings["server_share"])
if LIN_CONST.GVFS_GENERATION <= 3:
cmd = [
LIN_CONST.CMD_GVFS_MOUNT,
r"smb://{realm_domain}\;{realm_username}@{server_name}/{share}"
.format(share=share, **mount.settings)
]
else:
cmd = [
LIN_CONST.CMD_GIO_MOUNT, "mount",
r"smb://{realm_domain}\;{realm_username}@{server_name}/{share}"
.format(share=share, **mount.settings)
]
Output.verbose("cmd: " + " ".join(cmd))
process_meta = {
"was_cancelled": False,
}
try:
(output, exit_status) = pexpect.runu(
command=" ".join(cmd),
events={
"Password:"******"auth_realms": [(r"Password:"******"realm"])],
"key_chain": mount.key_chain,
"process_meta": process_meta,
},
env=dict(os.environ, LANG="C", LC_ALL="C"),
withexitstatus=True,
timeout=CONST.MOUNT_TIMEOUT,
)
except pexpect.ExceptionPexpect as exc:
mount.ui.notify_user("Error while mounting :<br>{}".format(
exc.value))
return False
if "error" not in output.lower() and exit_status == 0:
mount.key_chain.ack_password(mount.settings["realm"])
else:
mount.key_chain.invalidate_if_no_ack_password(
mount.settings["realm"])
if process_meta["was_cancelled"]:
return False
else:
mount.ui.notify_user("Mount failure :<br>{}".format(output))
return False
cifs_uncache_is_mounted(mount)
else: # "mount.cifs"
if LIN_CONST.CMD_MOUNT_CIFS is None:
mount.ui.notify_user(
"Error missing binary <b>mount.cifs</b>. On Ubuntu you can install it with <i>sudo apt-get install cifs-utils</i>"
)
return False
# 1) Make mount dir (remove broken symlink if needed)
if (os.path.lexists(mount.settings["local_path"])
and not os.path.exists(mount.settings["local_path"])):
os.unlink(mount.settings["local_path"])
if not os.path.exists(mount.settings["local_path"]):
try:
os.makedirs(mount.settings["local_path"])
except OSError:
pass
if (os.path.islink(mount.settings["local_path"])
or not os.path.isdir(mount.settings["local_path"])):
mount.ui.notify_user("Error while creating dir : %s" %
mount.settings["local_path"])
return False
# 2) Mount
s_path = re.sub(" ", "\\ ", mount.settings["server_path"])
cmd = [
LIN_CONST.CMD_MOUNT_CIFS, "//{server_name}/{s_path}",
"{local_path}", "-o",
"user={realm_username},domain={realm_domain},"
"uid={local_uid},gid={local_gid},"
"file_mode={Linux_mountcifs_filemode},"
"dir_mode={Linux_mountcifs_filemode},"
"{Linux_mountcifs_options}"
]
if CONST.LOCAL_UID != 0:
cmd.insert(0, "sudo")
cmd = [s.format(s_path=s_path, **mount.settings) for s in cmd]
Output.verbose("cmd: " + " ".join(cmd))
# for i in xrange(3): # 3 attempts (for passwords mistyped)
process_meta = {
"was_cancelled": False,
}
(output, exit_status) = pexpect.runu(
command=" ".join(cmd),
events={
"(?i)password": pexpect_ask_password,
},
extra_args={
"auth_realms": [(r"\[sudo\] password", "sudo"),
(r"Password", mount.settings["realm"])],
"key_chain":
mount.key_chain,
"process_meta":
process_meta,
},
env=dict(os.environ, LANG="C", LC_ALL="C"),
withexitstatus=True,
timeout=CONST.MOUNT_TIMEOUT,
)
if exit_status == 0:
mount.key_chain.ack_password("sudo")
mount.key_chain.ack_password(mount.settings["realm"])
else:
mount.key_chain.invalidate_if_no_ack_password("sudo")
mount.key_chain.invalidate_if_no_ack_password(
mount.settings["realm"])
if process_meta["was_cancelled"]:
if (os.path.exists(mount.settings["local_path"])
and os.listdir(mount.settings["local_path"]) == []):
try:
os.rmdir(mount.settings["local_path"])
except OSError as e:
Output.warning("Could not rmdir : {0}".format(e))
else:
mount.ui.notify_user("Mount failure : {}".format(output))
return False
return True
def main():
module = AnsibleModule(
argument_spec=dict(
command=dict(required=True),
chdir=dict(),
creates=dict(),
removes=dict(),
responses=dict(type='dict', required=True),
timeout=dict(type='int', default=30),
echo=dict(type='bool', default=False),
)
)
if not HAS_PEXPECT:
module.fail_json(msg='The pexpect python module is required')
chdir = module.params['chdir']
args = module.params['command']
creates = module.params['creates']
removes = module.params['removes']
responses = module.params['responses']
timeout = module.params['timeout']
echo = module.params['echo']
events = dict()
for key, value in responses.iteritems():
if isinstance(value, list):
response = response_closure(module, key, value)
else:
response = u'%s\n' % value.rstrip('\n').decode()
events[key.decode()] = response
if args.strip() == '':
module.fail_json(rc=256, msg="no command given")
if chdir:
chdir = os.path.abspath(os.path.expanduser(chdir))
os.chdir(chdir)
if creates:
# do not run the command if the line contains creates=filename
# and the filename already exists. This allows idempotence
# of command executions.
v = os.path.expanduser(creates)
if os.path.exists(v):
module.exit_json(
cmd=args,
stdout="skipped, since %s exists" % v,
changed=False,
stderr=False,
rc=0
)
if removes:
# do not run the command if the line contains removes=filename
# and the filename does not exist. This allows idempotence
# of command executions.
v = os.path.expanduser(removes)
if not os.path.exists(v):
module.exit_json(
cmd=args,
stdout="skipped, since %s does not exist" % v,
changed=False,
stderr=False,
rc=0
)
startd = datetime.datetime.now()
try:
try:
# Prefer pexpect.run from pexpect>=4
out, rc = pexpect.run(args, timeout=timeout, withexitstatus=True,
events=events, cwd=chdir, echo=echo,
encoding='utf-8')
except TypeError:
# Use pexpect.runu in pexpect>=3.3,<4
out, rc = pexpect.runu(args, timeout=timeout, withexitstatus=True,
events=events, cwd=chdir, echo=echo)
except (TypeError, AttributeError), e:
# This should catch all insufficient versions of pexpect
# We deem them insufficient for their lack of ability to specify
# to not echo responses via the run/runu functions, which would
# potentially leak sensentive information
module.fail_json(msg='Insufficient version of pexpect installed '
'(%s), this module requires pexpect>=3.3. '
'Error was %s' % (pexpect.__version__, e))
def main():
module = AnsibleModule(argument_spec=dict(
command=dict(required=True),
chdir=dict(),
creates=dict(),
removes=dict(),
responses=dict(type='dict', required=True),
timeout=dict(type='int', default=30),
echo=dict(type='bool', default=False),
))
if not HAS_PEXPECT:
module.fail_json(msg='The pexpect python module is required')
chdir = module.params['chdir']
args = module.params['command']
creates = module.params['creates']
removes = module.params['removes']
responses = module.params['responses']
timeout = module.params['timeout']
echo = module.params['echo']
events = dict()
for key, value in responses.iteritems():
if isinstance(value, list):
response = response_closure(module, key, value)
else:
response = u'%s\n' % value.rstrip('\n').decode()
events[key.decode()] = response
if args.strip() == '':
module.fail_json(rc=256, msg="no command given")
if chdir:
chdir = os.path.abspath(os.path.expanduser(chdir))
os.chdir(chdir)
if creates:
# do not run the command if the line contains creates=filename
# and the filename already exists. This allows idempotence
# of command executions.
v = os.path.expanduser(creates)
if os.path.exists(v):
module.exit_json(cmd=args,
stdout="skipped, since %s exists" % v,
changed=False,
stderr=False,
rc=0)
if removes:
# do not run the command if the line contains removes=filename
# and the filename does not exist. This allows idempotence
# of command executions.
v = os.path.expanduser(removes)
if not os.path.exists(v):
module.exit_json(cmd=args,
stdout="skipped, since %s does not exist" % v,
changed=False,
stderr=False,
rc=0)
startd = datetime.datetime.now()
try:
try:
# Prefer pexpect.run from pexpect>=4
out, rc = pexpect.run(args,
timeout=timeout,
withexitstatus=True,
events=events,
cwd=chdir,
echo=echo,
encoding='utf-8')
except TypeError:
# Use pexpect.runu in pexpect>=3.3,<4
out, rc = pexpect.runu(args,
timeout=timeout,
withexitstatus=True,
events=events,
cwd=chdir,
echo=echo)
except (TypeError, AttributeError), e:
# This should catch all insufficient versions of pexpect
# We deem them insufficient for their lack of ability to specify
# to not echo responses via the run/runu functions, which would
# potentially leak sensentive information
module.fail_json(msg='Insufficient version of pexpect installed '
'(%s), this module requires pexpect>=3.3. '
'Error was %s' % (pexpect.__version__, e))
def update(self): self.lock.acquire() self._state = pexpect.runu(self.command)[:-2] self.lock.release()
def build_tests(compiler="", standard="", platform="", sanitizer=""):
log.info(
"Building tests: compiler = %s, standard = %s, platform = %s, "
"sanitizer = %s", compiler, standard, platform, sanitizer)
startTime = time.time()
success = True
while True:
shutil.rmtree(binPath, ignore_errors=True)
if os.path.exists(binPath):
log.error("bin folder exists")
success = False
break
shutil.rmtree(cmakeFilesPath, ignore_errors=True)
if os.path.exists(cmakeFilesPath):
log.error("cmake_files folder exists")
success = False
break
os.mkdir(cmakeFilesPath)
config = ""
if len(compiler):
config = "{} -DCOMPILER={}".format(config, compiler)
if len(standard):
config = "{} -DSTANDARD={}".format(config, standard)
if len(platform):
config = "{} -DPLATFORM={}".format(config, platform)
if sanitizer == "address":
config += " -DADDRESS_SANITIZER=ON"
elif sanitizer == "thread":
config += " -DTHREAD_SANITIZER=ON"
elif sanitizer == "leak":
config += " -DLEAK_SANITIZER=ON"
elif sanitizer == "ub":
config += " -DUNDEFINED_BEHAVIOR_SANITIZER=ON"
elif sanitizer != "":
log.error("Unknown sanitizer %s", sanitizer)
success = False
break
if enableCoverage:
config += " -DCODE_COVERAGE=ON"
# run cmake
cmakeCmdline = "cmake -S {} -B {} {}".format(buildPath, cmakeFilesPath,
config)
output, rc = pexpect.runu(cmakeCmdline, withexitstatus=1, timeout=-1)
if rc != 0:
log.error("cmake failed: rc = %d, cmdline = %s", rc, cmakeCmdline)
print_process_output(output, True, "cmake")
success = False
break
print_process_output(output, False, "cmake")
# run make
tests = ""
for t in testsArray:
tests += " " + t
makeCmdline = "make -C {} -j{} {}".format(cmakeFilesPath, numberOfCpus,
tests)
output, rc = pexpect.runu(makeCmdline, withexitstatus=1, timeout=-1)
if rc != 0:
log.error("make failed: rc = %d, cmdline = %s", rc, makeCmdline)
print_process_output(output, True, "make")
success = False
break
print_process_output(output, False, "make")
break
endTime = time.time()
duration = endTime - startTime
update_build_statistics(success)
increment_build_time(duration)
if success:
log.info("%sBuild completed successfully%s (%.6fs)", COLOR_GREEN,
COLOR_RESET, duration)
else:
log.error("%sBuild tests failed%s (%.6fs)", COLOR_RED, COLOR_RESET,
duration)
return success
def generate_coverage_report(compiler):
global coverageFileId
log.info("Generating coverage report...")
coverageFiles = ""
success = True
gcovTool = ""
if compiler != "":
if compiler in supportedCompilersDict:
gcovTool = get_gcov_tool(compiler)
else:
log.error("Unknown compiler %s", compiler)
return False
for testName in testsArray:
log.info("Generating coverage report for test %s", testName)
testBinPath = os.path.join(binPath, testName)
coverageFile = os.path.join(testBinPath, "coverage.info")
coverageFiles += " -a " + coverageFile
testCMakePath = os.path.join(cmakeFilesPath, "CMakeFiles",
testName + ".dir")
testCMakePath += testPath
testCMakePath = os.path.join(testCMakePath, testName)
gcdaFiles = glob.iglob(os.path.join(testCMakePath, "*.gcda"))
for f in gcdaFiles:
if os.path.isfile(f):
shutil.copy2(f, testBinPath)
gcnoFiles = glob.iglob(os.path.join(testCMakePath, "*.gcno"))
for f in gcnoFiles:
if os.path.isfile(f):
shutil.copy2(f, testBinPath)
lcovCmd = "lcov --directory {} --capture --output-file {} {}".format(
testBinPath, coverageFile, gcovTool)
output, rc = pexpect.runu(lcovCmd, timeout=-1, withexitstatus=1)
if rc != 0:
log.error("lcov failed: rc = %d, cmdline = %s", rc, lcovCmd)
print_process_output(output, True, "lcov")
success = False
break
print_process_output(output, False, "lcov")
if success:
coverageFilename = "coverage_" + str(coverageFileId) + ".info"
coverageFileId += 1
coverageReportFile = os.path.join(coveragePath, coverageFilename)
lcovCmd = "lcov {} -o {}".format(coverageFiles, coverageReportFile)
output, rc = pexpect.runu(lcovCmd, timeout=-1, withexitstatus=1)
if rc != 0:
log.error("lcov failed: rc = %d, cmdline = %s", rc, lcovCmd)
print_process_output(output, True, "lcov")
success = False
else:
print_process_output(output, False, "lcov")
if not remove_abs_path_from_coverage_report(coverageReportFile):
success = False
if success:
log.info("%sCoverage report generated successfully%s", COLOR_GREEN,
COLOR_RESET)
else:
log.error("%sGenerate coverage report failed%s", COLOR_RED,
COLOR_RESET)
return success
def cifs_mount(mount):
# 1) Make mountpoint
if not os.path.exists(mount.settings["local_path"]):
try:
os.makedirs(mount.settings["local_path"])
except OSError:
pass
if not os.path.isdir(mount.settings["local_path"]):
raise Exception("Error while creating dir : %s" %
mount.settings["local_path"])
# 2) Mount
s_path = re.sub(r" ", r"%20", mount.settings["server_path"])
cmd = [
OSX_CONST.CMD_MOUNT_SMBFS,
r"//{realm_domain}\;{realm_username}@{server_name}/{s_path}".format(
s_path=s_path, **mount.settings),
"\"{local_path}\"".format(**mount.settings)
]
Output.verbose("cmd: " + " ".join(cmd))
for _ in range(3): # 3 attempts (for passwords mistyped)
process_meta = {
"was_cancelled": False,
}
try:
(output, exit_status) = pexpect.runu(
command=" ".join(cmd),
events={
"(?i)password": pexpect_ask_password,
},
extra_args={
"auth_realms": [(r"Password", mount.settings["realm"])],
"key_chain": mount.key_chain,
"process_meta": process_meta,
},
env=dict(os.environ, LANG="C", LC_ALL="C"),
withexitstatus=True,
timeout=CONST.MOUNT_TIMEOUT,
)
except pexpect.ExceptionPexpect as e:
raise Exception("Error while mounting : %s" % e.value)
if exit_status == 0:
mount.key_chain.ack_password(mount.settings["realm"])
return True
elif exit_status == 77: # Bad password
mount.key_chain.invalidate_if_no_ack_password(
mount.settings["realm"])
else:
mount.key_chain.invalidate_if_no_ack_password(
mount.settings["realm"])
if process_meta["was_cancelled"]:
if (os.path.isdir(mount.settings["local_path"])
and os.listdir(mount.settings["local_path"]) == []):
try:
os.rmdir(mount.settings["local_path"])
except OSError as e:
Output.warning("Could not rmdir : {}".format(e))
return False
else:
mount.ui.notify_user("Mount failure :<br>{}".format(output))
return False
mount.ui.notify_user("Mount failure")
def cifs_umount(mount):
def _cb_gvfs(success, output, exit_code):
if not success:
mount.ui.notify_user("Umount failure :<br>{}".format(output))
cifs_uncache_is_mounted(mount)
if mount.settings["Linux_CIFS_method"] == "gvfs":
# gvfs umount apparently never locks on open files.
# 1) Umount
share = re.sub(r" ", r"%20", mount.settings["server_share"])
if LIN_CONST.GVFS_GENERATION <= 3:
cmd = [
LIN_CONST.CMD_GVFS_MOUNT, "-u",
r"smb://{realm_domain};{realm_username}@{server_name}/{share}".
format(share=share, **mount.settings)
]
else:
cmd = [
LIN_CONST.CMD_GIO_MOUNT, "mount", "-u",
r"smb://{realm_domain};{realm_username}@{server_name}/{share}".
format(share=share, **mount.settings)
]
Output.verbose("cmd: " + " ".join(cmd))
if mount.ui.UI_TYPE == "GUI":
NonBlockingQtProcess(
cmd,
_cb_gvfs,
env=dict(os.environ, LANG="C", LC_ALL="C", LANGUAGE="C"),
)
else:
_cb_gvfs(**BlockingProcess.run(
cmd,
env=dict(os.environ, LANG="C", LC_ALL="C", LANGUAGE="C"),
))
else: # "mount.cifs"
# 1) uMount
cmd = [LIN_CONST.CMD_UMOUNT, "{local_path}"]
if CONST.LOCAL_UID != 0:
cmd.insert(0, "sudo")
cmd = [s.format(**mount.settings) for s in cmd]
Output.verbose("cmd: " + " ".join(cmd))
# for i in xrange(3): # 3 attempts (for passwords mistyped)
process_meta = {
"was_cancelled": False,
}
(output, exit_status) = pexpect.runu(
command=" ".join(cmd),
events={
"(?i)password": pexpect_ask_password,
},
extra_args={
"auth_realms": [
(r"\[sudo\] password", "sudo"),
],
"key_chain": mount.key_chain,
"process_meta": process_meta,
},
env=dict(os.environ, LANG="C", LC_ALL="C"),
withexitstatus=True,
timeout=CONST.UMOUNT_TIMEOUT,
)
if exit_status == 0:
mount.key_chain.ack_password("sudo")
else:
if process_meta["was_cancelled"]:
mount.key_chain.invalidate_if_no_ack_password("sudo")
elif "device is busy" in output:
mount.key_chain.ack_password("sudo")
mount.ui.notify_user("Umount failure: Device is busy.")
return False
else:
mount.key_chain.invalidate_if_no_ack_password("sudo")
mount.ui.notify_user("Umount failure")
return False
def main():
module = AnsibleModule(
argument_spec=dict(
command=dict(required=True),
chdir=dict(type='path'),
creates=dict(type='path'),
removes=dict(type='path'),
responses=dict(type='dict', required=True),
timeout=dict(type='int', default=30),
echo=dict(type='bool', default=False),
)
)
if not HAS_PEXPECT:
module.fail_json(msg='The pexpect python module is required')
chdir = module.params['chdir']
args = module.params['command']
creates = module.params['creates']
removes = module.params['removes']
responses = module.params['responses']
timeout = module.params['timeout']
echo = module.params['echo']
events = dict()
for key, value in responses.items():
if isinstance(value, list):
response = response_closure(module, key, value)
else:
response = u'%s\n' % to_text(value).rstrip(u'\n')
events[to_text(key)] = response
if args.strip() == '':
module.fail_json(rc=256, msg="no command given")
if chdir:
chdir = os.path.abspath(chdir)
os.chdir(chdir)
if creates:
# do not run the command if the line contains creates=filename
# and the filename already exists. This allows idempotence
# of command executions.
if os.path.exists(creates):
module.exit_json(
cmd=args,
stdout="skipped, since %s exists" % creates,
changed=False,
rc=0
)
if removes:
# do not run the command if the line contains removes=filename
# and the filename does not exist. This allows idempotence
# of command executions.
if not os.path.exists(removes):
module.exit_json(
cmd=args,
stdout="skipped, since %s does not exist" % removes,
changed=False,
rc=0
)
startd = datetime.datetime.now()
try:
try:
# Prefer pexpect.run from pexpect>=4
out, rc = pexpect.run(args, timeout=timeout, withexitstatus=True,
events=events, cwd=chdir, echo=echo,
encoding='utf-8')
except TypeError:
# Use pexpect.runu in pexpect>=3.3,<4
out, rc = pexpect.runu(args, timeout=timeout, withexitstatus=True,
events=events, cwd=chdir, echo=echo)
except (TypeError, AttributeError) as e:
# This should catch all insufficient versions of pexpect
# We deem them insufficient for their lack of ability to specify
# to not echo responses via the run/runu functions, which would
# potentially leak sensentive information
module.fail_json(msg='Insufficient version of pexpect installed '
'(%s), this module requires pexpect>=3.3. '
'Error was %s' % (pexpect.__version__, to_native(e)))
except pexpect.ExceptionPexpect as e:
module.fail_json(msg='%s' % to_native(e), exception=traceback.format_exc())
endd = datetime.datetime.now()
delta = endd - startd
if out is None:
out = ''
result = dict(
cmd=args,
stdout=out.rstrip('\r\n'),
rc=rc,
start=str(startd),
end=str(endd),
delta=str(delta),
changed=True,
)
if rc is None:
module.fail_json(msg='command exceeded timeout', **result)
elif rc != 0:
module.fail_json(msg='non-zero return code', **result)
module.exit_json(**result)
def main():
module = AnsibleModule(
argument_spec=dict(
command=dict(required=True),
chdir=dict(),
creates=dict(),
removes=dict(),
responses=dict(type='dict', required=True),
timeout=dict(type='int', default=30),
echo=dict(type='bool', default=False),
)
)
if not HAS_PEXPECT:
module.fail_json(msg='The pexpect python module is required')
chdir = module.params['chdir']
args = module.params['command']
creates = module.params['creates']
removes = module.params['removes']
responses = module.params['responses']
timeout = module.params['timeout']
echo = module.params['echo']
events = dict()
for key, value in responses.iteritems():
events[key.decode()] = u'%s\n' % value.rstrip('\n').decode()
if args.strip() == '':
module.fail_json(rc=256, msg="no command given")
if chdir:
chdir = os.path.abspath(os.path.expanduser(chdir))
os.chdir(chdir)
if creates:
# do not run the command if the line contains creates=filename
# and the filename already exists. This allows idempotence
# of command executions.
v = os.path.expanduser(creates)
if os.path.exists(v):
module.exit_json(
cmd=args,
stdout="skipped, since %s exists" % v,
changed=False,
stderr=False,
rc=0
)
if removes:
# do not run the command if the line contains removes=filename
# and the filename does not exist. This allows idempotence
# of command executions.
v = os.path.expanduser(removes)
if not os.path.exists(v):
module.exit_json(
cmd=args,
stdout="skipped, since %s does not exist" % v,
changed=False,
stderr=False,
rc=0
)
startd = datetime.datetime.now()
try:
out, rc = pexpect.runu(args, timeout=timeout, withexitstatus=True,
events=events, cwd=chdir, echo=echo)
except pexpect.ExceptionPexpect, e:
module.fail_json(msg='%s' % e)
def main():
module = AnsibleModule(argument_spec=dict(
command=dict(required=True),
chdir=dict(),
creates=dict(),
removes=dict(),
responses=dict(type='dict', required=True),
timeout=dict(type='int', default=30),
echo=dict(type='bool', default=False),
))
if not HAS_PEXPECT:
module.fail_json(msg='The pexpect python module is required')
chdir = module.params['chdir']
args = module.params['command']
creates = module.params['creates']
removes = module.params['removes']
responses = module.params['responses']
timeout = module.params['timeout']
echo = module.params['echo']
events = dict()
for key, value in responses.iteritems():
events[key.decode()] = u'%s\n' % value.rstrip('\n').decode()
if args.strip() == '':
module.fail_json(rc=256, msg="no command given")
if chdir:
chdir = os.path.abspath(os.path.expanduser(chdir))
os.chdir(chdir)
if creates:
# do not run the command if the line contains creates=filename
# and the filename already exists. This allows idempotence
# of command executions.
v = os.path.expanduser(creates)
if os.path.exists(v):
module.exit_json(cmd=args,
stdout="skipped, since %s exists" % v,
changed=False,
stderr=False,
rc=0)
if removes:
# do not run the command if the line contains removes=filename
# and the filename does not exist. This allows idempotence
# of command executions.
v = os.path.expanduser(removes)
if not os.path.exists(v):
module.exit_json(cmd=args,
stdout="skipped, since %s does not exist" % v,
changed=False,
stderr=False,
rc=0)
startd = datetime.datetime.now()
try:
out, rc = pexpect.runu(args,
timeout=timeout,
withexitstatus=True,
events=events,
cwd=chdir,
echo=echo)
except pexpect.ExceptionPexpect, e:
module.fail_json(msg='%s' % e)
def main():
module = AnsibleModule(argument_spec=dict(
command=dict(required=True),
chdir=dict(type='path'),
creates=dict(type='path'),
removes=dict(type='path'),
responses=dict(type='dict', required=True),
timeout=dict(type='int', default=30),
echo=dict(type='bool', default=False),
))
if not HAS_PEXPECT:
module.fail_json(msg=missing_required_lib("pexpect"),
exception=PEXPECT_IMP_ERR)
chdir = module.params['chdir']
args = module.params['command']
creates = module.params['creates']
removes = module.params['removes']
responses = module.params['responses']
timeout = module.params['timeout']
echo = module.params['echo']
events = dict()
for key, value in responses.items():
if isinstance(value, list):
response = response_closure(module, key, value)
else:
response = u'%s\n' % to_text(value).rstrip(u'\n')
events[to_text(key)] = response
if args.strip() == '':
module.fail_json(rc=256, msg="no command given")
if chdir:
chdir = os.path.abspath(chdir)
os.chdir(chdir)
if creates:
# do not run the command if the line contains creates=filename
# and the filename already exists. This allows idempotence
# of command executions.
if os.path.exists(creates):
module.exit_json(cmd=args,
stdout="skipped, since %s exists" % creates,
changed=False,
rc=0)
if removes:
# do not run the command if the line contains removes=filename
# and the filename does not exist. This allows idempotence
# of command executions.
if not os.path.exists(removes):
module.exit_json(cmd=args,
stdout="skipped, since %s does not exist" %
removes,
changed=False,
rc=0)
startd = datetime.datetime.now()
try:
try:
# Prefer pexpect.run from pexpect>=4
out, rc = pexpect.run(args,
timeout=timeout,
withexitstatus=True,
events=events,
cwd=chdir,
echo=echo,
encoding='utf-8')
except TypeError:
# Use pexpect.runu in pexpect>=3.3,<4
out, rc = pexpect.runu(args,
timeout=timeout,
withexitstatus=True,
events=events,
cwd=chdir,
echo=echo)
except (TypeError, AttributeError) as e:
# This should catch all insufficient versions of pexpect
# We deem them insufficient for their lack of ability to specify
# to not echo responses via the run/runu functions, which would
# potentially leak sensitive information
module.fail_json(msg='Insufficient version of pexpect installed '
'(%s), this module requires pexpect>=3.3. '
'Error was %s' % (pexpect.__version__, to_native(e)))
except pexpect.ExceptionPexpect as e:
module.fail_json(msg='%s' % to_native(e),
exception=traceback.format_exc())
endd = datetime.datetime.now()
delta = endd - startd
if out is None:
out = ''
result = dict(
cmd=args,
stdout=out.rstrip('\r\n'),
rc=rc,
start=str(startd),
end=str(endd),
delta=str(delta),
changed=True,
)
if rc is None:
module.fail_json(msg='command exceeded timeout', **result)
elif rc != 0:
module.fail_json(msg='non-zero return code', **result)
module.exit_json(**result)
def run(s, framework, endpoint):
mem_samples = []
cpu_samples = []
wrk_cmd = CMDS['wrk'] % (RUN_TIME, endpoint)
wrk_warmup_cmd = CMDS['wrk'] % (30, endpoint)
cd_cmd = CMDS['cd-framework'] % framework
framework_processname = 'unknown'
def sample_it(processname):
time.sleep(RUN_TIME * 0.2)
def task():
time.sleep(RUN_TIME * 0.6 / SAMPLE_NUM / 2)
s.sendline(CMDS['cpu-usage'] % processname)
s.expect(r"___([0-9]+\.?[0-9]?)___", timeout=100)
cpu_samples.append(float(s.match.group(1)))
s.sendline(CMDS['mem-usage'] % processname)
s.expect(r"___([0-9]+\.?[0-9]?)((?:M|G|K|T)i.*?)___", timeout=100)
mem = float(s.match.group(1))
mem_units = s.match.group(1)
if mem_units.startswith('Ki'):
mem /= 1024
elif mem_units.startswith('Gi'):
mem *= 1024
mem_samples.append(int(mem))
counter(task, SAMPLE_NUM)
print(
'=' * 15, 'Working. %s server. %s endpoint.' %
(framework.upper(), endpoint.upper()), '=' * 15)
# set bash-prompt for further reliable checks
s.sendline('PS1=' + PROMPT)
s.expect(PROMPT)
# cd framework's directory
print('cd framework... ', cd_cmd)
s.sendline(cd_cmd)
s.expect(PROMPT)
s.sendline('pwd')
s.expect(framework)
# get framework's processname for sampling (read it from the host machine)
with open('./%s/processname.txt' % framework) as f:
framework_processname = f.read().strip()
# run framework in docker
print('run docker... ', CMDS['docker-run'])
s.sendline(CMDS['docker-run'])
s.expect('Launching container')
s.expect(PROMPT)
# Warm-up
print('Running wrk to warm-up...')
pexpect.runu(wrk_warmup_cmd, timeout=1000000)
# start background sampling thread
t = threading.Thread(target=partial(sample_it, framework_processname))
t.start()
# Run wrk benchmark
print('Running wrk benchmark...')
res = pexpect.runu(wrk_cmd, timeout=1000000)
for l in res.splitlines():
print(l)
m = re.search('Socket errors.*timeout.*?(\d+)', res)
run_timeout_number = int(m.group(1).strip()) if m else 'unknown'
m = re.search(', (.+?) read', res)
data_read = m.group(1).strip() if m else 'unknown'
m = re.search('Requests/sec:\W*([\.\d]+)', res)
run_req_sec = m.group(1).strip() if m else 'unknown'
m = re.search('Latency\W*([\.\w]+)', res)
run_latency_avg = m.group(1).strip() if m else 'unknown'
run_latencies = re.findall('Latency percentile\W*([\.\d]+)%,\W*([\.\d]+)',
res)
m = re.search('Non-2xx or 3xx responses:\W*([\.\d]+)', res)
bad_resps = m.group(1).strip() if m else '0'
t.join(RUN_TIME + 30)
print('Reporting measurements made during benchmark...')
do_report(cpu_samples=cpu_samples,
mem_samples=mem_samples,
framework=framework,
endpoint=endpoint,
test_command=wrk_cmd,
timeouted=run_timeout_number,
data_read=data_read,
requests_per_second=run_req_sec,
latency=run_latency_avg,
latency_percentiles=run_latencies,
bad_responses=bad_resps)
print('Exiting...')
