def run_script( filename, timeout = 30 ):
global sdir_
global result_
global pypath_
# Run the script in the directory of file.
tgtdir = os.path.dirname( os.path.realpath( filename ) )
# copy matplotlibrc file to this directory.
try:
shutil.copy( os.path.join( sdir_, 'matplotlibrc' ), tgtdir )
except Exception as e:
pass
status = 'FAILED'
res = None
try:
res = subprocess.run( [ pypath_, filename ], cwd = tgtdir
, timeout = timeout
, stdout = subprocess.PIPE
, stderr = subprocess.PIPE
)
if res.returncode == 0:
status = 'PASSED'
else:
status = 'FAILED'
except subprocess.TimeoutExpired as e:
status = 'TIMEOUT'
stamp = datetime.datetime.now().isoformat()
print( '[%s] %10s %s' % (stamp, status,filename) )
if res is not None:
result_[status].append( (filename,res.stdout+res.stderr) )
else:
result_[status].append( (filename,'UNKNOWN') )
def get_git_root():
""" Returns the root directory of the git repository, assuming this script is run from within the repository. """
result = subprocess.run(['git', 'rev-parse', '--show-toplevel'], stdout=subprocess.PIPE, check=True)
if result.stdout is None:
# TODO: concrete exception
raise Exception('Did not get any output from git: stderr is "{}"'.format(result.stderr))
return result.stdout.decode('utf-8').rstrip('\n')
def run_process(self, inp=""):
"""Run the program with `inp` as input. Return result of `subprocess.run()`."""
res = subprocess.run(self.program,
input=inp,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
return res
def getVaa3dPluginHelp(pluginName):
vaa3d = getVaa3DExecutable()
try:
if platform.system() == "Windows":
completedProcess = subprocess.run([vaa3d, '/h', '/x', pluginName],
stdout=subprocess.PIPE)
else:
completedProcess = subprocess.run([vaa3d, '-h', '-x', pluginName],
stdout=subprocess.PIPE)
except subprocess.CalledProcessError as cpe:
print(cpe.stderr)
raise cpe
return completedProcess.stdout.decode("utf-8")
def girth(relator, verbose=False, **kwargs):
"""
Given a word defining a shortlex automatic one-relator group, return the girth of the Cayley graph for the corresponding one-relator presentation and a pair of equivalent words witnessing shortest relation.
Default generator ordering is [...,'B','A','a','b',...]. Use keyword 'generators' to specify different ordering of generators and inverses.
Use filename=kbmagbasefilename to specify existing kbmag files defining the automatric structure.
Use cleanup=True to delete all auxilliary kbmag files or cleanup=False to keep them. Default is to cleanup if this function creates the files and not if they already exist.
Set timeout=n to set time limit of n seconds on attempt to generate automatric group structure.
"""
if not relator:
return float('inf')
F, r = fg.parseinputword(relator)
relatorasstring = r()
relatoraslist = r.letters
if 'filename' in kwargs:
thefilename = kwargs['filename']
else:
thefilename = "OneRelatorGroup-" + relatorasstring
if 'cleanup' in kwargs:
cleanup = kwargs['cleanup']
else:
cleanup = not os.path.isfile(
thefilename
) # default is to cleanup if the files are not already existing
if not os.path.isfile(thefilename):
if 'generators' in kwargs:
generators = kwargs['generators']
else:
generators = [x.upper() for x in F.gens[::-1]] + F.gens
ag.writetokbmagfile(thefilename, generators, [relatorasstring])
if 'timeout' in kwargs:
timeout = kwargs['timeout']
else:
timeout = 10 # default timeout 10s
if not os.path.isfile(thefilename + '.diff1'):
try:
subprocess32.run(['autgroup', '-silent', thefilename],
check=True,
timeout=timeout)
except subprocess32.TimeoutExpired, subprocess32.CalledProcessError:
if cleanup:
files = glob.glob('./' + thefilename + "*")
for file in files:
os.remove(file)
raise
def run_command(cmd, input_=None):
p = sp.run(
shlex.split(cmd),
input=input_,
stdout=sp.PIPE,
stderr=sp.PIPE,
encoding='utf-8',
)
return p.stdout, p.stderr
def loop_seq(cmd_seq, wait_time): # pylint: disable=missing-param-doc,missing-type-doc
"""Call a sequence of commands in a loop.
If any fails, sleep(wait_time) and go back to the beginning of the sequence."""
i = 0
while True:
i += 1
print("localLoop #%d!" % i)
for cmd in cmd_seq:
try:
subprocess.run(cmd, check=True)
except subprocess.CalledProcessError as ex:
print("Something went wrong when calling: %r" % (cmd, ))
print("%r" % (ex, ))
import traceback
print(traceback.format_exc())
print("Waiting %d seconds..." % wait_time)
time.sleep(wait_time)
break
def Query(args: rdf_osquery.OsqueryArgs) -> str:
"""Calls osquery with given query and returns its output.
Args:
args: A query to call osquery with.
Returns:
A "parsed JSON" representation of the osquery output.
Raises:
TimeoutError: If a call to the osquery executable times out.
Error: If anything goes wrong with the subprocess call, including if the
query is incorrect.
"""
timeout = args.timeout_millis / 1000 # `subprocess.run` uses seconds.
# TODO: pytype is not aware of the backport.
# pytype: disable=module-attr
try:
# We use `--S` to enforce shell execution. This is because on Windows there
# is only `osqueryd` and `osqueryi` is not available. However, by passing
# `--S` we can make `osqueryd` behave like `osqueryi`. Since this flag also
# works with `osqueryi`, by passing it we simply expand number of supported
# executable types.
command = [
config.CONFIG["Osquery.path"],
"--S", # Enforce shell execution.
"--logger_stderr=false", # Only allow errors to be written to stderr.
"--logger_min_status=3", # Disable status logs.
"--logger_min_stderr=2", # Only ERROR-level logs to stderr.
"--json", # Set output format to JSON.
args.query,
]
proc = subprocess.run(
command,
timeout=timeout,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# TODO: Since we use a backported API, `SubprocessError` is hard
# to work with. Until support for Python 2 is dropped we re-raise with simpler
# exception type because we don't really care that much (the exception message
# should be detailed enough anyway).
except subprocess.TimeoutExpired as error:
raise TimeoutError(cause=error)
except subprocess.CalledProcessError as error:
stderr = error.stderr.decode("utf-8")
raise Error(message=f"Osquery error on the client: {stderr}")
# pytype: enable=module-attr
stderr = proc.stderr.decode("utf-8").strip()
if stderr:
# Depending on the version, in case of a syntax error osquery might or might
# not terminate with a non-zero exit code, but it will always print the
# error to stderr.
raise Error(message=f"Osquery error on the client: {stderr}")
return proc.stdout.decode("utf-8")
def getVaa3dHelpInternal(vaa3d):
try:
if platform.system() == "Windows":
completedProcess = subprocess.run([vaa3d, '/h'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
else:
completedProcess = subprocess.run([vaa3d, '-h'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except subprocess.CalledProcessError as cpe:
print(cpe.stderr)
raise cpe
return completedProcess.stdout.decode("utf-8")
def used_dns():
cmd = ["dig", "www.google.com", "+noquestion", "+nocomments", "+noanswer"]
try:
out = run(cmd, timeout=10, check=True, stdout=PIPE).stdout
start = out.find('SERVER: ') + 8
end = out[start:].find('(')
return out[start:start + end]
except Exception:
return ""
def findCommonAncestor(repo_dir, a, b): # pylint: disable=invalid-name,missing-docstring,missing-return-doc
# pylint: disable=missing-return-type-doc
return subprocess.run(
["hg", "-R", str(repo_dir), "log", "-r", "ancestor(" + a + "," + b + ")", "--template={node|short}"],
cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member
check=True,
stdout=subprocess.PIPE,
timeout=999
).stdout.decode("utf-8", errors="replace")
def print_machine_info():
"""Log information about the machine."""
print("Platform details: %s" % " ".join(platform.uname()))
print("hg info: %s" % subprocess.run(
["hg", "-q", "version"], check=True,
stdout=subprocess.PIPE).stdout.decode("utf-8",
errors="replace").rstrip())
if which("gdb"):
gdb_version = subprocess.run(["gdb", "--version"],
stdout=subprocess.PIPE).stdout.decode(
"utf-8", errors="replace")
print("gdb info: %s" % gdb_version.split("\n")[0])
if which("git"):
print("git info: %s" % subprocess.run(
["git", "version"], check=True,
stdout=subprocess.PIPE).stdout.decode("utf-8",
errors="replace").rstrip())
print("Python version: %s" % sys.version.split()[0])
print("Number of cores visible to OS: %d" % multiprocessing.cpu_count())
if sys.version_info.major == 2:
rootdir_free_space = psutil.disk_usage("/").free / (1024**3)
else:
rootdir_free_space = shutil.disk_usage("/").free / (1024**3) # pylint: disable=no-member
print("Free space (GB): %.2f" % rootdir_free_space)
hgrc_path = Path("~/.hg/hgrc").expanduser()
if hgrc_path.is_file():
print("The hgrc of this repository is:")
with io.open(str(hgrc_path), "r", encoding="utf-8",
errors="replace") as f:
hgrc_contents = f.readlines()
for line in hgrc_contents:
print(line.rstrip())
try:
# resource library is only applicable to Linux or Mac platforms.
import resource # pylint: disable=import-error
# pylint: disable=no-member
print("Corefile size (soft limit, hard limit) is: %r" %
(resource.getrlimit(resource.RLIMIT_CORE), ))
except ImportError:
print("Not checking corefile size as resource module is unavailable")
def run_command(command_tokens):
completed_process = subprocess.run(command_tokens,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
if completed_process.returncode != 0:
command = " ".join(command_tokens)
output = completed_process.stdout.decode('utf-8')
fail("Failed to execute '{command}'. Output was:\n\n{output}\n".format(
**locals()))
def test_strip_strange_font(test_font_dir):
install_command = "pyfiglet -L %s/TEST_ONLY.flf " % test_font_dir
subprocess32.run(install_command, shell=True, check=True)
command = "pyfiglet -f TEST_ONLY -s 0"
expected = '''\
0000000000
000 000
000 000
000 000
0000000000
'''
result = subprocess32.run(command, shell=True, stdout=subprocess32.PIPE)
assert result.stdout.decode() == expected
assert result.returncode == 0
def mount(partition):
mount_point = '/media/{}'.format(partition)
subprocess.run(['sudo', 'mkdir', mount_point], stderr=subprocess.PIPE)
already_mounted, _ = getstatusoutput('mountpoint {}'.format(mount_point))
if already_mounted == 0:
mount_point = '/media/{}'.format(uuid.uuid4().hex)
subprocess.run(['sudo', 'mkdir', mount_point], stderr=subprocess.PIPE)
success, message = getstatusoutput('sudo mount -o ro /dev/{} {}'.format(
partition, mount_point))
if success == 0:
print('Successfully mounted /dev/{} to {}\n'.format(
partition, mount_point))
else:
print('Something went wrong:')
print(message, end='\n\n')
def isAncestor(repo_dir, a, b): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc
# pylint: disable=missing-return-type-doc,missing-type-doc
"""Return true iff |a| is an ancestor of |b|. Throw if |a| or |b| does not exist."""
return subprocess.run(
["hg", "-R", str(repo_dir), "log", "-r", a + " and ancestor(" + a + "," + b + ")", "--template={node|short}"],
cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member
check=True,
stdout=subprocess.PIPE,
timeout=999
).stdout.decode("utf-8", errors="replace") != ""
def time_cmd(cmd, cwd=None, env=None, stderr=None, timeout=None):
"""Calculates and outputs the time a command takes.
Args:
cmd (list): Command to be run.
cwd (str): Working directory command is to be executed in.
env (dict): Working environment command is to be executed in.
stderr (bytes): stderr shown during command execution.
timeout (int): Timeout for the command.
"""
if not env:
env = os.environ.copy()
logger.info("Running `%s` now..", " ".join(cmd))
cmd_start = time.time()
subprocess.run(cmd, cwd=cwd, env=env, stderr=stderr, timeout=timeout)
cmd_end = time.time()
logger.info("`%s` took %.3f seconds.", " ".join(cmd), cmd_end - cmd_start)
def pinpoint(itest, logPrefix, jsEngine, engineFlags, infilename, # pylint: disable=invalid-name,missing-param-doc
bisectRepo, build_options_str, targetTime, suspiciousLevel):
# pylint: disable=missing-return-doc,missing-return-type-doc,missing-type-doc,too-many-arguments,too-many-locals
"""Run Lithium and autobisectjs.
itest must be an array of the form [module, ...] where module is an interestingness module.
The module's "interesting" function must accept [...] + [jsEngine] + engineFlags + infilename
(If it's not prepared to accept engineFlags, engineFlags must be empty.)
"""
lithArgs = itest + [str(jsEngine)] + engineFlags + [str(infilename)] # pylint: disable=invalid-name
(lithResult, lithDetails) = reduction_strat( # pylint: disable=invalid-name
logPrefix, infilename, lithArgs, targetTime, suspiciousLevel)
print()
print("Done running Lithium on the part in between DDBEGIN and DDEND. To reproduce, run:")
print(" ".join(quote(str(x)) for x in [sys.executable, "-u", "-m", "lithium", "--strategy=check-only"] + lithArgs))
print()
# pylint: disable=literal-comparison
if (bisectRepo is not "none" and targetTime >= 3 * 60 * 60 and
build_options_str is not None and testJsShellOrXpcshell(jsEngine) != "xpcshell"):
autobisectCmd = ( # pylint: disable=invalid-name
[sys.executable, "-u", "-m", "funfuzz.autobisectjs"] +
["-b", build_options_str] +
["-p", " ".join(engineFlags + [str(infilename)])] +
["-i"] + [str(x) for x in itest]
)
print(" ".join(quote(str(x)) for x in autobisectCmd))
autobisect_log = (logPrefix.parent / (logPrefix.stem + "-autobisect")).with_suffix(".txt")
with io.open(str(autobisect_log), "w", encoding="utf-8", errors="replace") as f:
subprocess.run(autobisectCmd, stderr=subprocess.STDOUT, stdout=f)
print("Done running autobisectjs. Log: %s" % autobisect_log)
with io.open(str(autobisect_log), "r", encoding="utf-8", errors="replace") as f:
lines = f.readlines()
autobisect_log_trunc = file_manipulation.truncateMid(lines, 50, ["..."])
else:
autobisect_log_trunc = []
return (lithResult, lithDetails, autobisect_log_trunc)
def start(databasePort):
logMsg('start {}'.format(databasePort), level='info')
if not databaseAlive(databasePort):
startMysql = ["/usr/bin/numactl", "--interleave=all", "/usr/local/mysql_{}/bin/mysqld_safe".format(
databasePort), "--defaults-file=/data/mysql_{}/my.cnf".format(databasePort)]
logMsg('run startMysql command: {}'.format(startMysql), level='debug')
try:
subprocess32.run(startMysql, stdout=subprocess32.PIPE, stderr=subprocess32.PIPE,
timeout=10)
except subprocess32.TimeoutExpired:
logMsg('start command:{} and get subprocess32.TimeoutExpired, but stop {} successful'.format(
startMysql, databasePort
), level='debug')
print "start {} successful".format(databasePort)
except:
logMsg('start {} failed'.format(databasePort), level='error')
print "start {} failed".format(databasePort)
else:
print "mysql-server: {} is started".format(databasePort)
logMsg("mysql-server {} is already started".format(databasePort), level='error')
def register_binfmt_misc():
try:
subprocess.run(
[
"sudo",
"docker",
"run",
"--rm",
"--privileged",
"multiarch/qemu-user-static:register",
],
check=True,
stdout=get_std(),
stderr=get_std(),
)
except subprocess.CalledProcessError:
# fmt: off
raise click.ClickException("""Could not register binfmt_misc.
Please register it with \
`sudo docker run --rm --privileged multiarch/qemu-user-static:register` command."""
)
def updateRepo(repo): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc,missing-return-doc
# pylint: disable=missing-return-type-doc,missing-type-doc
"""Update a repository. Return False if missing; return True if successful; raise an exception if updating fails."""
repo.is_dir()
repo_type = typeOfRepo(repo)
if repo_type == "hg":
hg_pull_cmd = ["hg", "--time", "pull", "-u"]
logger.info("\nRunning `%s` now..\n", " ".join(hg_pull_cmd))
out_hg_pull = subprocess.run(hg_pull_cmd,
check=True,
cwd=str(repo),
stderr=subprocess.PIPE)
logger.info(
'"%s" had the above output and took - %s',
subprocess.list2cmdline(out_hg_pull.args),
out_hg_pull.stderr.decode("utf-8", errors="replace").rstrip())
hg_log_default_cmd = ["hg", "--time", "log", "-r", "default"]
logger.info("\nRunning `%s` now..\n", " ".join(hg_log_default_cmd))
out_hg_log_default = subprocess.run(hg_log_default_cmd,
check=True,
cwd=str(repo),
stderr=subprocess.PIPE)
logger.info(
'"%s" had the above output and took - %s',
subprocess.list2cmdline(out_hg_log_default.args),
out_hg_log_default.stderr.decode("utf-8",
errors="replace").rstrip())
elif repo_type == "git":
# Ignore exit codes so the loop can continue retrying up to number of counts.
gitenv = deepcopy(os.environ)
if platform.system() == "Windows":
gitenv[
"GIT_SSH_COMMAND"] = "~/../../mozilla-build/msys/bin/ssh.exe -F ~/.ssh/config"
time_cmd([GITBINARY, "pull"], cwd=str(repo), env=gitenv)
else:
raise Exception("Unknown repository type: " + repo_type)
return True
def set_default_gw(ifname):
del_gw = ["route", "del", "default"]
add_gw = ["route", "add", "default", "gw"]
gw_ip = [g[0] for g in ni.gateways().get(ni.AF_INET, []) if g[1] == ifname]
# Check so we actually got a gw for that interface
if not gw_ip:
print("No gw set for {}".format(ifname))
print("Gws : {}".format(ni.gateways()))
return False
add_gw.extend([gw_ip[0], ifname])
try:
# We do not check delete as it might be that no default gw is set
# We do set a timeout for 10 seconds as a indicationm that
# something went bad though
if verbosity > 2:
print(del_gw)
run(del_gw, timeout=10)
if verbosity > 2:
print(add_gw)
# We check so this went OK
run(add_gw, timeout=10, check=True)
gws = ni.gateways()['default'].get(ni.AF_INET)
if not gws:
print("Default gw could no be set")
return False
if gws[1] == ifname:
print("Source interface is set to {}".format(ifname))
else:
print("Source interface {} is different from {}".format(
gws[1], ifname))
return False
except (CalledProcessError, TimeoutExpired) as e:
print("Error in set default gw : {}".format(e))
return False
return True
def existsAndIsAncestor(repo_dir, a, b): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc
# pylint: disable=missing-return-type-doc,missing-type-doc
"""Return true iff |a| exists and is an ancestor of |b|."""
# Note that if |a| is the same as |b|, it will return True
# Takes advantage of "id(badhash)" being the empty set, in contrast to just "badhash", which is an error
out = subprocess.run(
["hg", "-R", str(repo_dir), "log", "-r", a + " and ancestor(" + a + "," + b + ")", "--template={node|short}"],
cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
timeout=999
).stdout.decode("utf-8", errors="replace")
return out != "" and out.find("abort: unknown revision") < 0
def diffFiles(f1, f2): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc
"""Return a command to diff two files, along with the diff output (if it's short)."""
diffcmd = ["diff", "-u", str(f1), str(f2)]
s = " ".join(diffcmd) + "\n\n" # pylint: disable=invalid-name
diff = subprocess.run(diffcmd,
cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member
stdout=subprocess.PIPE,
timeout=99).stdout.decode("utf-8", errors="replace")
if len(diff) < 10000:
s += diff + "\n\n" # pylint: disable=invalid-name
else:
s += diff[:10000] + "\n(truncated after 10000 bytes)... \n\n" # pylint: disable=invalid-name
return s
def gather_coverage(dirpath):
"""Gathers coverage data.
Args:
dirpath (Path): Directory in which build is to be downloaded in.
Returns:
Path: Path to the coverage results file
"""
RUN_COV_LOG.info(
"Coverage build is being run in the following directory: %s",
str(dirpath))
bin_name = "js" + (".exe" if platform.system() == "Windows" else "")
cov_build_bin_path = dirpath / "cov-build" / "dist" / "bin" / bin_name
assert cov_build_bin_path.is_file()
loop_args = [
"--compare-jit", "--random-flags",
str(JS_SHELL_DEFAULT_TIMEOUT), "KNOWNPATH",
str(cov_build_bin_path), "--fuzzing-safe"
]
cov_timeout = 85000 # 85,000 seconds is just under a day
RUN_COV_LOG.info("Fuzzing a coverage build for %s seconds...",
str(cov_timeout))
many_timed_runs(cov_timeout, dirpath, loop_args,
create_collector.make_collector(), True)
RUN_COV_LOG.info("Finished fuzzing the coverage build")
fm_conf = configparser.SafeConfigParser()
fm_conf.read(
str(dirpath / "cov-build" / "dist" / "bin" / "js.fuzzmanagerconf"))
RUN_COV_LOG.info("Generating grcov data...")
cov_output = subprocess.run([
str(dirpath / "grcov-bin" / "grcov"),
str(dirpath), "-t", "coveralls+", "--commit-sha",
fm_conf.get("Main", "product_version"), "--token", "NONE", "-p",
"/srv/jenkins/jobs/mozilla-central-clone/workspace/"
],
check=True,
stdout=subprocess.PIPE).stdout.decode(
"utf-8", errors="replace")
RUN_COV_LOG.info("Finished generating grcov data")
RUN_COV_LOG.info("Writing grcov data to disk...")
cov_output_file = dirpath / "results_cov.json"
with io.open(str(cov_output_file), "w", encoding="utf-8",
errors="replace") as f:
f.write(cov_output)
RUN_COV_LOG.info("Finished writing grcov data to disk")
return cov_output_file
def runVaa3dPlugin(inFile, pluginName,
funcName, timeout = 30 * 60):
assert pl.Path(inFile).is_file(), "Input File {} not found".format(inFile)
vaa3dExec = getVaa3DExecutable()
pluginLabel = "{}_{}".format(pluginName, funcName)
virtDisplay = startVirtDisplay()
if platform.system() == "Windows":
toRun = [
vaa3dExec, "/i", inFile, "/x", pluginName, "/f", funcName
]
else:
toRun = [
vaa3dExec, "-i", inFile, "-x", pluginName, "-f", funcName
]
logging.info("[{}] Running {}".format(pluginLabel, toRun))
try:
compProc = subprocess.run(toRun,
timeout=timeout,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
)
log_subprocess_output(compProc.stdout, pluginLabel)
procOutput = compProc.stdout.decode("utf-8")
except subprocess.TimeoutExpired as te:
log_subprocess_output(te.stdout, pluginLabel)
procOutput = compProc.stdout.decode("utf-8")
logging.error("{} Process did not finish within {} seconds!!!".format(pluginLabel, timeout))
log_subprocess_output(te.stderr, pluginLabel)
except OSError as ose:
log_subprocess_output(ose.stdout, pluginLabel)
procOutput = compProc.stdout.decode("utf-8")
logging.error("{} OSError while running vaa3d plugin, for example,"
"a file is non existant".format(pluginLabel))
log_subprocess_output(ose.stderr, pluginLabel)
except ValueError as ve:
log_subprocess_output(ve.stdout, pluginLabel)
procOutput = compProc.stdout.decode("utf-8")
logging.error("{} Invalid arguments passed to the subprocess".format(pluginLabel))
log_subprocess_output(ve.stderr, pluginLabel)
except subprocess.CalledProcessError as spe:
log_subprocess_output(spe.stdout, pluginLabel)
procOutput = compProc.stdout.decode("utf-8")
logging.error("{} Subprocess exited with an unknown error!".format(pluginLabel))
log_subprocess_output(spe.stderr, pluginLabel)
stopVirtualDisplay(virtDisplay)
return procOutput
def check_dns(dns_list):
cmd = ["dig", "www.google.com", "+noquestion", "+nocomments", "+noanswer"]
data = dns_list.replace("\n", " ")
try:
out = run(cmd, timeout=10, check=True, stdout=PIPE).stdout
for line in out.splitlines():
for ip in re.findall(r'(?:\d{1,3}\.)+(?:\d{1,3})', data):
if ip in line:
print(line)
return True
except (CalledProcessError, TimeoutExpired) as e:
print("Error in dns check : {}".format(e))
return False
def run_packmol():
"""Run and check that Packmol worked correctly"""
try:
p = subprocess.run('packmol < {}'.format(PACKMOL_INP),
check=True,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except subprocess.CalledProcessError as e:
raise ValueError("Packmol failed with errorcode {}"
" and stderr: {}".format(e.returncode, e.stderr))
else:
with open('packmol.stdout', 'w') as out:
out.write(p.stdout.decode())
def __init__(self,
thestring,
thefilename,
location_of_autgroup_binary=None,
location_of_wordreduce_binary=None,
timeout=30):
if os.path.isfile(thefilename):
self.groupfilename = thefilename
self.location_of_autgroup_binary = location_of_autgroup_binary
self.location_of_wordreduce_binary = location_of_wordreduce_binary
else:
raise NameError('group definition file not found')
if not os.path.isfile(thefilename + '.diff1'):
if self.location_of_autgroup_binary is None:
subprocess32.run(['autgroup', '-silent', thefilename],
check=True,
timeout=timeout)
else:
subprocess32.run(
[self.location_of_autgroup_binary, '-silent', thefilename],
check=True,
timeout=timeout)
try:
self.string = wordreduce(thestring, thefilename,
self.location_of_wordreduce_binary)
except OSError: #sometimes wordreduce fails for unknown reasons and it is sufficient to just try again
try:
self.string = wordreduce(thestring, thefilename,
self.location_of_wordreduce_binary)
except OSError:
try:
self.string = wordreduce(
thestring, thefilename,
self.location_of_wordreduce_binary)
except OSError:
print thestring, thefilename
assert (False)
def qpop_qrm_applied_patch(patch_file, repo_dir):
"""Remove applied patch using `hg qpop` and `hg qdelete`.
Args:
patch_file (Path): Full path to the patch
repo_dir (Path): Working directory path
Raises:
OSError: Raises when `hg qpop` did not return a return code of 0
"""
qpop_result = subprocess.run(
["hg", "-R", str(repo_dir), "qpop"],
cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
timeout=99)
qpop_output, qpop_return_code = qpop_result.stdout.decode("utf-8", errors="replace"), qpop_result.returncode
if qpop_return_code != 0:
print("`hg qpop` output is: " + qpop_output)
raise OSError("Return code from `hg qpop` is: " + str(qpop_return_code))
print("Patch qpop'ed...", end=" ")
subprocess.run(["hg", "-R", str(repo_dir), "qdelete", patch_file.name], check=True)
print("Patch qdelete'd.")
def testBinary(shellPath, args, useValgrind): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc
# pylint: disable=missing-return-type-doc,missing-type-doc
"""Test the given shell with the given args."""
test_cmd = (constructVgCmdList() if useValgrind else []) + [str(shellPath)] + args
sps.vdump("The testing command is: " + " ".join(quote(str(x)) for x in test_cmd))
test_cmd_result = subprocess.run(
test_cmd,
cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member
env=env_with_path(str(shellPath.parent)),
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
timeout=999)
out, return_code = test_cmd_result.stdout.decode("utf-8", errors="replace"), test_cmd_result.returncode
sps.vdump("The exit code is: " + str(return_code))
return out, return_code
def test(self):
# TODO: add default values
build_dir = self.params.get('build_dir', default='.')
arch = self.params.get('arch', default='arm')
distro = self.params.get('distro', default='stretch')
self.log.info('===================================================')
self.log.info('Running Isar build test for (' + distro + '-' + arch + ')')
self.log.info('Isar build folder is: ' + build_dir)
self.log.info('===================================================')
#isar_root = dirname(__file__) + '/..'
os.chdir(build_dir)
cmdline = ['bitbake', 'multiconfig:qemu' + arch + '-' + distro + ':isar-image-base']
p1 = subprocess32.run(cmdline)
if p1.returncode:
self.fail('Test failed')
def Query(args):
"""Calls osquery with given query and returns its output.
Args:
args: A query to call osquery with.
Returns:
A "parsed JSON" representation of the osquery output.
Raises:
QueryError: If the query is incorrect.
TimeoutError: If a call to the osquery executable times out.
Error: If anything else goes wrong with the subprocess call.
"""
query = args.query.encode("utf-8")
timeout = args.timeout_millis / 1000 # `subprocess.run` uses seconds.
# TODO: pytype is not aware of the backport.
# pytype: disable=module-attr
try:
# We use `--S` to enforce shell execution. This is because on Windows there
# is only `osqueryd` and `osqueryi` is not available. However, by passing
# `--S` we can make `osqueryd` behave like `osqueryi`. Since this flag also
# works with `osqueryi`, by passing it we simply expand number of supported
# executable types.
command = [config.CONFIG["Osquery.path"], "--S", "--json", query]
proc = subprocess.run(
command,
timeout=timeout,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# TODO: Since we use a backported API, `SubprocessError` is hard
# to work with. Until support for Python 2 is dropped we re-raise with simpler
# exception type because we don't really care that much (the exception message
# should be detailed enough anyway).
except subprocess.TimeoutExpired as error:
raise TimeoutError(cause=error)
except subprocess.CalledProcessError as error:
raise Error("osquery invocation error", cause=error)
# pytype: enable=module-attr
stdout = proc.stdout.decode("utf-8")
stderr = proc.stderr.decode("utf-8").strip()
return ProcOutput(stdout=stdout, stderr=stderr)
def collectComputerTemperature(sensor_bin="sensors"):
result = subprocess.run((sensor_bin, '-u'),
universal_newlines=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
if result.returncode:
# This can happen on environments where sensors are not available,
# such as virtual machines.
if "No sensors found!" not in result.stdout:
raise RuntimeError("Error executing {}: {}".format(sensor_bin, result.stdout))
sensor_output_list = result.stdout.splitlines()
adapter_name = ""
sensor_temperature_list = []
for line_number, sensor_output in enumerate(sensor_output_list):
stripped_line = sensor_output.strip()
if stripped_line.startswith("Adapter:"):
adapter_name = sensor_output_list[line_number-1]
elif stripped_line.startswith("temp") and "_input" in stripped_line:
temperature = sensor_output.split()[-1]
found_sensor = ["%s %s" % (adapter_name, sensor_output_list[line_number-1]), float(temperature)]
critical = '1000'
maximal = '1000'
for next_line in sensor_output_list[line_number+1:line_number+3]:
stripped_next_line = next_line.strip()
if stripped_next_line.startswith("temp") and "_max" in stripped_next_line:
maximal = stripped_next_line.split()[-1]
elif stripped_next_line.startswith("temp") and "_crit" in stripped_next_line:
critical = stripped_next_line.split()[-1]
found_sensor.extend([float(maximal), float(critical)])
found_sensor.append(checkAlarm(float(temperature), float(maximal), float(critical)))
sensor_temperature_list.append(found_sensor)
return sensor_temperature_list
