def runBenchmark(self, cmd, *args, **kwargs):
if not isinstance(cmd, list):
cmd = shlex.split(cmd)
# profiling is not supported on android
if "platform_args" in kwargs and \
"enable_profiling" in kwargs["platform_args"]:
del kwargs["platform_args"]["enable_profiling"]
if "platform_args" in kwargs and \
"profiler_args" in kwargs["platform_args"]:
del kwargs["platform_args"]["profiler_args"]
# meta is used to store any data about the benchmark run
# that is not the output of the command
meta = {}
# We know this command may fail. Avoid propogating this
# failure to the upstream
success = getRunStatus()
self.util.logcat('-b', 'all', '-c')
setRunStatus(success, overwrite=True)
if self.app:
log = self.runAppBenchmark(cmd, *args, **kwargs)
else:
log = self.runBinaryBenchmark(cmd, *args, **kwargs)
return log, meta
def run(self):
raw_args = self._getRawArgs()
if "--remote" in raw_args or "--lab" in raw_args:
# server address must start with http
assert "--server_addr" in raw_args
idx = raw_args.index("--server_addr")
assert raw_args[idx + 1].startswith("http") or len(
raw_args[idx + 1]) == 0
if "--lab" in raw_args and "--remote_reporter" not in raw_args:
raw_args.extend([
"--remote_reporter",
raw_args[idx + 1] +
("" if raw_args[idx + 1][-1] == "/" else "/") +
"benchmark/store-result|oss",
])
app = self.repoCls(raw_args=raw_args)
ret = app.run()
if "--query_num_devices" in self.unknowns:
return ret
if "--fetch_status" in self.unknowns or "--fetch_result" in self.unknowns:
return ret
if "--list_devices" in self.unknowns:
return ret
if ret is not None:
setRunStatus(ret >> 8)
return getRunStatus()
def __init__(self, **kwargs):
setRunStatus(0, overwrite=True)
self.status = 0
raw_args = kwargs.get("raw_args", None)
self.usb_controller = kwargs.get("usb_controller")
self.args, self.unknowns = parser.parse_known_args(raw_args)
self._lock = threading.Lock()
def preprocess(self, *args, **kwargs):
assert "programs" in kwargs, "Must have programs specified"
programs = kwargs["programs"]
# find the first zipped app file
assert "program" in programs, "program is not specified"
program = programs["program"]
assert program.endswith(".ipa"), \
"IOS program must be an ipa file"
processRun(["unzip", "-o", "-d", self.tempdir, program])
# get the app name
app_dir = os.path.join(self.tempdir, "Payload")
dirs = [f for f in os.listdir(app_dir)
if os.path.isdir(os.path.join(app_dir, f))]
assert len(dirs) == 1, "Only one app in the Payload directory"
app_name = dirs[0]
self.app = os.path.join(app_dir, app_name)
del programs["program"]
bundle_id, _ = processRun(["osascript", "-e",
"id of app \"" + self.app + "\""])
assert len(bundle_id) > 0, "bundle id cannot be found"
self.util.setBundleId(bundle_id[0].strip())
# We know this command will fail. Avoid propogating this
# failure to the upstream
success = getRunStatus()
self.util.run(["--bundle", self.app, "--uninstall", "--justlaunch"])
setRunStatus(success, overwrite=True)
def run(self):
raw_args = self._getRawArgs()
app = self.repoCls(raw_args=raw_args)
ret = app.run()
if ret and json.loads(ret):
return ret
if ret is not None:
setRunStatus(ret >> 8)
return getRunStatus()
def run(self):
try:
if self.args.interval:
while not stopRun(self.args.status_file):
self._buildExecutables()
time.sleep(self.args.interval)
else:
# single run
self._buildExecutables()
except Exception:
setRunStatus(2)
getLogger().error(traceback.format_exc())
def runBenchmark(self, cmd, *args, **kwargs):
if not isinstance(cmd, list):
cmd = shlex.split(cmd)
# We know this command may fail. Avoid propogating this
# failure to the upstream
success = getRunStatus()
self.util.logcat('-b', 'all', '-c')
setRunStatus(success, overwrite=True)
if self.app:
log = self.runAppBenchmark(cmd, *args, **kwargs)
else:
log = self.runBinaryBenchmark(cmd, *args, **kwargs)
return log
def _setLogCatSize(self):
repeat = True
size = 131072
while (repeat and size > 256):
repeat = False
# We know this command may fail. Avoid propogating this
# failure to the upstream
success = getRunStatus()
ret = self.util.logcat("-G", str(size) + "K")
setRunStatus(success, overwrite=True)
if len(ret) > 0 and ret[0].find("failed to") >= 0:
repeat = True
size = int(size / 2)
def _runOneBenchmarkSuite(self, repo_info):
cmd = self._getCommand(repo_info)
getLogger().info("Running: %s", cmd)
if not _runIndividual():
# always sleep 10 seconds to make the phone in a more
# consistent state
time.sleep(10)
# cannot use subprocess because it conflicts with requests
ret = os.system(cmd)
setRunStatus(ret >> 8)
if getArgs().commit_file and getArgs().regression:
with open(getArgs().commit_file, 'w') as file:
file.write(repo_info['treatment']['commit'])
getLogger().info("One benchmark run {} for ".format(
"successful" if ret == 0 else "failed") +
repo_info['treatment']['commit'])
def killProgram(self, program):
basename = os.path.basename(program)
# if the program doesn't exist, the grep may fail
# do not update status code
success = getRunStatus()
res = self.util.shell(["ps", "|", "grep", basename])
setRunStatus(success, overwrite=True)
if len(res) == 0:
return
results = res[0].split("\n")
pattern = re.compile(r"^shell\s+(\d+)\s+")
for result in results:
match = pattern.match(result)
if match:
pid = match.group(1)
self.util.shell(["kill", pid])
def run(self):
raw_args = self._getRawArgs()
if "--remote" in raw_args:
# server address must start with http
assert "--server_addr" in raw_args
idx = raw_args.index("--server_addr")
assert raw_args[idx+1].startswith("http") or len(raw_args[idx+1]) == 0
app = self.repoCls(raw_args=raw_args)
ret = app.run()
if "--query_num_devices" in self.unknowns:
return ret
if "--fetch_status" in self.unknowns or "--fetch_result" in self.unknowns:
return ret
if ret is not None:
setRunStatus(ret >> 8)
return getRunStatus()
def _runOneBenchmarkSuite(self, repo_info):
raw_args = self._getRawArgs(repo_info)
if not _runIndividual(self.args.interval, self.args.regression,
self.args.ab_testing):
# always sleep 10 seconds to make the phone in a more
# consistent state
time.sleep(10)
# cannot use subprocess because it conflicts with requests
app = BenchmarkDriver(raw_args=raw_args)
app.run()
ret = 0
setRunStatus(ret >> 8)
if self.args.commit_file and self.args.regression:
with open(self.args.commit_file, 'w') as file:
file.write(repo_info['treatment']['commit'])
getLogger().info("One benchmark run {} for ".format(
"successful" if ret == 0 else "failed") +
repo_info['treatment']['commit'])
def runBenchmark(self, cmd, *args, **kwargs):
if not isinstance(cmd, list):
cmd = shlex.split(cmd)
# meta is used to store any data about the benchmark run
# that is not the output of the command
meta = {}
# We know this command may fail. Avoid propogating this
# failure to the upstream
success = getRunStatus()
self.util.logcat('-b', 'all', '-c')
setRunStatus(success, overwrite=True)
if self.app:
log, meta = self.runAppBenchmark(cmd, *args, **kwargs)
else:
log, meta = self.runBinaryBenchmark(cmd, *args, **kwargs)
return log, meta
def run(self):
cmd = self._getCMD()
getLogger().info("Running: %s", cmd)
ret = os.system(cmd)
setRunStatus(ret >> 8)
def runOneBenchmark(info,
benchmark,
framework,
platform,
backend,
reporters,
lock,
cooldown=None,
user_identifier=None,
local_reporter=None):
assert "treatment" in info, "Treatment is missing in info"
getLogger().info("Running {}".format(benchmark["path"]))
status = 0
minfo = copy.deepcopy(info["treatment"])
mbenchmark = copy.deepcopy(benchmark)
if "shared_libs" in info:
minfo["shared_libs"] = info["shared_libs"]
try:
data = _runOnePass(minfo, mbenchmark, framework, platform)
meta = None
if "control" in info:
cinfo = copy.deepcopy(info["control"])
if "shared_libs" in info:
cinfo["shared_libs"] = info["shared_libs"]
# cool down between treatment and control
if "model" in benchmark and "cooldown" in benchmark["model"]:
cooldown = float(benchmark["model"]["cooldown"])
time.sleep(cooldown)
control = _runOnePass(cinfo, benchmark, framework, platform)
bname = benchmark["model"]["name"]
data = _mergeDelayData(data, control, bname)
if benchmark["tests"][0]["metric"] != "generic":
data = _adjustData(info, data)
meta = _retrieveMeta(info, benchmark, platform, framework, backend,
user_identifier)
result = {"meta": meta, "data": data}
except Exception as e:
# Catch all exceptions so that failure in one test does not
# affect other tests
getLogger().info("Exception caught when running benchmark")
getLogger().info(e)
data = None
status = 2
setRunStatus(status)
getLogger().error(traceback.format_exc())
if data is None or len(data) == 0:
name = platform.getMangledName()
model_name = ""
if "model" in benchmark and "name" in benchmark["model"]:
model_name = benchmark["model"]["name"]
commit_hash = ""
if "commit" in info["treatment"]:
commit_hash = info["treatment"]["commit"]
getLogger().info("No data collected for ".format(model_name) +
"on {}. ".format(name) +
"The run may be failed for " +
"{}".format(commit_hash))
return status
with lock:
for reporter in reporters:
reporter.report(result)
if "regression_commits" in info and \
info["run_type"] == "benchmark" and local_reporter:
from regression_detectors.regression_detectors import checkRegressions
checkRegressions(info, platform, framework, benchmark, reporters,
result['meta'], local_reporter)
return status
def postprocess(self, *args, **kwargs):
success = getRunStatus()
self.util.run(["--bundle", self.app, "--uninstall_only"])
setRunStatus(success, overwrite=True)
def run(self):
raw_args = self._getRawArgs()
app = RepoDriver(raw_args=raw_args)
ret = app.run()
setRunStatus(ret >> 8)
sys.exit(getRunStatus())
def runOneBenchmark(
info,
benchmark,
framework,
platform,
backend,
reporters,
lock,
cooldown=None,
user_identifier=None,
local_reporter=None,
):
assert "treatment" in info, "Treatment is missing in info"
getLogger().info("Running {}".format(benchmark["path"]))
status = 0
minfo = copy.deepcopy(info["treatment"])
mbenchmark = copy.deepcopy(benchmark)
if "shared_libs" in info:
minfo["shared_libs"] = info["shared_libs"]
try:
# invalidate CPU cache
[1.0 for _ in range(20 << 20)]
gc.collect()
data = _runOnePass(minfo, mbenchmark, framework, platform)
status = status | getRunStatus()
meta = None
if "control" in info:
cinfo = copy.deepcopy(info["control"])
if "shared_libs" in info:
cinfo["shared_libs"] = info["shared_libs"]
# cool down between treatment and control
if "model" in benchmark and "cooldown" in benchmark["model"]:
cooldown = float(benchmark["model"]["cooldown"])
time.sleep(cooldown)
# invalidate CPU cache
[1.0 for _ in range(20 << 20)]
gc.collect()
control = _runOnePass(cinfo, benchmark, framework, platform)
status = status | getRunStatus()
bname = benchmark["model"]["name"]
data = _mergeDelayData(data, control, bname)
if benchmark["tests"][0]["metric"] != "generic":
data = _adjustData(info, data)
meta = _retrieveMeta(info, benchmark, platform, framework, backend,
user_identifier)
data = _retrieveInfo(info, data)
result = {"meta": meta, "data": data}
except Exception as e:
# Catch all exceptions so that failure in one test does not
# affect other tests
getLogger().info("Exception caught when running benchmark")
getLogger().info(e)
data = None
status = 2
setRunStatus(status)
getLogger().error(traceback.format_exc())
# Set result meta and data to default values to that
# the reporter will not try to key into a None
result = {"meta": {}, "data": []}
if data is None or len(data) == 0:
_logNoData(benchmark, info, platform.getMangledName())
return status
with lock:
for reporter in reporters:
reporter.report(result)
if ("regression_commits" in info and info["run_type"] == "benchmark"
and local_reporter):
from regression_detectors.regression_detectors import checkRegressions
checkRegressions(
info,
platform,
framework,
benchmark,
reporters,
result["meta"],
local_reporter,
)
return status
def runBinaryBenchmark(self, cmd, *args, **kwargs):
log_to_screen_only = 'log_to_screen_only' in kwargs and \
kwargs['log_to_screen_only']
platform_args = {}
meta = {}
if "platform_args" in kwargs:
platform_args = kwargs["platform_args"]
if "taskset" in platform_args:
taskset = platform_args["taskset"]
cmd = ["taskset", taskset] + cmd
del platform_args["taskset"]
if "sleep_before_run" in platform_args:
sleep_before_run = str(platform_args["sleep_before_run"])
cmd = ["sleep", sleep_before_run, "&&"] + cmd
del platform_args["sleep_before_run"]
if "power" in platform_args and platform_args["power"]:
# launch settings page to prevent the phone
# to go into sleep mode
self.util.shell(
["am", "start", "-a", "android.settings.SETTINGS"])
time.sleep(1)
cmd = ["nohup"] + ["sh", "-c", "'" + " ".join(cmd) + "'"] + \
[">", "/dev/null", "2>&1"]
platform_args["non_blocking"] = True
del platform_args["power"]
if platform_args.get("enable_profiling", False):
# attempt to run with profiling, else fallback to standard run
success = getRunStatus()
try:
simpleperf = getProfilerByUsage(
"android",
None,
platform=self,
model_name=platform_args.get("model_name", None),
cmd=cmd)
if simpleperf:
f = simpleperf.start()
output, meta = f.result()
log_logcat = []
if not log_to_screen_only:
log_logcat = self.util.logcat('-d')
return output + log_logcat, meta
# if this has not succeeded for some reason reset run status and run without profiling.
except RuntimeError as ex:
getLogger().error(
"An error occurred when running Simpleperf profiler. {}"
.format(ex))
setRunStatus(success, overwrite=True)
except FileNotFoundError as ex:
getLogger().error(
"An error occurred when running Simpleperf profiler. {}"
.format(ex))
setRunStatus(success, overwrite=True)
except Exception:
getLogger().exception(
"An error has occurred when running Simpleperf profiler."
)
setRunStatus(success, overwrite=True)
log_screen = self.util.shell(cmd, **platform_args)
log_logcat = []
if not log_to_screen_only:
log_logcat = self.util.logcat('-d')
return log_screen + log_logcat, meta
