def consumption_lower_than_app(sample, app, use_case=None):
"""Test that a given sample spends less energy than a known app.
Args:
sample (list of Measurement): sample of measurements
app (string): identifier/package of the app to be compared
use_case (string): select only data from a given use case
"""
baseline_measurements = Measurement.get_all_entries_of_app(app, use_case)
baseline_consumption = Measurement.mean_energy_consumption(
baseline_measurements)
consumption_below(sample, baseline_consumption)
def profile(self,
power_meter=default_power_meter,
verbose=True,
count=30,
retry_limit=1,
save_to_csv=None):
"""Run a batch of measurements.
Args:
power_meter Power meter to use in measurements.
verbose Log activiy (default=True).
count Run experiment several times (default=30).
retry_limit Number of times to retry on error.
save_to_csv File name to store measurement.
Returns: Set of measurements
"""
results = []
for i in range(count):
result = self.run(power_meter=power_meter, retry_limit=retry_limit)
if result:
results.append(result)
if save_to_csv:
result.save_to_csv(save_to_csv)
else:
click.secho("Error in execution {} of {}. Skipping.".format(
i, self.name),
fg="red")
if verbose and results:
click.secho("Energy consumption results for {}: "
"{:.3f} Joules (s = {:.3f}).\n"
"It took {:.1f} seconds (s = {:.1f}).".format(
self.app_pkg, *Measurement.describe(results)),
fg='green')
return results
def test_get_unique_use_cases(self):
measurements = []
for _ in range(10):
measurements.append(create_measurement(use_case="one"))
measurements.append(create_measurement(use_case="two"))
measurements.append(create_measurement(use_case="three"))
self.assertEqual(Measurement.get_unique_use_cases(measurements),
{"one", "two", "three"})
def consumption_below(sample, energy_consumption_baseline):
"""Test for energy consumption lower than a given value in Joules (avg).
Args:
sample (list of Measurement): sample of measurements
energy_consumption (number): baseline energy consumption in Joules.
"""
energy_consumption_mean = Measurement.mean_energy_consumption(sample)
assert energy_consumption_mean < energy_consumption_baseline
def test_get_unique_apps(self):
for _ in range(10):
measurement = create_measurement(app_pkg="com.test.one")
measurement.persist()
measurement = create_measurement(app_pkg="com.test.two")
measurement.persist()
measurement = create_measurement(app_pkg="com.test.three")
measurement.persist()
self.assertEqual(set(Measurement.get_unique_apps()),
{"com.test.one", "com.test.two", "com.test.three"})
def test_get_energy_ranking(self):
sample = (create_random_sample(10, 1, app_pkg="com.app1") +
create_random_sample(11, 1, app_pkg="com.app2") +
create_random_sample(12, 1, app_pkg="com.app3") +
create_random_sample(13, 1, app_pkg="com.app4") +
create_random_sample(14, 1, app_pkg="com.app5") +
create_random_sample(15, 1, app_pkg="com.app6"))
for measurement in sample:
measurement.persist()
ranking = Measurement.get_energy_ranking()
self.assertEqual(list(ranking.keys()), [
"com.app1",
"com.app2",
"com.app3",
"com.app4",
"com.app5",
"com.app6",
])
compare_sample = create_random_sample(12.5, 0.5, app_pkg="com.app7")
self.assertEqual(Measurement.get_position_in_ranking(compare_sample),
(4, 6))
def test_describe_app_use_case(self):
for i in range(10):
measurement = create_measurement(use_case="login",
duration=i,
energy_consumption=i * 2)
measurement.persist()
real_stats = Measurement.describe_app_use_case(measurement.app_pkg,
measurement.use_case)
expected_stats = (9, 5.745, 4.5, 2.872)
# pairwise assertion:
for (first, second) in zip(real_stats, expected_stats):
self.assertAlmostEqual(first, second, places=3)
def create_measurement(use_case='login',
app_pkg='com.package',
duration=2,
energy_consumption=30):
"""Fake data for measurement."""
return Measurement(
1485634263.096069, # timestamp
use_case, # use_case
app_pkg, # application package
'1.0.0', # version
'Nexus 5X', # device model
duration, # duration
energy_consumption # energy consumption
)
def top_percentile(sample, nth):
"""Test that a given sample is in the top nth percentile.
Args:
sample (list of Measurement): sample of measurements
nth (number): percentage of the position in which the sample should fit
app (string): identifier of the application within the sample should be compared
use_case (string: identifier of the use case used to create the ranking
"""
position, total = Measurement.get_position_in_ranking(sample)
percentile_position = float(position) / total * 100
assert percentile_position <= nth,\
("Given sample is not on {:.1f}% top percentile "
"(Position: {:.1f}%)".format(
nth,
percentile_position
))
def run(self, power_meter=default_power_meter, retry_limit=1):
"""Measure the routine stored in `_run`.
Returns:
Measurement: data collected from experiment
"""
try:
self.prepare()
power_meter.start()
success = self._run()
energy_consumption, duration, error_flag = power_meter.stop()
self.cleanup()
if error_flag:
raise PhysaliaExecutionFailed()
measurement = Measurement(time.time(), self.name, self.app_pkg,
self.app_version,
android_utils.get_device_model(),
duration, energy_consumption,
str(power_meter), success is None
or success, self.notes)
return measurement
except KeyboardInterrupt as error:
raise error
except BaseException as error:
click.secho("Measurement {} has failed".format(self.name),
fg='red')
click.secho(str(error), fg='red')
if retry_limit == 1:
# click.secho("Waiting 5 minutes for Monsoon to recover...", fg='red')
# time.sleep(5*60)
# hack: reinit power_meter
power_meter.reinit()
if retry_limit > 0:
click.secho("Retrying...", fg='yellow')
return self.run(power_meter, retry_limit - 1)
printstack = getattr(error, "printStackTrace", None)
if callable(printstack):
printstack()
raise error
def tool(results_input, results_output):
with open(results_input, 'rt') as csv_file:
csv_reader = csv.reader(csv_file)
data = []
for row in csv_reader:
# row[6] = float(row[6])*1000 # convert to mJ
data.append(Measurement(*row))
if not os.path.isdir(results_output):
os.makedirs(results_output)
use_case_categories = [
"find_by_id",
"find_by_description",
"find_by_content",
"tap",
"long_tap",
"multi_finger_tap",
"dragndrop",
"swipe",
"pinch_and_spread",
"back_button",
"input_text",
]
scores = defaultdict(lambda: 0)
for use_case_category in use_case_categories:
click.secho("----------------------------------------", fg="blue")
click.secho(" {}".format(use_case_category), fg="blue")
click.secho("----------------------------------------", fg="blue")
use_case_data = list(
Measurement.get_entries_with_name_like("-" + use_case_category,
data))
unique_use_cases = list(
Measurement.get_unique_use_cases(use_case_data))
number_of_frameworks = len(unique_use_cases)
names_dict = {
name: name.replace("-" + use_case_category, "")
for name in unique_use_cases
}
groups = [
list(Measurement.get_entries_with_name(use_case, use_case_data))
for use_case in unique_use_cases
]
names = [
name.replace("-" + use_case_category, "")
for name in unique_use_cases
]
names, groups = zip(*sorted(zip(names, groups)))
title = use_case_category.title().replace('_', " ")
violinplot(*groups,
save_fig=results_output + "/" + use_case_category + ".pdf",
names_dict=names_dict,
sort=True,
millijoules=True)
n_loop_iterations = _get_interactions_count(use_case_category)
# Descriptive statistics
with open(
results_output + "/table_description_" + use_case_category +
".tex", "w") as file:
table = describe(*groups,
names=names,
loop_count=n_loop_iterations,
ranking=True,
out=file,
table_fmt="latex",
float_fmt='.3f',
mili_joules=True)
# Update Ranking
for name, row in zip(names, table):
scores[name] += (number_of_frameworks -
row["Rank"]) / float(number_of_frameworks)
# Welchs ttest
with open(
results_output + "/table_welchsttest_" + use_case_category +
".tex", "w") as file:
pairwise_welchs_ttest(*groups,
names=names,
out=file,
table_fmt='latex')
# Ranking
click.secho("\nRanking".format(use_case_category), fg="blue")
sorted_scores = sorted(scores.items(), key=itemgetter(1), reverse=True)
with open(results_output + "/table_ranking.tex", "w") as file:
file.write(
tabulate(sorted_scores,
headers=["Framework", "Score"],
tablefmt="latex",
floatfmt=".4f"))
frameworks = [
"AndroidViewClient",
"Appium",
"Calabash",
"Espresso",
"Monkeyrunner",
"PythonUiAutomator",
"Robotium",
"UiAutomator",
]
framework_results_dir = results_output + "/frameworks/"
if not os.path.isdir(framework_results_dir):
os.makedirs(framework_results_dir)
for framework in frameworks:
means = []
for interaction in use_case_categories:
use_case = "{}-{}".format(framework, interaction)
use_case_data = np.array(list(
Measurement.get_entries_with_name(use_case, data)),
dtype='float')
if len(use_case_data):
n_loop_iterations = _get_interactions_count(interaction)
mean = np.mean(use_case_data) / n_loop_iterations * 1000
else:
mean = 0
bisect.insort(means, (interaction, mean))
# means = means[::-1]
figure = plt.figure()
figure.suptitle(framework)
X = range(len(means))
names, Y = zip(*means)
plt.bar(X, Y)
axes = figure.gca()
axes.set_xticks(X)
axes.set_xticklabels(
[name.replace("_", " ").title() for name in names],
rotation='vertical')
axes.set_ylabel("Energy Consumption (mJ)")
figure.subplots_adjust(bottom=0.31)
bar_labels = [y == 0 and "n.a." or format(y, ".2f") for y in Y]
for x, y, label in zip(X, Y, bar_labels):
plt.text(x, y, label, ha='center', va='bottom')
figure.savefig(results_output + "/frameworks/" + framework)