def test_registry_add(self):
HEADING()
Benchmark.Start()
title = spec["info"]["title"]
url = spec["servers"][0]["url"]
print(f"add {title} -> {url}")
registry = Registry()
before = len(registry.list(name=title))
pid = 1
entry = registry.add(name=title, url=url, pid=pid)
pprint(entry)
after = len(registry.list(name=title))
assert after == before + 1
Benchmark.Stop()
def test_provider_volume_create(self):
HEADING()
os.system(f"cms volume list --cloud={cloud}")
name_generator.incr()
Benchmark.Start()
params = {
"NAME": name,
'size': None,
'volume_type': None,
'description': None,
'region': None,
'path': None
}
data = provider.create(**params)
Benchmark.Stop()
status = None
if cloud == "openstack" or cloud == "google":
for v in data:
status = v['status']
elif cloud == "oracle":
for v in data:
status = v['lifecycle_state']
elif cloud == "aws" or cloud == "multipass":
start_timeout = 360
time = 0
while time <= start_timeout:
sleep(15)
time += 15
status = provider.status(name=name)[0]['State']
if status == "available":
break
elif cloud == "azure":
status = provider.status(name=name)[0]['disk_state']
assert status in [
'available', 'AVAILABLE', 'PROVISIONING', 'READY', 'Unattached'
]
def test_generate(self):
"""
test whether python file can successfully be generated into yaml
:return:
"""
HEADING()
Benchmark.Start()
print("Generating yaml")
try:
Shell.run(
f"cms openapi generate --all_functions --filename={path}/image.py"
)
print("Successfully generated image.yaml")
except Exception as e:
print(e)
assert False, "Could not generate"
file_list = os.listdir(path)
assert 'image.yaml' in file_list
Benchmark.Stop()
def test_ls_la_wrapper_multi_options(self):
HEADING()
Benchmark.Start()
r = shell.ls("-a", "-l")
Benchmark.Stop()
print(r)
def test_ls_la_string(self):
HEADING()
Benchmark.Start()
r = shell.execute('ls', "-l -a")
Benchmark.Stop()
print(r)
def test_pwd(self):
HEADING()
Benchmark.Start()
r = shell.execute('pwd')
Benchmark.Stop()
print(r)
def test_list_secgroups(self):
HEADING()
Benchmark.Start()
groups = provider.list_secgroups()
Benchmark.Stop()
provider.Print(groups, output='json', kind="secgroup")
def test_clear_local_database(self):
HEADING()
Benchmark.Start()
cm.clear(collection=f"local-key")
Benchmark.Stop()
assert True
def test_key_delete(self):
HEADING()
Benchmark.Start()
self.test_clear_cloud_database()
Benchmark.Stop()
def test_key(self):
HEADING()
Benchmark.Start()
location = Location()
assert location.key == "CLOUDMESH_CONFIG_DIR"
Benchmark.Stop()
def test_equal(self):
HEADING()
Benchmark.Start()
location = Location()
Benchmark.Stop()
assert location == "a"
def main(argv):
Benchmark.Start()
home = os.environ['HOME']
script_output_dir = f"{home}/.cloudmesh/eigenfaces-svm/vm_script_output_multi/"
benchmark_output_dir = f"{home}/.cloudmesh/eigenfaces-svm/benchmark_output_multi/"
if not os.path.exists(f"{home}/.cloudmesh/eigenfaces-svm"):
os.mkdir(f"{home}/.cloudmesh/eigenfaces-svm")
if not os.path.exists(script_output_dir):
os.mkdir(script_output_dir)
if not os.path.exists(benchmark_output_dir):
os.mkdir(benchmark_output_dir)
# Run script to launch VMs and benchmark OpenAPI service if command line arg "run" passed
if len(argv) > 1 and argv[1] == "run":
clouds = ['aws', 'google', 'azure']
runtimes_dic = {'google': [], 'aws': [], 'azure': []}
num_trials = 1
vms = []
print(f"Running {num_trials} trials for each cloud in {clouds}")
for cloud in clouds:
Shell.run(f"cms set cloud={cloud}")
for i in range(num_trials):
vm_name = f"{cloud}-{i}"
print(f"Creating and running test on VM {vm_name}")
start = default_timer()
result = Shell.run(
f"{home}/cm/cloudmesh-openapi/tests/generator-eigenfaces-svm/eigenfaces-svm-full-multi-script {vm_name} > {script_output_dir}{cloud}-{i}"
)
end = default_timer()
print(f"Script on {vm_name} finished in {end - start} seconds")
runtimes_dic[cloud].append(end - start)
ip = Shell.run(
f'cms vm ssh {vm_name} --command="which" | grep -E -o -m 1 "([0-9]{{1,3}}[\.]){{3}}[0-9]{{1,3}}"'
).split()[0]
vms.append((ip, cloud, i))
runtimes = np.asarray(runtimes_dic[cloud])
print(f"\n{cloud} script run time mean: {runtimes.mean()}")
print(f"{cloud} script run time min: {runtimes.min()}")
print(f"{cloud} script run time max: {runtimes.max()}")
print(f"{cloud} script run time std: {runtimes.std()}\n")
#Run tests
with Pool(3) as p:
p.map(test_ai_workflow, vms)
#Delete VMs
for cloud in clouds:
Shell.run(f"cms set cloud={cloud}")
for i in range(num_trials):
Shell.run(f'cms vm terminate {cloud}-{i}')
# Scarpe benchmark output from script outputs
print(f'Scraping benchmarks from script output at {script_output_dir}')
script_outputs = os.listdir(script_output_dir)
for file in script_outputs:
with open(f"{script_output_dir}{file}", 'r') as f:
b = open(f"{benchmark_output_dir}{file}-benchmark", "w")
b.write(
"csv,timer,status,time,sum,start,tag,uname.node,user,uname.system,platform.version\n"
)
found_benchmark = False
for line in f.readlines():
if line.startswith(
"# csv,benchmark-eigenfaces-multi-cloud/test"
): # some shells return csv info twice once as error "info" and normal output
found_benchmark = True
b.write(line[2:]) # keep csv,...
b.close()
if not found_benchmark:
print(f"Error on script {script_output_dir}{file}")
if os.path.exists(f"{benchmark_output_dir}{file}-benchmark"):
os.remove(f"{benchmark_output_dir}{file}-benchmark")
# Read benchmark output and compute statistics
print(
f'Reading benchmarks from benchmark output at {benchmark_output_dir}')
columns = [
"csv", "timer", "status", "time", "sum", "start", "tag", "uname.node",
"user", "uname.system", "platform.version, cloud"
]
benchmark_df = pd.DataFrame(columns=columns)
benchmark_outputs = os.listdir(benchmark_output_dir)
for file in benchmark_outputs:
cloud = file.split("-")[0]
df = pd.read_csv(f"{benchmark_output_dir}{file}")
df['cloud'] = cloud
if cloud == 'aws':
df.loc[df['uname.node'].str.startswith("ip"),
["uname.node"]] = 'aws' + "-" + file.split("-")[1]
benchmark_df = pd.concat([benchmark_df, df])
benchmark_df['test_type'] = 'local'
benchmark_df.loc[benchmark_df['uname.node'] == gethostname(),
['test_type']] = 'remote'
print("Printing trial statistics:")
result = ""
stats_df = pd.DataFrame(
columns=['test', 'type', 'cloud', 'mean', 'min', 'max', 'std'])
for cloud in benchmark_df['cloud'].unique():
result += f"{cloud} has {len(benchmark_df.loc[benchmark_df['cloud'] == cloud]['uname.node'].unique())} VM samples.\n"
for timer in benchmark_df['timer'].unique():
for test_type in benchmark_df['test_type'].unique():
df = benchmark_df.loc[(benchmark_df['cloud'] == cloud) &
(benchmark_df['timer'] == timer) &
(benchmark_df['test_type'] == test_type),
['time']]
if len(df.values) > 0:
mean = df.values.mean()
min = df.values.min()
max = df.values.max()
std = df.values.std()
result += f"{cloud} {timer} {test_type} samples: {len(df.values)}\n"
result += f"{cloud} {timer} {test_type} mean: {mean}\n"
result += f"{cloud} {timer} {test_type} min: {min}\n"
result += f"{cloud} {timer} {test_type} max: {max}\n"
result += f"{cloud} {timer} {test_type} std: {std}\n\n"
to_append = [timer, test_type, cloud, mean, min, max, std]
stats_series = pd.Series(to_append, index=stats_df.columns)
stats_df = stats_df.append(stats_series, ignore_index=True)
print(result)
stats_df = stats_df.round(decimals=2)
stats_df['test'] = stats_df['test'].str.replace(
"benchmark-eigenfaces-multi-cloud/", "")
# print(stats_df_print.sort_values(by=['test', 'type', 'cloud']).to_markdown(index=False))
print(
stats_df.sort_values(by=['test', 'type', 'cloud']).to_latex(
index=False))
# graph 1: stacked bar graph of all tests
download_df = stats_df.loc[(stats_df['test'] == 'test_download_data')]
download_means = download_df["mean"]
download_mins = download_df["min"]
download_stds = download_df["std"]
download_labels = download_df["cloud"]
train_df = stats_df.loc[(stats_df['test'] == 'test_train')]
train_means = train_df["mean"]
train_mins = train_df["min"]
train_stds = train_df["std"]
upload_df = stats_df.loc[(stats_df['test'] == 'test_upload')]
upload_means = upload_df["mean"]
upload_mins = upload_df["min"]
upload_stds = upload_df["std"]
predict_df = stats_df.loc[(stats_df['test'] == 'test_predict')]
predict_means = predict_df["mean"]
predict_mins = predict_df["min"]
predict_stds = predict_df["std"]
plt.style.use('seaborn-whitegrid')
n = 3
ind = np.arange(n)
width = 0.35
p1 = plt.bar(ind,
download_means,
width,
yerr=download_stds,
color='orange',
capsize=3)
p2 = plt.bar(ind,
train_means,
width,
bottom=download_means,
yerr=train_stds,
color='green',
capsize=3)
p3 = plt.bar(ind,
upload_means,
width,
bottom=download_means.values + train_means.values,
yerr=upload_stds,
color='yellow',
capsize=3)
p4 = plt.bar(ind,
predict_means,
width,
bottom=download_means.values + train_means.values +
upload_means.values,
yerr=predict_stds,
color='blue',
capsize=3,
ecolor="gray")
plt.ylabel('Time (s)')
plt.title('AI Service Workflow Runtime')
plt.xticks(ind, download_labels)
plt.legend((p1[0], p2[0], p3[0], p4[0]),
('Download Data', 'Train', 'Upload', 'Predict'),
bbox_to_anchor=(0, 0),
loc='lower left',
ncol=4,
frameon=True)
plt.savefig('ai_service_workflow_runtime.png')
plt.savefig('ai_service_workflow_runtime.pdf')
plt.savefig('ai_service_workflow_runtime.svg')
plt.show()
def test_train(ip):
Benchmark.Start()
r = requests.get(f"http://{ip}:8080/cloudmesh/EigenfacesSVM/train")
Benchmark.Stop()
assert r.status_code == 200
def test_download_data(ip):
Benchmark.Start()
r = requests.get(f"http://{ip}:8080/cloudmesh/EigenfacesSVM/download_data")
Benchmark.Stop()
assert r.status_code == 200
def main(argv):
Benchmark.Start()
home = os.environ['HOME']
script_output_dir = f"{home}/.cloudmesh/eigenfaces-svm/vm_script_output/"
benchmark_output_dir = f"{home}/.cloudmesh/eigenfaces-svm/benchmark_output/"
if not os.path.exists(f"{home}/.cloudmesh/eigenfaces-svm"):
os.mkdir(f"{home}/.cloudmesh/eigenfaces-svm")
if not os.path.exists(script_output_dir):
os.mkdir(script_output_dir)
if not os.path.exists(benchmark_output_dir):
os.mkdir(benchmark_output_dir)
# Run script to launch VMs and benchmark OpenAPI service if command line arg "run" passed
if len(argv) > 1 and argv[1] == "run":
clouds = ['aws', 'google', 'azure']
runtimes_dic = {'google': [], 'aws': [], 'azure': []}
num_trials = 3
print(f"Running {num_trials} trials for each cloud in {clouds}")
for cloud in clouds:
Shell.run(f"cms set cloud={cloud}")
for i in range(num_trials):
vm_name = f"{cloud}-{i}"
print(f"Creating and running test on VM {vm_name}")
start = default_timer()
result = Shell.run(
f"{home}/cm/cloudmesh-openapi/tests/generator-eigenfaces-svm/eigenfaces-svm-full-script {vm_name} > {script_output_dir}{cloud}-{i}"
)
end = default_timer()
print(f"Script on {vm_name} finished in {end - start} seconds")
runtimes_dic[cloud].append(end - start)
runtimes = np.asarray(runtimes_dic[cloud])
print(f"\n{cloud} script run time mean: {runtimes.mean()}")
print(f"{cloud} script run time min: {runtimes.min()}")
print(f"{cloud} script run time max: {runtimes.max()}")
print(f"{cloud} script run time std: {runtimes.std()}\n")
# Scarpe benchmark output from script outputs
print(f'Scraping benchmarks from script output at {script_output_dir}')
script_outputs = os.listdir(script_output_dir)
for file in script_outputs:
with open(f"{script_output_dir}{file}", 'r') as f:
b = open(f"{benchmark_output_dir}{file}-benchmark", "w")
b.write(
"csv,timer,status,time,sum,start,tag,uname.node,user,uname.system,platform.version\n"
)
found_benchmark = False
for line in f.readlines():
if line[0:
10] == "# csv,test": #some shells return csv info twice once as error "info" and normal output
found_benchmark = True
b.write(line[2:]) #keep csv,...
b.close()
if not found_benchmark:
print(f"Error on script {script_output_dir}{file}")
if os.path.exists(f"{benchmark_output_dir}{file}-benchmark"):
os.remove(f"{benchmark_output_dir}{file}-benchmark")
# Read benchmark output and compute statistics
print(
f'Reading benchmarks from benchmark output at {benchmark_output_dir}')
columns = [
"csv", "timer", "status", "time", "sum", "start", "tag", "uname.node",
"user", "uname.system", "platform.version, cloud"
]
benchmark_df = pd.DataFrame(columns=columns)
benchmark_outputs = os.listdir(benchmark_output_dir)
for file in benchmark_outputs:
cloud = file.split("-")[0]
df = pd.read_csv(f"{benchmark_output_dir}{file}")
df['cloud'] = cloud
if cloud == 'aws':
df.loc[df['uname.node'].str.startswith("ip"),
["uname.node"]] = 'aws' + "-" + file.split("-")[1]
benchmark_df = pd.concat([benchmark_df, df])
benchmark_df['test_type'] = 'local'
benchmark_df.loc[benchmark_df['uname.node'] == gethostname(),
['test_type']] = 'remote'
print("Printing trial statistics:")
result = ""
stats_df = pd.DataFrame(
columns=['test', 'type', 'cloud', 'mean', 'min', 'max', 'std'])
for cloud in benchmark_df['cloud'].unique():
result += f"{cloud} has {len(benchmark_df.loc[benchmark_df['cloud']==cloud]['uname.node'].unique())-1} VM samples.\n"
for timer in benchmark_df['timer'].unique():
for test_type in benchmark_df['test_type'].unique():
df = benchmark_df.loc[(benchmark_df['cloud'] == cloud) &
(benchmark_df['timer'] == timer) &
(benchmark_df['test_type'] == test_type),
['time']]
if len(df.values) > 0:
mean = df.values.mean()
min = df.values.min()
max = df.values.max()
std = df.values.std()
result += f"{cloud} {timer} {test_type} samples: {len(df.values)}\n"
result += f"{cloud} {timer} {test_type} mean: {mean}\n"
result += f"{cloud} {timer} {test_type} min: {min}\n"
result += f"{cloud} {timer} {test_type} max: {max}\n"
result += f"{cloud} {timer} {test_type} std: {std}\n\n"
to_append = [timer, test_type, cloud, mean, min, max, std]
stats_series = pd.Series(to_append, index=stats_df.columns)
stats_df = stats_df.append(stats_series, ignore_index=True)
print(result)
stats_df = stats_df.round(decimals=2)
stats_df['test'] = stats_df['test'].str.replace(
"test_030_generator_eigenfaces_svm/test_", "")
#print(stats_df_print.sort_values(by=['test', 'type', 'cloud']).to_markdown(index=False))
print(
stats_df.sort_values(by=['test', 'type', 'cloud']).to_latex(
index=False))
#pi_series = pd.Series(["test_download_data", "local", "pi", 135.5, 135.5, 135.5, 0.0], index=stats_df.columns)
#stats_df = stats_df.append(pi_series, ignore_index=True)
#pi_series = pd.Series(["test_scikitlearn_train", "local", "pi", 232.0, 232.0, 232.0, 0.0], index=stats_df.columns)
#stats_df = stats_df.append(pi_series, ignore_index=True)
#pi_series = pd.Series(["test_train", "local", "pi", 231.0, 231.0, 231.0, 0.0], index=stats_df.columns)
#stats_df = stats_df.append(pi_series, ignore_index=True)
#pi_series = pd.Series(["test_upload", "local", "pi", 0.05, 0.05, 0.05, 0.0], index=stats_df.columns)
#stats_df = stats_df.append(pi_series, ignore_index=True)
#pi_series = pd.Series(["test_predict", "local", "pi", 0.4, 0.4, 0.4, 0.0], index=stats_df.columns)
#stats_df = stats_df.append(pi_series, ignore_index=True)
sorter = ['aws', 'azure', 'google', 'mac book', 'docker', 'pi 4', 'pi 3b+']
stats_df.cloud = stats_df.cloud.astype("category")
stats_df.cloud.cat.set_categories(sorter, inplace=True)
stats_df = stats_df.sort_values(["cloud"])
if "pi 3b+" in stats_df['cloud'].unique():
cost_df = stats_df[['test', 'type', 'cloud', 'mean']]
#cost_df['cost/s'] = 0
#cost_df['cost'] = 0
cost_df.loc[cost_df['cloud'] == 'aws', ['cost/s']] = 0.1 / 60.0 / 60.0
cost_df.loc[cost_df['cloud'] == 'azure',
['cost/s']] = 0.096 / 60.0 / 60.0
cost_df.loc[cost_df['cloud'] == 'google',
['cost/s']] = 0.0949995 / 60.0 / 60.0
cost_df.loc[cost_df['cloud'] == 'pi 3b+',
['cost/s']] = 0.006546804 / 60.0 / 60.0
cost_df.loc[cost_df['cloud'] == 'pi 4',
['cost/s']] = 0.013324201 / 60.0 / 60.0
cost_df['cost'] = cost_df['mean'].values * cost_df['cost/s'].values
for test in cost_df['test'].unique():
for type in cost_df['type'].unique():
if type == 'remote':
continue
sub_df = cost_df.loc[(cost_df['test'] == test)
& (cost_df['type'] == type)]
pi_cost = sub_df.loc[cost_df['cloud'] == 'pi 3b+',
'cost'].values
pi_mean = sub_df.loc[cost_df['cloud'] == 'pi 3b+',
'mean'].values
cost_inc = (sub_df['cost'].values - pi_cost) / pi_cost * 100
mean_dec = (sub_df['mean'].values * -1 +
pi_mean) / pi_mean * 100
cost_df.loc[(cost_df['test'] == test) &
(cost_df['type'] == type),
["% runtime decrease"]] = mean_dec
cost_df.loc[(cost_df['test'] == test) &
(cost_df['type'] == type),
["% cost increase"]] = cost_inc
cost_df["% cost increase"] = cost_df["% cost increase"].round(2)
cost_df["% runtime decrease"] = cost_df["% runtime decrease"].round(2)
cost_df = cost_df.drop(columns='cost/s')
#pd.set_option('display.float_format', '{:.2E}'.format)
print(
cost_df.sort_values(by=['test', 'type', 'cloud']).to_latex(
index=False, formatters={'cost': '{:,.2e}'.format}))
suffix = ""
if "pi 3b+" in stats_df['cloud'].unique():
suffix = "_pi"
# graph 1: download_data_local
download_df = stats_df.loc[(stats_df['test'] == 'download_data')]
download_means = download_df["mean"]
download_mins = download_df["min"]
download_stds = download_df["std"]
download_labels = download_df["cloud"]
#plt.style.use('ggplot')
plt.style.use('seaborn-whitegrid')
x = download_labels
x_pos = [i for i, _ in enumerate(x)]
#plt.bar(x_pos, download_means, yerr=download_stds,capsize=3, color=["green",'orange','blue', 'red'])
plt.bar(x_pos, download_means, yerr=download_stds, capsize=3)
plt.xlabel("Cloud")
plt.ylabel("Seconds")
plt.title("Time to Download and Extract Data")
plt.xticks(x_pos, x)
plt.savefig(f'sample_graph_1{suffix}.png')
plt.savefig(f'sample_graph_1{suffix}.pdf')
plt.savefig(f'sample_graph_1{suffix}.svg')
plt.show()
# graph 2: scikitlearn_train vs opeanpi_scikitlearn_train
openapi_df = stats_df.loc[(stats_df['test'] == 'train')]
openapi_means = openapi_df['mean']
openapi_mins = openapi_df['min']
openapi_stds = openapi_df['std']
openapi_labels = openapi_df['cloud']
scikitlearn_df = stats_df.loc[(stats_df['test'] == 'scikitlearn_train')]
scikit_means = scikitlearn_df['mean']
scikit_mins = scikitlearn_df['min']
scikit_stds = scikitlearn_df['std']
scikit_labels = scikitlearn_df['cloud']
x = openapi_labels
ind = np.arange(len(openapi_labels))
width = 0.35
#openapi_handles = plt.bar(ind, openapi_means, width, yerr=openapi_stds, capsize=3, color=["green", 'orange', 'blue', 'red'])
#scikit_handles = plt.bar(ind + width, scikit_means, width, yerr=scikit_stds, capsize=3, color=["springgreen", 'bisque', 'skyblue', 'lightcoral'])
openapi_handles = plt.bar(ind,
openapi_means,
width,
yerr=openapi_stds,
capsize=3)
scikit_handles = plt.bar(ind + width,
scikit_means,
width,
yerr=scikit_stds,
capsize=3)
plt.xlabel("Cloud")
plt.ylabel("Seconds")
plt.title("Model Training Time")
plt.xticks(ind + width / 2, scikit_labels)
#plt.legend([tuple(openapi_handles), tuple(scikit_handles)], ['OpenAPI service', 'Scikit-learn example'], numpoints=1,
# handler_map={tuple: HandlerTuple(ndivide=None)},frameon=True)
plt.legend(['train', 'scikitlearn train'], frameon=True)
plt.savefig(f'sample_graph_2{suffix}.png')
plt.savefig(f'sample_graph_2{suffix}.pdf')
plt.savefig(f'sample_graph_2{suffix}.svg')
plt.show()
# graph 3: upload_local vs upload_remote
local_df = stats_df.loc[(stats_df['test'] == 'upload')
& (stats_df['type'] == 'local')]
local_means = local_df['mean']
local_mins = local_df['min']
local_stds = local_df['std']
local_labels = local_df['cloud']
remote_df = stats_df.loc[(stats_df['test'] == 'upload')
& (stats_df['type'] == 'remote')]
remote_means = remote_df['mean']
remote_mins = remote_df['min']
remote_stds = remote_df['std']
remote_labels = remote_df['cloud']
x = local_labels
ind = np.arange(len(local_labels))
width = 0.35
#local_handels = plt.bar(ind, local_means, width, yerr=local_stds, capsize=3, color=["green", 'orange', 'blue', 'red'])
local_handels = plt.bar(ind,
local_means,
width,
yerr=local_stds,
capsize=3)
ind = np.arange(len(remote_labels))
#remote_handles = plt.bar(ind + width, remote_means, width, yerr=remote_stds,capsize=3,
# color=["springgreen", 'bisque', 'skyblue'])
remote_handles = plt.bar(ind + width,
remote_means,
width,
yerr=remote_stds,
capsize=3)
ind = np.arange(len(local_labels))
plt.xlabel("Cloud")
plt.ylabel("Seconds")
plt.title("Upload Function Runtime")
plt.xticks(ind + width / 2, local_labels)
#plt.legend([tuple(local_handels), tuple(remote_handles)], ['OpenAPI server', 'Remote client'], numpoints=1,
# handler_map={tuple: HandlerTuple(ndivide=None)}, frameon=True)
plt.legend(['local', 'remote'], frameon=True)
plt.savefig(f'sample_graph_3{suffix}.png')
plt.savefig(f'sample_graph_3{suffix}.pdf')
plt.savefig(f'sample_graph_3{suffix}.svg')
plt.show()
# graph 4 predict_local vs predict_remote
local_df = stats_df.loc[(stats_df['test'] == 'predict')
& (stats_df['type'] == 'local')]
local_means = local_df['mean']
local_mins = local_df['min']
local_stds = local_df['std']
local_labels = local_df['cloud']
remote_df = stats_df.loc[(stats_df['test'] == 'predict')
& (stats_df['type'] == 'remote')]
remote_means = remote_df['mean']
remote_mins = remote_df['min']
remote_stds = remote_df['std']
remote_labels = remote_df['cloud']
x = local_labels
ind = np.arange(len(local_labels))
width = 0.35
#local_handels = plt.bar(ind, local_means, width, yerr=local_stds, capsize=3, color=["green", 'orange', 'blue', 'red'])
local_handels = plt.bar(ind,
local_means,
width,
yerr=local_stds,
capsize=3)
ind = np.arange(len(remote_labels))
#remote_handles = plt.bar(ind + width, remote_means, width, yerr=remote_stds, capsize=3,
# color=["springgreen", 'bisque', 'skyblue'])
remote_handles = plt.bar(ind + width,
remote_means,
width,
yerr=remote_stds,
capsize=3)
ind = np.arange(len(local_labels))
plt.xlabel("Cloud")
plt.ylabel("Seconds")
plt.title("Predict Function Runtime")
plt.xticks(ind + width / 2, local_labels)
#plt.legend([tuple(local_handels), tuple(remote_handles)], ['OpenAPI server', 'Remote client'], numpoints=1,
# handler_map={tuple: HandlerTuple(ndivide=None)}, frameon=True)
plt.legend(['local', 'remote'], frameon=True)
plt.savefig(f'sample_graph_4{suffix}.png')
plt.savefig(f'sample_graph_4{suffix}.pdf')
plt.savefig(f'sample_graph_4{suffix}.svg')
plt.show()
Benchmark.Stop()
Benchmark.print()
return
def test_cms_flavor_refresh(self):
HEADING()
Benchmark.Start()
os.system(
f"cms flavor list --cloud={cloud} --refresh > flavor-{cloud}.log")
Benchmark.Stop()
def test_cms_flavor(self):
HEADING()
Benchmark.Start()
os.system(f"cms flavor list > flavor-local.log")
Benchmark.Stop()
def test_rename(self):
HEADING()
Benchmark.Start()
provider.rename(source=self.name, destination=self.new_name)
Benchmark.Stop()
def test_cms_local(self):
HEADING()
Benchmark.Start()
os.system("cms key add")
os.system("cms key list > key-local.log")
Benchmark.Stop()
def test_sec_group_list_cloud(self):
HEADING()
Benchmark.Start()
result = run(f"cms sec group list --cloud={cloud}")
Benchmark.Stop()
g = groups.list()
def test_clear_cloud_database(self):
HEADING()
Benchmark.Start()
cm.clear(collection=f"{cloud}-key")
Benchmark.Stop()
assert True
def test_rule_load_to_cloud(self):
HEADING()
Benchmark.Start()
result = run(f"cms sec group load deleteme --cloud={cloud}")
Benchmark.Stop()
def test_list_secgroups_rules(self):
HEADING()
Benchmark.Start()
rule_groups = provider.list_secgroup_rules()
Benchmark.Stop()
provider.Print(output='json', kind="secgroup", data=rule_groups)
def test_list(self):
HEADING()
Benchmark.Start()
r = g.list(name='test')
Benchmark.Stop()
def test_ls_la_list(self):
HEADING()
Benchmark.Start()
r = shell.execute('ls', ["-l", "-a"])
Benchmark.Stop()
print(r)
def test_benchmark_stopwatch_1(self):
HEADING()
Benchmark.Start()
time.sleep(0.1)
Benchmark.Stop()
assert True
def test_ls_la_wrapper(self):
HEADING()
Benchmark.Start()
r = shell.ls("-la")
Benchmark.Stop()
print(r)
def test_empty_database(self):
HEADING()
Benchmark.Start()
cm.clear(collection=f"{cloud}-flavor")
Benchmark.Stop()
def test_cms_init(self):
HEADING()
Benchmark.Start()
result = os.system(f"cms init")
Benchmark.Stop()
def test_provider_flavor(self):
HEADING()
Benchmark.Start()
r = provider.flavors()
Benchmark.Stop()
