def generic_event_generator(workload) -> Tuple[Dict[str, List[float]], int]:
"""
This function returns a list of times and applications calls given a workload description.
"""
logger_eg = ScriptLogger('event_generator', "SWI.log")
logger_eg.info("Started Generic Event Generator")
test_duration_in_seconds = workload['test_duration_in_seconds']
all_events = {}
event_count = 0
random_seed = workload['random_seed']
logger_eg.info('random_seed: ' + str(random_seed))
instance: str
desc: Dict[str, Any]
for (instance, desc) in workload['instances'].items():
if 'interarrivals_list' in desc.keys():
instance_events = cast(List[Any], desc['interarrivals_list'])
logger_eg.info('Read the invocation time trace for ' + instance)
# enforcing maximum test duration
list_len = 0
cutoff_index = None
for i in range(len(instance_events)):
list_len += instance_events[i]
if list_len > test_duration_in_seconds:
cutoff_index = i
break
if cutoff_index is not None:
instance_events = instance_events[:cutoff_index]
else:
instance_events = create_events(instance=instance,
dist=desc['distribution'],
rate=desc['rate'],
duration=test_duration_in_seconds,
seed=random_seed)
try:
start_time = desc['activity_window'][0]
end_time = desc['activity_window'][1]
instance_events = EnforceActivityWindow(
start_time, end_time, instance_events)
except:
instance_events = EnforceActivityWindow(
0, workload['test_duration_in_seconds'], instance_events)
all_events[instance] = instance_events
event_count += len(instance_events)
logger_eg.info("Returning workload event list")
return (all_events, event_count)
def GenericEventGenerator(workload):
"""
This function returns a list of times and applications calls given a workload description.
"""
log_file = workload['log_dir'] + '/SWI.log'
logger = ScriptLogger('workload_invoker', log_file)
logger.info("Started Generic Event Generator")
test_duration_in_seconds = workload['test_duration_in_seconds']
all_events = {}
event_count = 0
random_seed = workload['random_seed']
logger.info('random_seed: ' + str(random_seed))
for (instance, desc) in workload['instances'].items():
if 'interarrivals_list' in desc.keys():
instance_events = desc['interarrivals_list']
logger.info('Read the invocation time trace for ' + instance)
else:
instance_events = CreateEvents(instance=instance,
dist=desc['distribution'],
rate=desc['rate'],
duration=test_duration_in_seconds,
seed=random_seed)
try:
start_time = desc['activity_window'][0]
end_time = desc['activity_window'][1]
instance_events = EnforceActivityWindow(
start_time, end_time, instance_events)
except:
instance_events = EnforceActivityWindow(0,
workload['test_duration_in_seconds'],
instance_events)
all_events[instance] = instance_events
event_count += len(instance_events)
logger.info("Returning workload event list")
return [all_events, event_count]
# Local imports
sys.path = ['./', '../'] + sys.path
from GenConfigs import *
sys.path = [FAAS_ROOT + '/synthetic-workload-invoker'] + sys.path
from EventGenerator import GenericEventGenerator
from commons.JSONConfigHelper import CheckJSONConfig, ReadJSONConfig
from commons.Logger import ScriptLogger
from WorkloadChecker import CheckWorkloadValidity
logging.captureWarnings(True)
# Global variables
supported_distributions = {'Poisson', 'Uniform'}
logger = ScriptLogger('workload_invoker', 'SWI.log')
APIHOST = subprocess.check_output(WSK_PATH + " property get --apihost",
shell=True).split()[3]
APIHOST = 'https://' + APIHOST.decode("utf-8")
AUTH_KEY = subprocess.check_output(WSK_PATH + " property get --auth",
shell=True).split()[2]
AUTH_KEY = AUTH_KEY.decode("utf-8")
user_pass = AUTH_KEY.split(':')
NAMESPACE = subprocess.check_output(WSK_PATH + " property get --namespace",
shell=True).split()[2]
NAMESPACE = NAMESPACE.decode("utf-8")
RESULT = 'false'
base_url = APIHOST + '/api/v1/namespaces/' + NAMESPACE + '/actions/'
base_gust_url = APIHOST + '/api/v1/web/guest/default/'
import yaml
import sys, getopt
from typing import List
import traceback
import asyncio
import json
import logging
from Clusters import BaseDeployment
from Clusters import OpenWhiskDeployment
from Clusters import GoogleDeployment
functions_meta = []
from commons.Logger import ScriptLogger
logging.basicConfig(level=logging.DEBUG)
logger = ScriptLogger(__name__, 'SWI.log')
logger.setLevel(logging.DEBUG)
logging.captureWarnings(True)
async def deploy_to_clusters(configfile: str, provider: str, scenario_name: str, functions_list: list,
cluster_obj: BaseDeployment = None,
providers_list: list = None, all_clusters: bool = False):
with open(configfile, 'r') as stream:
try:
data = yaml.safe_load(stream)
if all_clusters:
for cluster in data['providers'][provider]:
curr_cluster = data['providers'][provider][cluster]
scenario = data['scenarios'][scenario_name]
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1.inset_locator import inset_axes, mark_inset, zoomed_inset_axes, InsetPosition
import os
import pandas as pd
import pickle
import seaborn as sns
import sys
import time
sys.path = ['./', '../'] + sys.path
# Local
from GenConfigs import *
from commons.Logger import ScriptLogger
logger = ScriptLogger(loggername='comparative_analyzer', logfile='CA.log')
archive_folder = FAAS_ROOT + "/data_archive/"
def GetTimeFromDFName(dfname):
res = dfname[0:16]
res = datetime.strptime(res, '%Y_%m_%d_%H_%M')
return res
def ComparativePlotting(t_df, p_df_dic):
"""
Plotting result comparisons.
"""
dims = {
's': 'start',
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
import datetime
import logging
import os
import sys
# Local imports
sys.path = ['./', '../'] + sys.path
from GenConfigs import *
sys.path = [FAAS_ROOT + '/synthetic-workload-invoker'] + sys.path
from commons.Logger import ScriptLogger
logger_wlch = ScriptLogger('workload_checker', 'SWI.log')
def CheckWorkloadValidity(workload, supported_distributions):
"""
Checks whether a loaded workload is valid.
"""
logger_wlch.info("Started CheckWorkloadValidity")
# 1 - Check if the workload has been successfully read in ReadJSONConfig
if workload is None:
logger_wlch.info('Workload not valid => Terminating')
return False
# 2 - Check for validity of general field
print(workload)
fields_to_check = [['test_name', str], ['blocking_cli', bool]]
for field in fields_to_check:
def CheckWorkloadValidity(workload, supported_distributions):
"""
Checks whether a loaded workload is valid.
"""
log_file = workload['log_dir'] + '/SWI.log'
logger = ScriptLogger('workload_invoker', log_file)
logger.info("Started CheckWorkloadValidity")
# 1 - Check if the workload has been successfully read in ReadJSONConfig
if workload is None:
logger.info('Workload not valid => Terminating')
return False
# 2 - Check for validity of general field
print(workload)
fields_to_check = [['test_name', str], ['blocking_cli', bool]]
for field in fields_to_check:
try:
print([field, workload[field[0]]])
if type(workload[field[0]]) is not field[1]:
test_name('Input of the ' + field[0] + ' field should be a ' +
str(field[1]))
return False
except:
logger.error('No ' + field[0] + ' field provided!')
return False
# # 3 - Check if invocation scripts exists for all functions/applications in the workload
application_set = set()
distribution_set = set()
for (instance, specs) in workload['instances'].items():
application_set.add(specs['application'])
try:
distribution_set.add(specs['distribution'])
except:
pass
logger.info('Required applications: ' + str(application_set))
# all_scripts_available = True
# for application in application_set:
# if not os.path.isfile(FAAS_ROOT + '/invocation-scripts/'+application+'.sh'):
# logger.error(
# 'No invocation script available in invocation-scripts for the following application: '+application)
# all_scripts_available = False
# if not all_scripts_available:
# logger.info('Incomplete invocation scripts => Terminating')
# return False
# else:
# logger.info('Script files for all applications exist')
# 4 - Check for supported distributions
if not distribution_set.issubset(supported_distributions):
logger.error(
'At least one specified distribution is not supported. Supported distribution(s): '
+ str(supported_distributions))
return False
# 5 - Check for valid test duration
try:
test_duration_in_seconds = workload['test_duration_in_seconds']
if test_duration_in_seconds is None:
logger.error(
'Please enter a valid value for test_duration_in_seconds field in the config file.'
)
return False
elif int(test_duration_in_seconds) <= 0:
logger.error(
'test_duration_in_seconds should be greater than zero!')
return False
except:
logger.error(
'test_duration_in_seconds field not specified in the json config file'
)
return False
# 6 - Check that the random_seed field is entered
try:
random_seed = workload['random_seed']
except:
logger.error("No random_seed field specified in the config file")
return False
return True
# Copyright (c) 2019 Princeton University
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
import datetime
import logging
import numpy as np
import random
from commons.Logger import ScriptLogger
logger_eg = ScriptLogger('event_generator', 'SWI.log')
def CreateEvents(instance, dist, rate, duration, seed=None):
"""
Creates a dictionary of application instances
"""
inter_arrivals = []
if rate == 0:
return inter_arrivals
if dist == "Uniform":
random.seed(seed + hash(instance))
shift = random.random() / rate
inter_arrivals = int(duration * rate) * [1.0 / rate]
inter_arrivals[0] += shift
elif dist == "Poisson":
np.random.seed(seed)
def main(argv):
"""
The main function.
"""
parser = OptionParser()
parser.add_option("-n",
"--test_name",
dest="test_name",
default="latest_test",
help="Name of test",
metavar="FILE")
parser.add_option("-r",
"--rate_override",
dest="rate_override",
help="Override rate of invocation from arguments",
metavar="FILE")
parser.add_option(
"-c",
"--config_json",
dest="config_json",
help="The input json config file describing the synthetic workload.",
metavar="FILE")
parser.add_option("-b", "--benchmark", dest="benchmark", metavar="FILE")
parser.add_option("-p", "--param_file", dest="param_file", metavar="FILE")
(options, args) = parser.parse_args()
log_dir, log_file = createDir(options.test_name)
logger = ScriptLogger('workload_invoker', log_file)
logger.info("Workload Invoker started")
print("Log file -> ", log_file, "\n")
if not CheckJSONConfig(options.config_json):
logger.error("Invalid or no JSON config file!")
return False # Abort the function if json file not valid
workload = ReadJSONConfig(options.config_json)
workload = ApplyJSONOverrides(workload, log_dir, options.rate_override,
options.benchmark, options.param_file)
if not CheckWorkloadValidity(
workload=workload,
supported_distributions=supported_distributions):
return False # Abort the function if json file not valid
[all_events, event_count] = GenericEventGenerator(workload)
threads = []
for (instance, instance_times) in all_events.items():
# Previous method to run processes
# instance_script = 'bash ' + FAAS_ROOT + '/invocation-scripts/' + \
# workload['instances'][instance]['application']+'.sh'
# threads.append(threading.Thread(target=PROCESSInstanceGenerator, args=[instance, instance_script, instance_times, workload['blocking_cli']]))
# New method
action = workload['instances'][instance]['application']
try:
param_file = workload['instances'][instance]['param_file']
except:
param_file = None
blocking_cli = workload['blocking_cli']
if 'data_file' in workload['instances'][instance].keys():
data_file = workload['instances'][instance]['data_file']
threads.append(
threading.Thread(
target=BinaryDataHTTPInstanceGenerator,
args=[action, instance_times, blocking_cli, data_file]))
else:
threads.append(
threading.Thread(target=HTTPInstanceGenerator,
args=[
action, instance_times, blocking_cli,
log_dir, param_file
]))
pass
# Dump Test Metadata
metadata_file = log_dir + "/test_metadata.out"
os.system("date +%s%N | cut -b1-13 > " + FAAS_ROOT + '/' + metadata_file)
os.system("echo " + options.config_json + " >> " + FAAS_ROOT + '/' +
metadata_file)
os.system("echo " + str(event_count) + " >> " + FAAS_ROOT + '/' +
metadata_file)
try:
if workload['perf_monitoring']['runtime_script']:
runtime_script = 'bash ' + FAAS_ROOT + '/' + workload['perf_monitoring']['runtime_script'] + \
' ' + str(int(workload['test_duration_in_seconds'])) + ' ' + FAAS_ROOT + '/' + log_dir + '/perf-mon.out' + ' &'
logger.info(runtime_script)
os.system(runtime_script)
logger.info("Runtime monitoring script ran")
except:
pass
logger.info("Test started")
for thread in threads:
thread.start()
logger.info("Test ended")
return True
class WorkloadInvoker:
supported_distributions = {'Poisson', 'Uniform'}
@staticmethod
def gen_invocation_id(test_name, runid):
out = subprocess.check_output(["git", "rev-parse", "HEAD"],
cwd=FAAS_ROOT)
commit_hash = out.decode('utf-8').strip()
test_dir_name = "{}_{}_{}".format(commit_hash[0:10], test_name, runid)
return (commit_hash, test_dir_name)
def __init__(self, config_json):
# Global variables
self.logger = None
APIHOST = subprocess.check_output(
WSK_PATH + " property get --apihost",
shell=True).split()[3].decode("utf-8")
APIHOST = APIHOST if APIHOST.lower().startswith(
"http") else 'https://' + APIHOST
AUTH_KEY = subprocess.check_output(WSK_PATH + " property get --auth",
shell=True).split()[2]
AUTH_KEY = AUTH_KEY.decode("utf-8")
self.user_pass = AUTH_KEY.split(':')
NAMESPACE = subprocess.check_output(WSK_PATH +
" property get --namespace",
shell=True).split()[2]
NAMESPACE = NAMESPACE.decode("utf-8")
self.RESULT = 'false'
self.base_url = APIHOST + '/api/v1/namespaces/' + NAMESPACE + '/actions/'
self.base_gust_url = APIHOST + '/api/v1/web/guest/default/'
# Generate a runid for this instance
self.runid = util.gen_random_hex_string(8)
self.param_file_cache = {} # a cache to keep json of param files
self.binary_data_cache = {} # a cache to keep binary data
# (image files, etc.)
# Count of function invocations successfully submitted to FaaS queue
self.invocation_success_tally = 0
self.invocation_failure_tally = 0
self.invocation_expected_tally = 0
self.tally_lock = threading.Lock()
self.config_json = config_json
self.workload = WorkloadInvoker.read_json_config(config_json)
# Set name and commit hash and create destination dir if
# missing
self.my_commit_hash, test_result_dir_name =\
WorkloadInvoker.gen_invocation_id(self.workload["test_name"], self.runid)
self.test_result_dir_path = util.ensure_directory_exists(
os.path.join(DATA_DIR, test_result_dir_name))
@staticmethod
def read_json_config(config_json: str) -> Dict[Any, Any]:
if not check_json_config(config_json):
raise Exception("Invalid or no JSON config file!")
workload = read_json_config(config_json)
if not check_workload_validity(workload=workload,
supported_distributions=WorkloadInvoker.
supported_distributions):
# Abort the function if json file not valid
raise Exception("Workload JSON is invalid")
return workload
def handle_futures(self, futures) -> Tuple[int, int]:
"""
Wait for all futures in futures to complete. Returnes a tuple
containing the number of succseful and failed requests.
"""
failures = 0
successes = 0
for future in concurrent.futures.as_completed(futures):
try:
res = future.result()
except Exception as e:
self.logger.info("Request failed: " + str(e))
failures += 1
else:
prefix = ""
if res.status_code >= 200 and res.status_code <= 299:
prefix = "Request successful: "
successes += 1
else:
prefix = "Request failed: "
failures += 1
self.logger.info(prefix + str(res.status_code) + " " +
res.url)
return (successes, failures)
# @staticmethod
# def PROCESSInstanceGenerator(instance, instance_script, instance_times, blocking_cli):
# if len(instance_times) == 0:
# return False
# after_time, before_time = 0, 0
# if blocking_cli:
# pass
# else:
# instance_script = instance_script + ' &'
# for t in instance_times:
# time.sleep(max(0, t - (after_time - before_time)))
# before_time = time.time()
# os.system(instance_script)
# after_time = time.time()
# return True
#class InstanceGenerator(
def http_instance_generator(self,
action,
instance_times,
blocking_cli,
query_file=None,
param_file=None) -> None:
if len(instance_times) == 0:
raise Exception(
"http_instance_generator called without instance times")
session = FuturesSession(max_workers=100)
url = f"{self.base_url}{action}?{self.runid}"
assert (self.runid)
parameters = {'blocking': blocking_cli, 'result': self.RESULT}
args = {'testid': self.runid, 'body': None}
authentication = (self.user_pass[0], self.user_pass[1])
futures = []
if (not param_file is None) and (not query_file is None):
raise Exception(
"Only one of param_file and query_stirng can be set")
if param_file:
try:
param_file_body = self.param_file_cache[param_file]
except:
with open(param_file, 'r') as f:
param_file_body = json.load(f)
args['body'] = param_file_body
param_file_body = args
self.param_file_cache[param_file] = param_file_body
if query_file:
try:
query_file_body = self.param_file_cache[query_file]
except:
with open(query_file, 'r') as f:
query_file_body = json.load(f)
self.param_file_cache[query_file] = query_file_body
print("Updating paramters", str(parameters), str(query_file_body))
parameters.update(query_file_body)
print("Final parameters", str(parameters))
st = 0
after_time, before_time = 0, 0
for t in instance_times:
st = st + t - (after_time - before_time)
before_time = time.time()
if st > 0:
time.sleep(st)
# self.logger.info("Url " + url)
# self.logger.info("Setting params" + str(parameters))
future = session.post(url,
params=parameters,
auth=authentication,
json=args,
verify=False)
futures.append(future)
#print(future.result())
after_time = time.time()
(successes, failures) = self.handle_futures(futures)
with self.tally_lock:
self.invocation_success_tally += successes
self.invocation_failure_tally += failures
self.invocation_expected_tally += len(instance_times)
def binary_data_http_instance_generator(self, action, instance_times,
blocking_cli, data_file):
"""
TODO: Automate content type
"""
url = f"{self.base_gust_url}{action}?{self.runid}"
session = FuturesSession(max_workers=100)
if len(instance_times) == 0:
return False
after_time, before_time = 0, 0
futures = []
if not data_file in self.binary_data_cache:
data = open(data_file, 'rb').read()
self.binary_data_cache[data_file] = {}
self.binary_data_cache[data_file]["body"] = data
self.binary_data_cache[data_file]["mime"] = MimeTypes().guess_type(
data_file)[0]
file_body = self.binary_data_cache[data_file]["body"]
file_mime = self.binary_data_cache[data_file]["mime"]
for t in instance_times:
st = t - (after_time - before_time)
if st > 0:
time.sleep(st)
before_time = time.time()
#self.logger.info("Url " + url)
assert (self.runid)
future = session.post(url=url,
headers={'Content-Type': file_mime},
params={
'blocking': blocking_cli,
'result': self.RESULT,
'payload': {
'testid': self.runid
}
},
data=file_body,
auth=(self.user_pass[0], self.user_pass[1]),
verify=False)
futures.append(future)
after_time = time.time()
(successes, failures) = self.handle_futures(futures)
with self.tally_lock:
self.invocation_success_tally += successes
self.invocation_failure_tally += failures
self.invocation_expected_tally += len(instance_times)
@staticmethod
def write_test_metadata(metadata, destdir):
destfile = os.path.join(destdir, "test_metadata.json")
with open(destfile, 'w') as f:
f.write(json.dumps(metadata))
async def maybe_start_runtime_script(self, workload, destdir):
if workload['perf_monitoring']['runtime_script']:
runtime_script_cmdline = [
os.path.join(FAAS_ROOT,
workload['perf_monitoring']['runtime_script']),
str(int(workload['test_duration_in_seconds'])), destdir
]
runtime_script = await asyncio.create_subprocess_exec(
*runtime_script_cmdline)
self.logger.info("Invoked runtime monitoring script pid=%s" %
runtime_script.pid)
return runtime_script
return None
async def invoke_benchmark_async(self) -> InvocationMetadata:
"""
The main function.
"""
workload = self.workload
#self.logger.info("Workload Invoker started")
#print("Log file -> ../profiler_results/logs/SWI.log")
# log_path = os.path.join(
# util.ensure_directory_exists(
# os.path.join(self.test_result_dir_path, 'log')),
# 'SWI.log')
self.logger = ScriptLogger('workload_invoker', "SWI.log")
(all_events, event_count) = generic_event_generator(workload)
threads = []
for (instance, instance_times) in all_events.items():
action = workload['instances'][instance]['application']
if action == "long_run":
print("Invoking long_run")
try:
param_file = os.path.join(
FAAS_ROOT, workload['instances'][instance]['param_file'])
except:
param_file = None
try:
query_file = os.path.join(
FAAS_ROOT, workload['instances'][instance]['query_string'])
except:
query_file = None
blocking_cli = workload['blocking_cli']
if 'data_file' in workload['instances'][instance].keys():
data_file = workload['instances'][instance]['data_file']
threads.append(
threading.Thread(
target=self.binary_data_http_instance_generator,
args=[action, instance_times, blocking_cli,
data_file]))
else:
threads.append(
threading.Thread(target=self.http_instance_generator,
args=[
action, instance_times, blocking_cli,
query_file, param_file
]))
# Dump Test Metadata
test_metadata: InvocationMetadata = {
'start_time': math.ceil(time.time() * 1000),
'workload_name': self.workload["test_name"],
'test_config': self.config_json,
'event_count': event_count,
'commit_hash': self.my_commit_hash,
'runid': self.runid,
'runtime_script': False,
"failures": 0,
"successes": 0,
"expected": 0
}
runtime_script = await self.maybe_start_runtime_script(
workload, self.test_result_dir_path)
self.logger.info("Test started")
for thread in threads:
thread.start()
if runtime_script:
_, _ = await runtime_script.communicate()
if runtime_script.returncode > 0:
raise Exception("Runtime script failed")
else:
self.logger.info("Runtime script completed successfully")
test_metadata['runtime_script'] = True
for thread in threads:
thread.join()
# Save post-benchmark stats to metadata
test_metadata["failures"] = self.invocation_failure_tally
test_metadata["successes"] = self.invocation_success_tally
test_metadata["expected"] = self.invocation_expected_tally
self.write_test_metadata(test_metadata, self.test_result_dir_path)
self.logger.info("Test ended")
return test_metadata
def invoke_benchmark(self) -> InvocationMetadata:
return asyncio.run(self.invoke_benchmark_async())
async def invoke_benchmark_async(self) -> InvocationMetadata:
"""
The main function.
"""
workload = self.workload
#self.logger.info("Workload Invoker started")
#print("Log file -> ../profiler_results/logs/SWI.log")
# log_path = os.path.join(
# util.ensure_directory_exists(
# os.path.join(self.test_result_dir_path, 'log')),
# 'SWI.log')
self.logger = ScriptLogger('workload_invoker', "SWI.log")
(all_events, event_count) = generic_event_generator(workload)
threads = []
for (instance, instance_times) in all_events.items():
action = workload['instances'][instance]['application']
if action == "long_run":
print("Invoking long_run")
try:
param_file = os.path.join(
FAAS_ROOT, workload['instances'][instance]['param_file'])
except:
param_file = None
try:
query_file = os.path.join(
FAAS_ROOT, workload['instances'][instance]['query_string'])
except:
query_file = None
blocking_cli = workload['blocking_cli']
if 'data_file' in workload['instances'][instance].keys():
data_file = workload['instances'][instance]['data_file']
threads.append(
threading.Thread(
target=self.binary_data_http_instance_generator,
args=[action, instance_times, blocking_cli,
data_file]))
else:
threads.append(
threading.Thread(target=self.http_instance_generator,
args=[
action, instance_times, blocking_cli,
query_file, param_file
]))
# Dump Test Metadata
test_metadata: InvocationMetadata = {
'start_time': math.ceil(time.time() * 1000),
'workload_name': self.workload["test_name"],
'test_config': self.config_json,
'event_count': event_count,
'commit_hash': self.my_commit_hash,
'runid': self.runid,
'runtime_script': False,
"failures": 0,
"successes": 0,
"expected": 0
}
runtime_script = await self.maybe_start_runtime_script(
workload, self.test_result_dir_path)
self.logger.info("Test started")
for thread in threads:
thread.start()
if runtime_script:
_, _ = await runtime_script.communicate()
if runtime_script.returncode > 0:
raise Exception("Runtime script failed")
else:
self.logger.info("Runtime script completed successfully")
test_metadata['runtime_script'] = True
for thread in threads:
thread.join()
# Save post-benchmark stats to metadata
test_metadata["failures"] = self.invocation_failure_tally
test_metadata["successes"] = self.invocation_success_tally
test_metadata["expected"] = self.invocation_expected_tally
self.write_test_metadata(test_metadata, self.test_result_dir_path)
self.logger.info("Test ended")
return test_metadata
