class StorageActionsTest(unittest.TestCase):
def setUp(self):
self._settings = self._get_settings()
self._client = MfnClient()
def test_list_keys(self):
key_list = self._client.list_keys()
old_len = len(key_list)
ts = str(time.time() * 1000.0)
key = "my_random_key_" + ts
self._client.put(key, ts)
key_list2 = self._client.list_keys()
new_len = len(key_list2)
if (old_len+1) == new_len:
self._report("test_list_keys", True)
else:
self._report("test_list_keys", False, old_len + 1, new_len)
def test_get_put_delete(self):
ts = str(time.time() * 1000.0)
key = "my_random_key_" + ts
val = self._client.get(key)
# should be None
if val is None:
self._report("test_get_non-existing_key", True)
else:
self._report("test_get_non-existing_key", False, None, val)
self._client.put(key, ts)
val2 = self._client.get(key)
# should be ts
if val2 == ts:
self._report("test_get_existing_key", True)
else:
self._report("test_get_existing_key", False, ts, val2)
self._client.delete(key)
val3 = self._client.get(key)
# should be None
if val3 is None:
self._report("test_delete_key", True)
else:
self._report("test_delete_key", False, None, val3)
def tearDown(self):
self._client.disconnect()
# internal functions
def _get_json_file(self, filename):
json_data = {}
if os.path.isfile(filename):
with open(filename) as json_file:
json_data = json.load(json_file)
return json_data
def _get_settings(self):
settings = {}
# read default global settings files
settings.update(self._get_json_file("../settings.json"))
# read test specific settings
settings.update(self._get_json_file("settings.json"))
if len(settings) == 0:
raise Exception("Empty settings")
# Defaults
settings.setdefault("timeout", 60)
return settings
def _report(self, test_name, success, expected=None, actual=None):
if success:
print(test_name + " test " + mfntestpassed)
else:
print(test_name + " test " + mfntestfailed + '(result: ' + json.dumps(actual) + ', expected: ' + json.dumps(expected) + ')')
class MFNTest():
def __init__(self, test_name=None, timeout=None, workflow_filename=None, new_user=False):
self._settings = self._get_settings()
if new_user:
random_str = str(random.randint(0, 10000)) + "_" + str(time.time())
random_user = hashlib.sha256(random_str.encode()).hexdigest()
random_user = "******" + random_user[0:31] + "@knix.io"
print("User: "******"User: "******"mfn_user"])
self._client = MfnClient()
if workflow_filename is None:
self._workflow_filename = self._settings["workflow_description_file"]
else:
self._workflow_filename = workflow_filename
ind = self._workflow_filename.rfind("/")
if ind != -1:
self._workflow_folder = self._workflow_filename[:ind+1]
else:
self._workflow_folder = "./"
print("Workflow folder: " + self._workflow_folder)
self._workflow_description = self._get_json_file(self._workflow_filename)
if "name" in self._workflow_description:
self._workflow_name = self._workflow_description["name"]
else:
self._workflow_name = self._workflow_filename[0:self._workflow_filename.rfind(".")]
if test_name is not None:
self._test_name = test_name
else:
self._test_name = self._workflow_filename
if timeout is not None:
self._settings["timeout"] = timeout
self._log_clear_timestamp = int(time.time() * 1000.0 * 1000.0)
# will be the deployed workflow object in self._client
self._workflow = None
self._deployment_error = ""
self._workflow_resources = []
self.upload_workflow()
self.deploy_workflow()
def _get_json_file(self, filename):
json_data = {}
if os.path.isfile(filename):
with open(filename) as json_file:
json_data = json.load(json_file)
return json_data
def _get_settings(self):
settings = {}
# read default global settings files
settings.update(self._get_json_file("../settings.json"))
# read test specific settings
settings.update(self._get_json_file("settings.json"))
if len(settings) == 0:
raise Exception("Empty settings")
return settings
def _get_resource_info(self, resource_ref):
#dir_list = next(os.walk('.'))[1]
dir_list = next(os.walk(self._workflow_folder))[1]
is_zip = False
is_jar = False
runtime = ""
found = False
if "zips" in dir_list:
resource_filename = self._workflow_folder + "zips/" + resource_ref + ".zip"
if os.path.isfile(resource_filename):
found = True
runtime = "Python 3.6"
is_zip = True
if not found:
if "python" in dir_list:
resource_filename = self._workflow_folder + "python/" + resource_ref + ".py"
if os.path.isfile(resource_filename):
found = True
runtime = "Python 3.6"
else:
resource_filename = self._workflow_folder + resource_ref + ".py"
if os.path.isfile(resource_filename):
found = True
runtime = "Python 3.6"
if not found and "jars" in dir_list:
resource_filename = self._workflow_folder + "jars/" + resource_ref + ".jar"
if os.path.isfile(resource_filename):
found = True
runtime = "Java"
is_jar = True
if not found:
if "java" in dir_list:
resource_filename = self._workflow_folder + "java/" + resource_ref + ".java"
if os.path.isfile(resource_filename):
found = True
runtime = "Java"
else:
resource_filename = self._workflow_folder + resource_ref + ".java"
if os.path.isfile(resource_filename):
found = True
runtime = "Java"
retval = {}
retval["resource_filename"] = resource_filename
retval["resource_runtime"] = runtime
retval["is_zip"] = is_zip
retval["is_jar"] = is_jar
return retval
def _get_resource_info_map(self, workflow_description=None, resource_info_map=None):
if workflow_description is None:
workflow_description = self._workflow_description
if resource_info_map is None:
resource_info_map = {}
if "functions" in self._workflow_description:
workflow_functions = workflow_description["functions"]
for wf_function in workflow_functions:
if "name" in wf_function:
resource_name = wf_function["name"]
resource_ref = resource_name
if "resource" in wf_function:
resource_ref = wf_function["resource"]
if resource_ref not in resource_info_map.keys():
resource_info = self._get_resource_info(resource_ref)
resource_info["resource_req_filename"] = "requirements/" + resource_ref + "_requirements.txt"
resource_info["resource_env_filename"] = "environment_variables/" + resource_ref + "_environment_variables.txt"
resource_info_map[resource_ref] = resource_info
elif "States" in workflow_description:
states = workflow_description["States"]
for sname in states:
state = states[sname]
if "Resource" in state:
resource_name = state["Resource"]
if resource_name not in resource_info_map.keys():
resource_info = self._get_resource_info(resource_name)
resource_info["resource_req_filename"] = "requirements/" + resource_name + "_requirements.txt"
resource_info["resource_env_filename"] = "environment_variables/" + resource_name + "_environment_variables.txt"
resource_info_map[resource_name] = resource_info
if "Type" in state and state["Type"] == "Parallel":
branches = state['Branches']
for branch in branches:
resource_info_map = self._get_resource_info_map(branch, resource_info_map)
if "Type" in state and state["Type"] == "Map":
branch = state['Iterator']
#print(str(branch))
resource_info_map = self._get_resource_info_map(branch, resource_info_map)
#print(str(resource_info_map))
else:
print("ERROR: invalid workflow description.")
assert False
return resource_info_map
def _delete_resource_if_existing(self, existing_resources, resource_name):
for g in existing_resources:
if g.name == resource_name:
self._client.delete_function(g)
break
print("deleted resource: " + resource_name)
def _create_and_upload_resource(self, resource_name, resource_info):
print("Deploying resource: " + resource_name)
resource_filename = resource_info["resource_filename"]
is_zip = resource_info["is_zip"]
is_jar = resource_info["is_jar"]
resource_req_filename = resource_info["resource_req_filename"]
resource_env_filename = resource_info["resource_env_filename"]
resource_runtime = resource_info["resource_runtime"]
self._workflow_resources.append(resource_name)
try:
# add the resource
g = self._client.add_function(resource_name, runtime=resource_runtime)
# upload the resource source
print('Uploading file: ' + resource_filename)
if is_zip or is_jar:
g.upload(resource_filename)
else:
source_text = ''
with open(resource_filename, 'r') as f:
source_text = f.read()
g.source = {"code": source_text}
# upload the resource requirements
if os.path.isfile(resource_req_filename):
with open(resource_req_filename, "r") as f:
reqs = f.read().strip()
g.requirements = reqs
#print("set requirements for function: " + resource_name + " " + reqs)
# resource environment variables
# upload the resource environment variables
if os.path.isfile(resource_env_filename):
with open(resource_env_filename, "r") as f:
env_vars = f.read().strip()
g.environment_variables = env_vars
#print("set environment variables for function: " + resource_name + " " + env_vars)
except Exception as e:
print("ERROR: Could not create resource.")
print(str(e))
assert False
def upload_workflow(self):
self.undeploy_workflow()
resource_info_map = self._get_resource_info_map()
existing_resources = self._client.functions
for resource_name in resource_info_map.keys():
self._delete_resource_if_existing(existing_resources, resource_name)
resource_info = resource_info_map[resource_name]
self._create_and_upload_resource(resource_name, resource_info)
def get_deployment_error(self):
return self._deployment_error
def deploy_workflow(self):
try:
wf = self._client.add_workflow(self._workflow_name)
wf.json = json.dumps(self._workflow_description)
wf.deploy(self._settings["timeout"])
self._workflow = wf
if self._workflow.status != "failed":
print("MFN workflow " + self._workflow_name + " deployed.")
else:
print("MFN workflow " + self._workflow_name + " could not be deployed.")
self._deployment_error = self._workflow.get_deployment_error()
except Exception as e:
print("ERROR: Could not deploy workflow.")
raise e
assert False
def undeploy_workflow(self):
existing_workflows = self._client.workflows
for wf in existing_workflows:
if wf.name == self._workflow_name:
if wf.status == "deployed":
wf.undeploy(self._settings["timeout"])
print("Workflow undeployed.")
self._client.delete_workflow(wf)
break
existing_resources = self._client.functions
for resource_name in self._workflow_resources:
self._delete_resource_if_existing(existing_resources, resource_name)
self._client.disconnect()
def get_test_workflow_endpoints(self):
if self._workflow.status == "deployed":
return self._workflow.endpoints
def execute(self, message, timeout=None, check_duration=False):
if timeout is None:
timeout = self._settings["timeout"]
return self._workflow.execute(message, timeout, check_duration)
def get_workflow_logs(self, num_lines=500):
data = self._workflow.logs(ts_earliest=self._log_clear_timestamp, num_lines=num_lines)
return data
def clear_workflow_logs(self):
self._log_clear_timestamp = int(time.time() * 1000.0 * 1000.0)
def report(self, success, inp, expected, actual):
short_inp = self._get_printable(inp)
if success:
print(self._test_name + " test " + mfntestpassed + " with input data:", short_inp)
else:
print(self._test_name + " test " + mfntestfailed + " with input data:", short_inp + '(result: ' + json.dumps(actual) + ', expected: ' + json.dumps(expected) + ')')
def exec_only(self, inp):
any_failed_tests = False
try:
rn = self.execute(json.loads(inp))
return rn
except Exception as e:
any_failed_tests = True
self.undeploy_workflow()
print(str(e))
raise e
finally:
time.sleep(2)
if any_failed_tests:
self._print_logs(self._workflow.logs())
def exec_tests(self, testtuplelist, check_just_keys=False, check_duration=False, should_undeploy=True):
any_failed_tests = False
durations = []
time.sleep(2)
try:
for tup in testtuplelist:
current_test_passed = False
inp, res = tup
if check_duration:
rn, t_total = self.execute(json.loads(inp), check_duration=check_duration)
else:
rn = self.execute(json.loads(inp))
if check_duration:
durations.append(t_total)
#print("Total time to execute: " + str(t_total) + " (ms)")
if check_just_keys:
if set(rn.keys()) == set(res.keys()):
current_test_passed = True
else:
if rn == json.loads(res):
current_test_passed = True
self.report(current_test_passed, inp, res, rn)
any_failed_tests = any_failed_tests or (not current_test_passed)
time.sleep(1)
except Exception as e:
print(str(e))
raise e
finally:
time.sleep(2)
if check_duration:
print("------")
print("Request/response latency statistics:")
print("Number of executions: " + str(len(durations)))
print("Average (ms): " + str(statistics.mean(durations)))
print("Median (ms): " + str(statistics.median(durations)))
print("Minimum (ms): " + str(min(durations)))
print("Maximum (ms): " + str(max(durations)))
print("Stdev (ms): " + str(statistics.stdev(durations)))
print("PStdev (ms): " + str(statistics.pstdev(durations)))
percentiles = [0.0, 50.0, 90.0, 95.0, 99.0, 99.9, 99.99, 100.0]
self.print_percentiles(durations, percentiles)
print("------")
if any_failed_tests:
self._print_logs(self._workflow.logs())
if should_undeploy:
self.undeploy_workflow()
def _print_logs(self, logs):
print(logs)
for t in logs:
if t == "timestamp":
continue
cur_log = logs[t]
lines = cur_log.split("\n")
for line in lines:
print(line)
print("------")
def print_percentiles(self, data, percentiles):
data.sort()
for perc in percentiles:
print(str(perc) + "th percentile (ms): " + str(self.percentile(data, perc/100.0)))
def percentile(self, data, percent):
k = (len(data)-1) * percent
f = math.floor(k)
c = math.ceil(k)
if f == c:
return data[int(k)]
d0 = data[int(f)] * (c-k)
d1 = data[int(c)] * (k-f)
return d0 + d1
def _get_printable(self, text, max_len=50):
if len(text) > max_len:
return text[:max_len] + " ... (showing " + str(max_len) + "/" + str(len(text)) + " characters.)"
return text
def plot_latency_breakdown(self, num_last_executions=15):
eidlist = self.extract_execution_ids(num_last_executions)
eid_filename = "eidlist_" + self._test_name + ".txt"
timestamps_filename = "timestamps_" + self._test_name + ".txt"
eidlist = eidlist[len(eidlist) - num_last_executions:]
with open(eid_filename, "w") as f:
for eid in eidlist:
f.write(eid + "\n")
self.parse_metrics(eid_filename, timestamps_filename)
cmd = "python3 ../plotmfnmetrics.py " + timestamps_filename
output, error = run_command_return_output(cmd)
# cleanup
cmd = "rm esresult.json " + eid_filename + " " + timestamps_filename
_, _ = run_command_return_output(cmd)
def parse_metrics(self, eid_filename, timestamps_filename):
cmd = "python3 ../mfnmetrics.py -eidfile " + eid_filename
output, error = run_command_return_output(cmd)
log_lines = combine_output(output, error)
with open(timestamps_filename, "w") as f:
for line in log_lines:
f.write(line + "\n")
def extract_execution_ids(self, num_last_executions, num_log_lines=2000):
cmd = "python3 ../wftail.py -n " + str(num_log_lines) + " -wname " + self._workflow_name
output, error = run_command_return_output(cmd)
log_lines = combine_output(output, error)
eidlist = []
for line in log_lines:
line = line.strip()
if line == "":
continue
tokens = line.split(" ")
eid = tokens[7]
if eid != "[0l]":
eid = eid[1:-1]
eidlist.append(eid)
#print(eid)
return eidlist
def exec_keys_check(self, testtuplelist):
self.exec_tests(testtuplelist, check_just_keys=True)
# compatibility with older tests
def cleanup(self):
return
class StorageActionsTest(unittest.TestCase):
def setUp(self):
self._settings = self._get_settings()
self._client = MfnClient()
# kv operations
#@unittest.skip("")
def test_get_put_delete(self):
key_list = self._client.list_keys()
old_len = len(key_list)
ts = str(time.time() * 1000.0)
key = "my_random_key_" + ts
val = self._client.get(key)
# should be None
if val is None:
self._report("test_get_non-existing_key", True)
else:
self._report("test_get_non-existing_key", False, None, val)
self._client.put(key, ts)
val2 = self._client.get(key)
key_list2 = self._client.list_keys()
new_len = len(key_list2)
if (old_len + 1) == new_len:
self._report("test_list_keys", True)
else:
self._report("test_list_keys", False, old_len + 1, new_len)
# should be ts
if val2 == ts:
self._report("test_get_existing_key", True)
else:
self._report("test_get_existing_key", False, ts, val2)
self._client.delete(key)
val3 = self._client.get(key)
# should be None
if val3 is None:
self._report("test_delete_key", True)
else:
self._report("test_delete_key", False, None, val3)
# map operations
def test_map_operations(self):
map_list = self._client.list_maps()
old_len = len(map_list)
ts = str(time.time() * 1000.0)
mapname = "my_random_mapname_" + ts
rval = ts + "_" + str(random.randint(0, 1000000))
rkey = "my_random_key_" + str(random.randint(0, 1000000))
rkey2 = "my_random_key_" + str(random.randint(0, 1000000))
rkey3 = "my_random_key_" + str(random.randint(0, 1000000))
self._client.create_map(mapname)
# the creation of a map doesn't actually take place unless key-value pair is added
self._client.put_map_entry(mapname, rkey, rval)
time.sleep(3)
map_list2 = self._client.list_maps()
new_len = len(map_list2)
if (old_len + 1) == new_len:
self._report("test_create_map", True)
self._report("test_list_maps", True)
else:
self._report("test_create_map", False, old_len + 1, new_len)
self._report("test_list_maps", False, old_len + 1, new_len)
val = self._client.get_map_entry(mapname, rkey)
val_none = self._client.get_map_entry(mapname, rkey2)
if val == rval and val_none is None:
self._report("test_get_map_entry", True)
self._report("test_put_map_entry", True)
else:
self._report("test_get_map_entry", False, val, rval)
self._report("test_put_map_entry", False, val, rval)
self._client.put_map_entry(mapname, rkey2, rval)
self._client.put_map_entry(mapname, rkey3, rval)
mapentries = self._client.retrieve_map(mapname)
if all (k in mapentries.keys() for k in (rkey, rkey2, rkey3)) and\
all (v == rval for v in mapentries.values()):
self._report("test_retrieve_map", True)
else:
self._report("test_retrieve_map", False, mapentries, {
rkey: rval,
rkey2: rval,
rkey3: rval
})
mapkeys = self._client.get_map_keys(mapname)
if all(k in mapkeys for k in mapentries.keys()):
self._report("test_get_map_keys", True)
else:
self._report("test_get_map_keys", False, mapkeys,
mapentries.keys())
contains = self._client.contains_map_key(mapname, rkey)
contains2 = self._client.contains_map_key(mapname, rkey2)
self._client.delete_map_entry(mapname, rkey2)
contains3 = self._client.contains_map_key(mapname, rkey2)
if contains and contains2 and not contains3:
self._report("test_contains_map_key", True)
self._report("test_delete_map_key", True)
else:
self._report("test_contains_map_key", False, True, True)
self._report("test_delete_map_key", False, contains3, False)
self._client.clear_map(mapname)
mapkeys2 = self._client.get_map_keys(mapname)
if not mapkeys2:
self._report("test_clear_map", True)
else:
self._report("test_clear_map", False, mapkeys2, [])
self._client.delete_map(mapname)
time.sleep(3)
map_list3 = self._client.list_maps()
new_len2 = len(map_list3)
if old_len == new_len2 and new_len == new_len2 + 1:
self._report("test_delete_map", True)
else:
self._report("test_delete_map", False, new_len2, old_len)
# set operations
#@unittest.skip("")
def test_set_operations(self):
set_list = self._client.list_sets()
old_len = len(set_list)
ts = str(time.time() * 1000.0)
setname = "my_random_setname_" + ts
ts2 = str(time.time() * 1000.0)
ritem = "my_random_item_" + ts2
self._client.create_set(setname)
# the creation of a set doesn't actually take place unless an item is added
self._client.add_set_entry(setname, ritem)
time.sleep(3)
set_list2 = self._client.list_sets()
new_len = len(set_list2)
if (old_len + 1) == new_len:
self._report("test_create_set", True)
self._report("test_list_sets", True)
else:
self._report("test_create_set", False, old_len + 1, new_len)
self._report("test_list_sets", False, old_len + 1, new_len)
contains = self._client.contains_set_item(setname, ritem)
if contains:
self._report("test_add_set_entry", True)
else:
self._report("test_add_set_entry", False, None, True)
content = self._client.retrieve_set(setname)
if isinstance(content, set) and ritem in content:
self._report("test_retrieve_set", True)
else:
self._report("test_retrieve_set", False, ritem in content, True)
self._client.remove_set_entry(setname, ritem)
content2 = self._client.retrieve_set(setname)
contains2 = self._client.contains_set_item(setname, ritem)
if not contains2 and ritem not in content2:
self._report("test_remove_set_entry", True)
self._report("test_retrieve_set", True)
else:
self._report("test_remove_set_entry", False, contains2, False)
self._report("test_retrieve_set", False, ritem in content2, False)
self._client.add_set_entry(setname, "randomitem1")
self._client.add_set_entry(setname, "randomitem2")
self._client.add_set_entry(setname, "randomitem3")
self._client.add_set_entry(setname, "randomitem4")
self._client.add_set_entry(setname, "randomitem5")
content3 = self._client.retrieve_set(setname)
self._client.clear_set(setname)
content4 = self._client.retrieve_set(setname)
if len(content3) == 5 and len(content4) == 0:
self._report("test_clear_set", True)
else:
self._report("test_clear_set", False, len(content4), 0)
self._client.delete_set(setname)
time.sleep(3)
set_list3 = self._client.list_sets()
new_len2 = len(set_list3)
if old_len == new_len2 and new_len == new_len2 + 1:
self._report("test_delete_set", True)
else:
self._report("test_delete_set", False, new_len2, old_len)
# counter operations
#@unittest.skip("")
def test_create_get_increment_decrement_delete_counter(self):
counter_list = self._client.list_counters()
old_len = len(counter_list)
ts = str(time.time() * 1000.0)
countername = "my_random_countername_" + ts
rval = random.randint(0, 100)
self._client.create_counter(countername, rval)
counter_list2 = self._client.list_counters()
new_len = len(counter_list2)
if (old_len + 1) == new_len:
self._report("test_list_counters", True)
else:
self._report("test_list_counters", False, old_len + 1, new_len)
if countername not in counter_list and countername in counter_list2:
self._report("test_create_counter", True)
else:
self._report("test_create_counter", False, None, countername)
val = self._client.get_counter(countername)
if val == rval:
self._report("test_get_counter", True)
else:
self._report("test_get_counter", False, rval, val)
r2 = random.randint(0, 100)
self._client.increment_counter(countername, r2)
val2 = self._client.get_counter(countername)
if val2 == val + r2:
self._report("test_increment_counter", True)
else:
self._report("test_increment_counter", False, val + r2, val2)
r3 = random.randint(0, 100)
self._client.decrement_counter(countername, r3)
val3 = self._client.get_counter(countername)
if val3 == val2 - r3:
self._report("test_decrement_counter", True)
else:
self._report("test_decrement_counter", False, val2 - r3, val3)
self._client.delete_counter(countername)
# sleep a little to make the change to take effect
time.sleep(3)
counter_list3 = self._client.list_counters()
if countername not in counter_list3:
self._report("test_delete_counter", True)
else:
self._report("test_delete_counter", False, None, countername)
def tearDown(self):
self._client.disconnect()
# internal functions
def _get_json_file(self, filename):
json_data = {}
if os.path.isfile(filename):
with open(filename) as json_file:
json_data = json.load(json_file)
return json_data
def _get_settings(self):
settings = {}
# read default global settings files
settings.update(self._get_json_file("../settings.json"))
# read test specific settings
settings.update(self._get_json_file("settings.json"))
if len(settings) == 0:
raise Exception("Empty settings")
# Defaults
settings.setdefault("timeout", 60)
return settings
def _report(self, test_name, success, expected=None, actual=None):
if success:
print(test_name + " test " + mfntestpassed)
else:
print(test_name + " test " + mfntestfailed + '(result: ' +
json.dumps(actual) + ', expected: ' + json.dumps(expected) +
')')