#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
clear: a script that DELETES ALL workflows, functions and objects of a
MicroFunctions account
"""
import getpass
from mfn_sdk import MfnClient
c = MfnClient()
print("URL: ", c.url)
print("USER: "******"THIS CLEARS ALL FUNCTIONS, WORKFLOWS AND DATA IN YOUR ACCOUNT")
if not input("Are you sure? (y/N): ").lower().strip()[:1] == "y": sys.exit(1)
for w in c.workflows:
print("Deleting workflow", w.name)
c.delete_workflow(w)
for g in c.functions:
print("Deleting function", g.name)
c.delete_function(g)
for k in list(c.keys()):
print("Deleting object", k)
c.delete(k)
class MFNTest():
def __init__(self, test_name=None, timeout=None, workflow_filename=None, new_user=False):
self._settings = self._get_settings()
if new_user:
random_str = str(random.randint(0, 10000)) + "_" + str(time.time())
random_user = hashlib.sha256(random_str.encode()).hexdigest()
random_user = "******" + random_user[0:31] + "@knix.io"
print("User: "******"User: "******"mfn_user"])
self._client = MfnClient()
if workflow_filename is None:
self._workflow_filename = self._settings["workflow_description_file"]
else:
self._workflow_filename = workflow_filename
ind = self._workflow_filename.rfind("/")
if ind != -1:
self._workflow_folder = self._workflow_filename[:ind+1]
else:
self._workflow_folder = "./"
print("Workflow folder: " + self._workflow_folder)
self._workflow_description = self._get_json_file(self._workflow_filename)
if "name" in self._workflow_description:
self._workflow_name = self._workflow_description["name"]
else:
self._workflow_name = self._workflow_filename[0:self._workflow_filename.rfind(".")]
if test_name is not None:
self._test_name = test_name
else:
self._test_name = self._workflow_filename
if timeout is not None:
self._settings["timeout"] = timeout
self._log_clear_timestamp = int(time.time() * 1000.0 * 1000.0)
# will be the deployed workflow object in self._client
self._workflow = None
self._deployment_error = ""
self._workflow_resources = []
self.upload_workflow()
self.deploy_workflow()
def _get_json_file(self, filename):
json_data = {}
if os.path.isfile(filename):
with open(filename) as json_file:
json_data = json.load(json_file)
return json_data
def _get_settings(self):
settings = {}
# read default global settings files
settings.update(self._get_json_file("../settings.json"))
# read test specific settings
settings.update(self._get_json_file("settings.json"))
if len(settings) == 0:
raise Exception("Empty settings")
return settings
def _get_resource_info(self, resource_ref):
#dir_list = next(os.walk('.'))[1]
dir_list = next(os.walk(self._workflow_folder))[1]
is_zip = False
is_jar = False
runtime = ""
found = False
if "zips" in dir_list:
resource_filename = self._workflow_folder + "zips/" + resource_ref + ".zip"
if os.path.isfile(resource_filename):
found = True
runtime = "Python 3.6"
is_zip = True
if not found:
if "python" in dir_list:
resource_filename = self._workflow_folder + "python/" + resource_ref + ".py"
if os.path.isfile(resource_filename):
found = True
runtime = "Python 3.6"
else:
resource_filename = self._workflow_folder + resource_ref + ".py"
if os.path.isfile(resource_filename):
found = True
runtime = "Python 3.6"
if not found and "jars" in dir_list:
resource_filename = self._workflow_folder + "jars/" + resource_ref + ".jar"
if os.path.isfile(resource_filename):
found = True
runtime = "Java"
is_jar = True
if not found:
if "java" in dir_list:
resource_filename = self._workflow_folder + "java/" + resource_ref + ".java"
if os.path.isfile(resource_filename):
found = True
runtime = "Java"
else:
resource_filename = self._workflow_folder + resource_ref + ".java"
if os.path.isfile(resource_filename):
found = True
runtime = "Java"
retval = {}
retval["resource_filename"] = resource_filename
retval["resource_runtime"] = runtime
retval["is_zip"] = is_zip
retval["is_jar"] = is_jar
return retval
def _get_resource_info_map(self, workflow_description=None, resource_info_map=None):
if workflow_description is None:
workflow_description = self._workflow_description
if resource_info_map is None:
resource_info_map = {}
if "functions" in self._workflow_description:
workflow_functions = workflow_description["functions"]
for wf_function in workflow_functions:
if "name" in wf_function:
resource_name = wf_function["name"]
resource_ref = resource_name
if "resource" in wf_function:
resource_ref = wf_function["resource"]
if resource_ref not in resource_info_map.keys():
resource_info = self._get_resource_info(resource_ref)
resource_info["resource_req_filename"] = "requirements/" + resource_ref + "_requirements.txt"
resource_info["resource_env_filename"] = "environment_variables/" + resource_ref + "_environment_variables.txt"
resource_info_map[resource_ref] = resource_info
elif "States" in workflow_description:
states = workflow_description["States"]
for sname in states:
state = states[sname]
if "Resource" in state:
resource_name = state["Resource"]
if resource_name not in resource_info_map.keys():
resource_info = self._get_resource_info(resource_name)
resource_info["resource_req_filename"] = "requirements/" + resource_name + "_requirements.txt"
resource_info["resource_env_filename"] = "environment_variables/" + resource_name + "_environment_variables.txt"
resource_info_map[resource_name] = resource_info
if "Type" in state and state["Type"] == "Parallel":
branches = state['Branches']
for branch in branches:
resource_info_map = self._get_resource_info_map(branch, resource_info_map)
if "Type" in state and state["Type"] == "Map":
branch = state['Iterator']
#print(str(branch))
resource_info_map = self._get_resource_info_map(branch, resource_info_map)
#print(str(resource_info_map))
else:
print("ERROR: invalid workflow description.")
assert False
return resource_info_map
def _delete_resource_if_existing(self, existing_resources, resource_name):
for g in existing_resources:
if g.name == resource_name:
self._client.delete_function(g)
break
print("deleted resource: " + resource_name)
def _create_and_upload_resource(self, resource_name, resource_info):
print("Deploying resource: " + resource_name)
resource_filename = resource_info["resource_filename"]
is_zip = resource_info["is_zip"]
is_jar = resource_info["is_jar"]
resource_req_filename = resource_info["resource_req_filename"]
resource_env_filename = resource_info["resource_env_filename"]
resource_runtime = resource_info["resource_runtime"]
self._workflow_resources.append(resource_name)
try:
# add the resource
g = self._client.add_function(resource_name, runtime=resource_runtime)
# upload the resource source
print('Uploading file: ' + resource_filename)
if is_zip or is_jar:
g.upload(resource_filename)
else:
source_text = ''
with open(resource_filename, 'r') as f:
source_text = f.read()
g.source = {"code": source_text}
# upload the resource requirements
if os.path.isfile(resource_req_filename):
with open(resource_req_filename, "r") as f:
reqs = f.read().strip()
g.requirements = reqs
#print("set requirements for function: " + resource_name + " " + reqs)
# resource environment variables
# upload the resource environment variables
if os.path.isfile(resource_env_filename):
with open(resource_env_filename, "r") as f:
env_vars = f.read().strip()
g.environment_variables = env_vars
#print("set environment variables for function: " + resource_name + " " + env_vars)
except Exception as e:
print("ERROR: Could not create resource.")
print(str(e))
assert False
def upload_workflow(self):
self.undeploy_workflow()
resource_info_map = self._get_resource_info_map()
existing_resources = self._client.functions
for resource_name in resource_info_map.keys():
self._delete_resource_if_existing(existing_resources, resource_name)
resource_info = resource_info_map[resource_name]
self._create_and_upload_resource(resource_name, resource_info)
def get_deployment_error(self):
return self._deployment_error
def deploy_workflow(self):
try:
wf = self._client.add_workflow(self._workflow_name)
wf.json = json.dumps(self._workflow_description)
wf.deploy(self._settings["timeout"])
self._workflow = wf
if self._workflow.status != "failed":
print("MFN workflow " + self._workflow_name + " deployed.")
else:
print("MFN workflow " + self._workflow_name + " could not be deployed.")
self._deployment_error = self._workflow.get_deployment_error()
except Exception as e:
print("ERROR: Could not deploy workflow.")
raise e
assert False
def undeploy_workflow(self):
existing_workflows = self._client.workflows
for wf in existing_workflows:
if wf.name == self._workflow_name:
if wf.status == "deployed":
wf.undeploy(self._settings["timeout"])
print("Workflow undeployed.")
self._client.delete_workflow(wf)
break
existing_resources = self._client.functions
for resource_name in self._workflow_resources:
self._delete_resource_if_existing(existing_resources, resource_name)
self._client.disconnect()
def get_test_workflow_endpoints(self):
if self._workflow.status == "deployed":
return self._workflow.endpoints
def execute(self, message, timeout=None, check_duration=False):
if timeout is None:
timeout = self._settings["timeout"]
return self._workflow.execute(message, timeout, check_duration)
def get_workflow_logs(self, num_lines=500):
data = self._workflow.logs(ts_earliest=self._log_clear_timestamp, num_lines=num_lines)
return data
def clear_workflow_logs(self):
self._log_clear_timestamp = int(time.time() * 1000.0 * 1000.0)
def report(self, success, inp, expected, actual):
short_inp = self._get_printable(inp)
if success:
print(self._test_name + " test " + mfntestpassed + " with input data:", short_inp)
else:
print(self._test_name + " test " + mfntestfailed + " with input data:", short_inp + '(result: ' + json.dumps(actual) + ', expected: ' + json.dumps(expected) + ')')
def exec_only(self, inp):
any_failed_tests = False
try:
rn = self.execute(json.loads(inp))
return rn
except Exception as e:
any_failed_tests = True
self.undeploy_workflow()
print(str(e))
raise e
finally:
time.sleep(2)
if any_failed_tests:
self._print_logs(self._workflow.logs())
def exec_tests(self, testtuplelist, check_just_keys=False, check_duration=False, should_undeploy=True):
any_failed_tests = False
durations = []
time.sleep(2)
try:
for tup in testtuplelist:
current_test_passed = False
inp, res = tup
if check_duration:
rn, t_total = self.execute(json.loads(inp), check_duration=check_duration)
else:
rn = self.execute(json.loads(inp))
if check_duration:
durations.append(t_total)
#print("Total time to execute: " + str(t_total) + " (ms)")
if check_just_keys:
if set(rn.keys()) == set(res.keys()):
current_test_passed = True
else:
if rn == json.loads(res):
current_test_passed = True
self.report(current_test_passed, inp, res, rn)
any_failed_tests = any_failed_tests or (not current_test_passed)
time.sleep(1)
except Exception as e:
print(str(e))
raise e
finally:
time.sleep(2)
if check_duration:
print("------")
print("Request/response latency statistics:")
print("Number of executions: " + str(len(durations)))
print("Average (ms): " + str(statistics.mean(durations)))
print("Median (ms): " + str(statistics.median(durations)))
print("Minimum (ms): " + str(min(durations)))
print("Maximum (ms): " + str(max(durations)))
print("Stdev (ms): " + str(statistics.stdev(durations)))
print("PStdev (ms): " + str(statistics.pstdev(durations)))
percentiles = [0.0, 50.0, 90.0, 95.0, 99.0, 99.9, 99.99, 100.0]
self.print_percentiles(durations, percentiles)
print("------")
if any_failed_tests:
self._print_logs(self._workflow.logs())
if should_undeploy:
self.undeploy_workflow()
def _print_logs(self, logs):
print(logs)
for t in logs:
if t == "timestamp":
continue
cur_log = logs[t]
lines = cur_log.split("\n")
for line in lines:
print(line)
print("------")
def print_percentiles(self, data, percentiles):
data.sort()
for perc in percentiles:
print(str(perc) + "th percentile (ms): " + str(self.percentile(data, perc/100.0)))
def percentile(self, data, percent):
k = (len(data)-1) * percent
f = math.floor(k)
c = math.ceil(k)
if f == c:
return data[int(k)]
d0 = data[int(f)] * (c-k)
d1 = data[int(c)] * (k-f)
return d0 + d1
def _get_printable(self, text, max_len=50):
if len(text) > max_len:
return text[:max_len] + " ... (showing " + str(max_len) + "/" + str(len(text)) + " characters.)"
return text
def plot_latency_breakdown(self, num_last_executions=15):
eidlist = self.extract_execution_ids(num_last_executions)
eid_filename = "eidlist_" + self._test_name + ".txt"
timestamps_filename = "timestamps_" + self._test_name + ".txt"
eidlist = eidlist[len(eidlist) - num_last_executions:]
with open(eid_filename, "w") as f:
for eid in eidlist:
f.write(eid + "\n")
self.parse_metrics(eid_filename, timestamps_filename)
cmd = "python3 ../plotmfnmetrics.py " + timestamps_filename
output, error = run_command_return_output(cmd)
# cleanup
cmd = "rm esresult.json " + eid_filename + " " + timestamps_filename
_, _ = run_command_return_output(cmd)
def parse_metrics(self, eid_filename, timestamps_filename):
cmd = "python3 ../mfnmetrics.py -eidfile " + eid_filename
output, error = run_command_return_output(cmd)
log_lines = combine_output(output, error)
with open(timestamps_filename, "w") as f:
for line in log_lines:
f.write(line + "\n")
def extract_execution_ids(self, num_last_executions, num_log_lines=2000):
cmd = "python3 ../wftail.py -n " + str(num_log_lines) + " -wname " + self._workflow_name
output, error = run_command_return_output(cmd)
log_lines = combine_output(output, error)
eidlist = []
for line in log_lines:
line = line.strip()
if line == "":
continue
tokens = line.split(" ")
eid = tokens[7]
if eid != "[0l]":
eid = eid[1:-1]
eidlist.append(eid)
#print(eid)
return eidlist
def exec_keys_check(self, testtuplelist):
self.exec_tests(testtuplelist, check_just_keys=True)
# compatibility with older tests
def cleanup(self):
return
class MFNTest():
def __init__(self,
test_name=None,
timeout=None,
workflow_filename=None,
new_user=False,
delete_user=False):
self._settings = self._get_settings()
if new_user:
random_str = str(random.randint(0, 10000)) + "_" + str(time.time())
random_user = hashlib.sha256(random_str.encode()).hexdigest()
random_user = "******" + random_user[0:31] + "@knix.io"
print("User: "******"User: "******"mfn_user"])
self._client = MfnClient()
if workflow_filename is None:
self._workflow_filename = self._settings[
"workflow_description_file"]
else:
self._workflow_filename = workflow_filename
ind = self._workflow_filename.rfind("/")
if ind != -1:
self._workflow_folder = self._workflow_filename[:ind + 1]
else:
self._workflow_folder = "./"
#print("Workflow folder: " + self._workflow_folder)
self._workflow_description = self._get_json_file(
self._workflow_filename)
if "name" in self._workflow_description:
self._workflow_name = self._workflow_description["name"]
else:
self._workflow_name = self._workflow_filename[
0:self._workflow_filename.rfind(".")]
if test_name is not None:
self._test_name = test_name
else:
self._test_name = self._workflow_filename
if timeout is not None:
self._settings["timeout"] = timeout
self._log_clear_timestamp = int(time.time() * 1000.0 * 1000.0)
# will be the deployed workflow object in self._client
self._workflow = None
self._deployment_error = ""
self._workflow_resources = []
self.upload_workflow()
self.deploy_workflow()
def _get_json_file(self, filename):
json_data = {}
if os.path.isfile(filename):
with open(filename) as json_file:
json_data = json.load(json_file)
return json_data
def _get_settings(self):
settings = {}
# read default global settings files
settings.update(self._get_json_file("../settings.json"))
# read test specific settings
settings.update(self._get_json_file("settings.json"))
if len(settings) == 0:
raise Exception("Empty settings")
# Defaults
settings.setdefault("timeout", 60)
return settings
def _get_resource_info(self, resource_ref):
#dir_list = next(os.walk('.'))[1]
dir_list = next(os.walk(self._workflow_folder))[1]
is_zip = False
is_jar = False
runtime = ""
found = False
if "zips" in dir_list:
resource_filename = self._workflow_folder + "zips/" + resource_ref + ".zip"
if os.path.isfile(resource_filename):
found = True
runtime = "Python 3.6"
is_zip = True
if not found:
if "python" in dir_list:
resource_filename = self._workflow_folder + "python/" + resource_ref + ".py"
if os.path.isfile(resource_filename):
found = True
runtime = "Python 3.6"
else:
resource_filename = self._workflow_folder + resource_ref + ".py"
if os.path.isfile(resource_filename):
found = True
runtime = "Python 3.6"
if not found and "jars" in dir_list:
resource_filename = self._workflow_folder + "jars/" + resource_ref + ".jar"
if os.path.isfile(resource_filename):
found = True
runtime = "Java"
is_jar = True
if not found:
if "java" in dir_list:
resource_filename = self._workflow_folder + "java/" + resource_ref + ".java"
if os.path.isfile(resource_filename):
found = True
runtime = "Java"
else:
resource_filename = self._workflow_folder + resource_ref + ".java"
if os.path.isfile(resource_filename):
found = True
runtime = "Java"
retval = {}
retval["resource_filename"] = resource_filename
retval["resource_runtime"] = runtime
retval["is_zip"] = is_zip
retval["is_jar"] = is_jar
return retval
def _get_resource_info_map(self,
workflow_description=None,
resource_info_map=None):
if workflow_description is None:
workflow_description = self._workflow_description
if resource_info_map is None:
resource_info_map = {}
if "functions" in self._workflow_description:
workflow_functions = workflow_description["functions"]
for wf_function in workflow_functions:
if "name" in wf_function:
resource_name = wf_function["name"]
resource_ref = resource_name
if "resource" in wf_function:
resource_ref = wf_function["resource"]
if resource_ref not in resource_info_map.keys():
resource_info = self._get_resource_info(resource_ref)
resource_info[
"resource_req_filename"] = "requirements/" + resource_ref + "_requirements.txt"
resource_info[
"resource_env_filename"] = "environment_variables/" + resource_ref + "_environment_variables.txt"
resource_info_map[resource_ref] = resource_info
elif "States" in workflow_description:
states = workflow_description["States"]
for sname in states:
state = states[sname]
if "Resource" in state:
resource_name = state["Resource"]
if resource_name not in resource_info_map.keys():
resource_info = self._get_resource_info(resource_name)
resource_info[
"resource_req_filename"] = "requirements/" + resource_name + "_requirements.txt"
resource_info[
"resource_env_filename"] = "environment_variables/" + resource_name + "_environment_variables.txt"
resource_info_map[resource_name] = resource_info
if "Type" in state and state["Type"] == "Parallel":
branches = state['Branches']
for branch in branches:
resource_info_map = self._get_resource_info_map(
branch, resource_info_map)
if "Type" in state and state["Type"] == "Map":
branch = state['Iterator']
#print(str(branch))
resource_info_map = self._get_resource_info_map(
branch, resource_info_map)
#print(str(resource_info_map))
else:
print("ERROR: invalid workflow description.")
assert False
return resource_info_map
def _delete_resource_if_existing(self, existing_resources, resource_name):
for g in existing_resources:
if g.name == resource_name:
self._client.delete_function(g)
break
print("deleted resource: " + resource_name)
def _create_and_upload_resource(self, resource_name, resource_info):
print("Deploying resource: " + resource_name)
resource_filename = resource_info["resource_filename"]
is_zip = resource_info["is_zip"]
is_jar = resource_info["is_jar"]
resource_req_filename = resource_info["resource_req_filename"]
resource_env_filename = resource_info["resource_env_filename"]
resource_runtime = resource_info["resource_runtime"]
self._workflow_resources.append(resource_name)
try:
# add the resource
g = self._client.add_function(resource_name,
runtime=resource_runtime)
# upload the resource source
print('Uploading file: ' + resource_filename)
if is_zip or is_jar:
g.upload(resource_filename)
else:
source_text = ''
with open(resource_filename, 'r') as f:
source_text = f.read()
g.source = {"code": source_text}
# upload the resource requirements
if os.path.isfile(resource_req_filename):
with open(resource_req_filename, "r") as f:
reqs = f.read().strip()
g.requirements = reqs
#print("set requirements for function: " + resource_name + " " + reqs)
# resource environment variables
# upload the resource environment variables
if os.path.isfile(resource_env_filename):
with open(resource_env_filename, "r") as f:
env_vars = f.read().strip()
g.environment_variables = env_vars
#print("set environment variables for function: " + resource_name + " " + env_vars)
except Exception as e:
print("ERROR: Could not create resource.")
print(str(e))
assert False
def upload_workflow(self):
self.undeploy_workflow()
resource_info_map = self._get_resource_info_map()
existing_resources = self._client.functions
for resource_name in resource_info_map.keys():
self._delete_resource_if_existing(existing_resources,
resource_name)
resource_info = resource_info_map[resource_name]
self._create_and_upload_resource(resource_name, resource_info)
def get_deployment_error(self):
return self._deployment_error
def deploy_workflow(self):
try:
wf = self._client.add_workflow(self._workflow_name)
wf.json = json.dumps(self._workflow_description)
wf.deploy(self._settings["timeout"])
self._workflow = wf
if self._workflow.status != "failed":
print("MFN workflow " + self._workflow_name + " deployed.")
else:
print("MFN workflow " + self._workflow_name +
" could not be deployed.")
self._deployment_error = self._workflow.get_deployment_error()
except Exception as e:
print("ERROR: Could not deploy workflow.")
raise e
assert False
def undeploy_workflow(self):
existing_workflows = self._client.workflows
for wf in existing_workflows:
if wf.name == self._workflow_name:
if wf.status == "deployed":
wf.undeploy(self._settings["timeout"])
print("Workflow undeployed.")
self._client.delete_workflow(wf)
break
existing_resources = self._client.functions
for resource_name in self._workflow_resources:
self._delete_resource_if_existing(existing_resources,
resource_name)
def get_test_workflow_endpoints(self):
if self._workflow.status == "deployed":
return self._workflow.endpoints
def execute(self,
message,
timeout=None,
check_duration=False,
async=False):
if timeout is None:
timeout = self._settings["timeout"]
if async:
return self._workflow.execute_async(message, timeout)
else:
return self._workflow.execute(message, timeout, check_duration)
