def _platformix_call_test(self): env = self._env value = [0, 101, 202] expected = [value[2], value[1], value[0]] r = env.transaction("@platform_b", new_message("platformix", "call", "call_test", *value), er.all_success, more_info=True) if r["result"] is False: eprint("Failed call_test transaction") return False r = r["replies"]["platform_b"] r_data = r.reply_data["value"] compare_failed = False if not isinstance(r_data, list): compare_failed = True elif len(r_data) != len(expected): compare_failed = True else: for i in range(0, len(expected)): if r_data[i] != expected[i]: compare_failed = True break if compare_failed: eprint("Reply of call_test: {} is not that expected: {}!".format(r.reply_data["value"], expected)) return False return True
def rpyc_send(self, data): """ Sends data to app via stdin :param data: Data to send over stdin. Could be an item like str, bytearray or list/tuple of items :return: True if data were sent successfully, otherwise - False """ start_time = time.time() if (self._mock is None or data == 'exit') and self._connection is None: return proto_failure("No connection. Ensure start is complete") try: if not isinstance(data, (list, tuple)): data = data, for m in data: if self._mock is None or data == 'exit': # TODO: why data=='exit' overrides self._mock? self._connection.root.send(m) else: try: if self._mock_eval: r = evaluate(m) else: r = m except Exception as e: r = None self._mock.append(str(r)) if self._mock is None or data == 'exit': # TODO: why data=='exit' overrides self._mock? self._connection.root.flush() except Exception as e: eprint("Platform {}: exception occurred during send: {}".format( self.name, e)) exprint() return proto_failure( "Failed to send due to exception {}".format(e), -2) tprint("rpyc_send elapsed {}".format(time.time() - start_time)) return proto_success(None)
def __init__(self, cmd, res, rules, rules_kwargs=None, clean_completed=False, **kwargs): """ :param cmd: dict with 'channel' and 'interface' to specify where commands are coming from :param res: dict with 'channel' and 'interface' to specify where responses are coming from :param rules: string with path to python's class which would actually handle cmmands and responses :param rules_kwargs: dict with keyworded args for rules instantiation :param clean_completed: cleanup completed commands data to free up memory :param kwargs: kwargs to PlatformBase """ super(Scoreboard, self).__init__(**kwargs) assert isinstance(cmd, dict) and "channel" in cmd and "interface" in cmd, "cmd should be a dict with" \ "records 'channel' and 'interface'" assert isinstance(res, dict) and "channel" in res and "interface" in res, "cmd should be a dict with" \ "records 'channel' and 'interface'" self._clean_completed = clean_completed self._cmd = cmd self._res = res if rules_kwargs is None: rules_kwargs = {} # Common Metrics self.commands = 0 # Total amount of received commands self.responses = 0 # Total amount of received responses self.success = 0 # Total amount of successfully passed responses (received expected result) self.errors = [] # List with errors self.unhandled = [] # List of unhandled commands self.expected = { } # Dict with expected values without received response. Key is messaging context lcls = {} try: exec("from {} import Scoreboard as sb; cl = sb".format(rules), globals(), lcls) except ModuleNotFoundError as e: eprint("Rules module '{}' wasn't found for scoreboard {}!".format( rules, self.name)) raise e except ImportError as e: eprint( "Scoreboard unit wasn't found in rules module '{}' for scoreboard {}!" .format(rules, self.name)) raise e self._rules = lcls["cl"](host=self, **rules_kwargs) self.subscribe("#scoreboard") self.subscribe(self._cmd["channel"]) if self._cmd["channel"] != self._res["channel"]: self.subscribe(self._res["channel"])
def rpyc_log(self): if self._connection is None: return proto_failure("No connection. Ensure start is complete") try: data = self._connection.root.log except Exception as e: eprint( "Platform {}: exception occurred while getting log: {}".format( self.name, e)) exprint() return proto_failure( "Failed to get log due to exception {}".format(e), -2) return proto_success(data)
def call_test(self, *args, **kwargs): """ Checks get method of platformix protocol """ try: if not self._platformix_call_test(*args, **kwargs): return False except Exception as e: eprint("Exception occurred during test platformix::{}({},{})!".format( inspect.currentframe().f_code.co_name, args, kwargs)) exprint() raise e return True
def cmd(self, context, message): # TODO: call host's method instead if not self._accept_cmd(context, message): return True if message.method not in ("sum", "sub", "mult", "div", "power"): self._unhandled( context, message, "unsupported method! Check scoreboard against interface") return True if len(message.args) != 2: self._unhandled(context, message, "wrong format (expected 2 args)") return True if any(not isinstance(a, (int, float)) for a in message.args): self._handle(context, message, PM.failure("", -2)) return True try: if message.method == "sum": self._handle(context, message, PM.success(message.args[0] + message.args[1])) elif message.method == "sub": self._handle(context, message, PM.success(message.args[0] - message.args[1])) elif message.method == "mult": self._handle(context, message, PM.success(message.args[0] * message.args[1])) elif message.method == "div": if message.args[1] != 0: self._handle(context, message, PM.success(message.args[0] / message.args[1])) else: self._handle(context, message, PM.success(None)) elif message.method == "power": if message.args[0] == 0 and message.args[1] < 0: self._handle(context, message, PM.success(None)) else: self._handle(context, message, PM.success(message.args[0]**message.args[1])) except Exception as e: eprint("ScoreboardArithAll {}: exception occurred: {}!".format( self._host.name, e)) exprint() self._unhandled(context, message, "exception occurred: {}!".format(e)) return True
def test_arith(self, *args, **kwargs): """ Checks how arith functions works """ try: if not self._test_arith(*args, **kwargs): return False except Exception as e: eprint("Exception occurred during test arith::{}!".format( inspect.currentframe().f_code.co_name, e)) exprint() raise e return True
def smoke_test(self, *args, **kwargs): """ Shallow checks for core functions of platformix protocol, platform_base component and testenv itself """ try: if not self._platformix_smoke_test(*args, **kwargs): return False except Exception as e: eprint("Exception occurred during test platformix::{}({},{})!".format( inspect.currentframe().f_code.co_name, args, kwargs)) exprint() raise e return True
def rpyc_receive(self, count=1, timeout=1.0): """ Get's data that were sent by app via stdout :param count: Amount of lines to receive. set count to 0 to receive as much as possible, and at least one message set count to -1 to receive as much as possible, but nothing is acceptable too :param timeout: Time in seconds to wait for data :return: True if successfully received Data, otherwise - False. Data itself is contained in a reply to channel and Data is list of strings """ start_time = time.time() if not isinstance(count, int) or count < -1: raise ValueError( "Count should be an integer in a range from -1 and up to +infinity" ) if self._mock is None and self._connection is None: return proto_failure("No connection. Ensure start is complete") try: data = [] if self._mock is not None: data = self._mock[:count] self._mock = self._mock[count:] else: while len(data) < count or count == 0 or count == -1: received = self._connection.root.receive(timeout) if received is None or len(received) == 0: break if isinstance(received, (list, tuple)): data += received else: data.append(received) except Exception as e: eprint("Platform {}: exception occurred during receive: {}".format( self.name, e)) exprint() return proto_failure( "Failed to receive due to exception {}".format(e), -2) tprint("rpyc_receive elapsed {}".format(time.time() - start_time)) if 0 < count != len(data): return proto_failure("Not all requested data were received") # TODO: need a way to return partially received data elif count == 0 and len(data) == 0: return proto_failure("No data were received") else: return proto_success(data)
def check_scoreboards(self, include=None, exclude=None): try: r = self.transaction( "#scoreboard", new_message("platformix", "get", "scoreboard"), more_info=True) # TODO: add Scoreboard interface except Exception as e: eprint("Exception occurred on get scoreboard: {}".format(e)) eprint() return False if r["result"] is False: eprint("Failed to get scoreboard") return False if len(r["replies"]) == 0: eprint("No scoreboards found") return False print( "\n============================================================\n" "Scoreboards:") summary = {} errors = False for sname in r["replies"]: sdata = r["replies"][sname].reply_data["scoreboard"] print(" {}:\n{}".format(sname, pprint.pformat(sdata, indent=4))) for f in sdata: if f not in summary: summary[f] = sdata[f] else: try: summary[f] += sdata[f] except Exception as e: eprint( "Exception occurred on scoreboard summary update: {}" .format(e)) exprint() errors = True if len(r["replies"]) > 1: print("Summary:\n{}".format(pprint.pformat(summary))) print("") if errors or summary["errors"] > 0 or summary[ "unhandled"] > 0 or summary["requests"] != summary["success"]: return False return True
def _stop(self, reply_contexts): """ 1. Sends exit sequence to app via stdin 2. Receives exit log if necessary 3. Closes connection to a Caller :return: """ try: if self._mock is None and self._sock is not None: if self._close_sequence is not None: self.tcp_send(self._close_sequence) time.sleep(self._timeout) self._sock.close() except Exception as e: eprint("Platform {} experienced exception on stop: {}".format(self.name, e)) exprint() self._reply_all(reply_contexts, PM.notify("abnormal_stop: experienced exception on stop: {}".format(e))) self._sock = None return super(TcpIO, self)._stop(reply_contexts)
def __init__(self, channel, interface, rules, rules_kwargs=None, **kwargs): super(Coverage, self).__init__(**kwargs) self._channel = channel self._interface = interface if rules_kwargs is None: rules_kwargs = {} lcls = {} try: exec("from {} import Scoreboard as sb; cl = sb".format(rules), globals(), lcls) except ModuleNotFoundError as e: eprint("Rules module '{}' wasn't found for scoreboard {}!".format(rules, self.name)) raise e except ImportError as e: eprint("Scoreboard unit wasn't found in rules module '{}' for scoreboard {}!".format(rules, self.name)) raise e self._rules = lcls["cl"](host=self, **rules_kwargs) self.subscribe("#coverage") self.subscribe(self._channel)
def _platformix_call(self, context, fake_reply, method, *args, **kwargs): """ Calls host's method Call result is returned in a success message as value item :param context: messaging context :param method: method symbolic name :param args: args to method call :param kwargs: kwargs to method call :return: None """ if hasattr(self.host, method): if not callable(getattr(self.host, method)): self._reply( context, proto_failure("Attribute {} of {} is a property".format( property, self.host.name)), fake_reply) return try: result = getattr(self.host, method)(*args, **kwargs) except Exception as e: eprint( "Platformix protocol: failed to call method {} of {} with args {}, kwargs {} " "due to exception {}".format(method, self.host.name, args, kwargs, e)) exprint() self._reply( context, proto_failure( "Failed to call method {} of {} with args {}, kwargs {} " "due to exception {}".format(method, self.host.name, args, kwargs, e)), fake_reply) return self._reply(context, proto_success(result), fake_reply) else: self._reply( context, proto_failure("Method {} not found on {}".format( property, self.host.name)), fake_reply)
def waiting_reply_on(self, context, interface): """ Checks if there is handlers assigned and waiting for replies with specific context and or interface Also checks if there is timeouted waits and invokes handlers with timeouted set to True to release handler :param context: messaging context. If None specified then all contexts are checked :param interface: interface. If None specified then context for all interfaces are checked :return: """ if len(self._wait_reply_from) == 0: return False if context is not None: if context.str not in self._wait_reply_from: return False else: contexts = [context.str] else: contexts = list(self._wait_reply_from.keys()) now = time.time() r = False for c in contexts: if interface is not None and TalkContext.deserialize( c).interface != interface: continue d = self._wait_reply_from[c] if now < d["timeout"]: r = True else: if not d["send_message"]: r = d["method"](context, False, True, *d["args"], **d["kwargs"]) else: r = d["method"](context, PM.failure(state={"timeouted": True}), False, True, *d["args"], **d["kwargs"]) eprint("{}:{} didn't get reply on {}:{}".format( self.name, context.interface, context.channel, context.thread)) return r
def _platformix_set(self, context, fake_reply, prop, value): """ Set host's property to a value :param context: messaging context :param prop: property symbolic name :param value: value to set :return: None """ if hasattr(self.host, prop): if not callable(getattr(self.host, prop)): try: setattr(self.host, prop, value) except Exception as e: eprint( "Platformix protocol: failed to set attribute {} of {} to value {} " "due to exception {}".format(prop, self.host.name, value, e)) exprint() self._reply( context, proto_failure( "Failed to set attribute {} of {} to value {} " "due to exception {}".format( prop, self.host.name, value, e)), fake_reply) return self._reply(context, proto_success(getattr(self.host, prop), prop), fake_reply) else: self._reply( context, proto_failure("Attribute {} of {} is a method".format( prop, self.host.name)), fake_reply) else: self._reply( context, proto_failure("Property {} not found on {}".format( prop, self.host.name)), fake_reply)
def tcp_send(self, data): """ Sends data to app via stdin :param data: Data to send over stdin. Could be an item like str, bytearray or list/tuple of items :return: True if data were sent successfully, otherwise - False """ start_time = time.time() if self._mock is None and self._sock is None: return proto_failure("No connection. Ensure start is complete") try: if not isinstance(data, (list, tuple)): data = data, for m in data: if self._mock is None: if not isinstance(m, bytearray): if isinstance(m, str): m = m.encode('UTF-8') else: return proto_failure("Send data is expected to be a string, bytearray or " "list/tuple of strings and bytearrays") self._sock.sendall(m) else: try: if self._mock_eval and isinstance(m, str): r = evaluate(m) else: r = m except Exception as e: r = None self._mock.append(str(r)) except Exception as e: eprint("Platform {} failed to send due to exception {}".format(self.name, e)) exprint() return proto_failure("Failed to send due to exception {}".format(e), -2) tprint("tcp_send elapsed {}".format(time.time() - start_time)) return proto_success(None)
def _start(self, reply_contexts): """ 1. Update's host from parent platform if necessary 2. Connects to a Caller :return: True if started successfully, otherwise - False """ if self._port is None: eprint("Platform {} failed to start - port should be specified".format(self.name)) return proto_failure("Platform {} failed to start - port or service should be specified") if self._host is None and self.parent is not None: c = self.request(new_message("platformix", "get", "host"), self._generic_request_handler, [], {"on_success": lambda d: setattr(self, "_host", d["host"]), "on_failure": lambda d: eprint("Failed to get host due to {}:{}".format( d["errcode"], d["state"])) }) if self._host is None: eprint("Platform {} failed to start - can't get host".format(self.name)) return proto_failure("Failed to start - can't get host") try: timeout = time.time() + self._connect_timeout while True: try: if self._mock is None: self._sock = sock = socket.socket() sock.connect((self._host, self._port)) sock.settimeout(self._timeout) break except ConnectionRefusedError as e: if time.time() > timeout: raise e except Exception as e: self._sock = None eprint("Platform {} failed to start due to exception {}".format(self.name, e)) exprint() return proto_failure("Failed to start due to exception {}", -2) return super(TcpIO, self)._start(reply_contexts)
def load_description(self, description, generics, verbose, extrapolate=True): # TODO: include could be at any level # TODO: include in specials should contain name self._sch_data = None if os.path.isfile(description): if verbose: vprint("openning {}".format(description)) with open(description, 'r') as stream: try: desc = yaml.safe_load(stream) if verbose: vprint("loaded data: {}".format(desc)) except yaml.YAMLError as exc: eprint(exc) desc = {} elif isinstance(description, str): description = re.sub(r"\\n", "\n", description) # TODO: unescape slashes desc = yaml.safe_load(description) else: desc = copy.deepcopy(description) assert isinstance( desc, dict), "Expecting a dict as scheme description! Got {}".format( type(desc)) if self._root_section is not None: assert self._root_section in desc, "Root section '{}' wasn't found in description".format( self._root_section) assert isinstance(desc[self._root_section], (dict, list, tuple)), "Root section should be a dict, list or tuple! " \ "Got {}".format(type(desc[self._root_section])) desc = desc[self._root_section] if isinstance(desc, (list, tuple)): assert len( desc ) == 1, "descriptions with more than 1 root section isn't supported yet!" desc = desc[0] # if "__metadata__" in desc: # raise ValueError("'__metadata__' section shouldn't be in testenv description") # if "__nodes__" in desc: # raise ValueError("'__nodes__' section shouldn't be in testenv description") # desc["__metadata__"] = {'name': None, 'description': None} # desc["__nodes__"] = [] if "name" not in desc: raise ValueError("No scheme name found in description!") self._name = desc["name"] for s in self.specials: # S is for SECTION if s in desc: assert isinstance( desc[s], dict ), "Specials like {} should be a dict only! Got: {} for {}".format( " ".join(self.specials), type(desc[s]), s) self._data[s] = copy.deepcopy(desc[s]) else: self._data[s] = {} if generics is not None: for g in generics: if g not in self._data["generics"]: raise ValueError( "Generic {} is not in environment".format(g)) else: if generics[g] in ("True", "False"): self._data["generics"][g] = generics[g] == "True" else: try: self._data["generics"][g] = int(generics[g]) except Exception as e: try: self._data["generics"][g] = float(generics[g]) except Exception as e: self._data["generics"][g] = generics[g] self._sch_data = desc errors = [] _sch_structure = self._sch_to_structured(errors, "", self._sch_data, 'root', None, None) if len(errors) > 0: message = "Errors occurred during description parsing:\n" + "\n".join( errors) raise ValueError(message) assert len(_sch_structure ) == 1, "Unexpectedly found multiple roots for scheme" self._sch_structure = _sch_structure[0] if extrapolate: self.extrapolate_description()
def tcp_receive(self, count=0, timeout=None, decode='UTF-8'): """ Get's data that were sent by app via stdout :param count: Amount of bytes to receive. set count to 0 to receive as much as possible, at least something set count to -1 to receive as much as possible, but nothing is acceptable too :param timeout: Time in seconds to wait for data. If None then TCP socket timeout is used :param deoode: If not None then received data is decoded into string using specified decoder. Default: 'UTF-8' :return: True if successfully received Data, otherwise - False. Data itself is contained in a reply to channel and Data is list of strings """ start_time = time.time() if not isinstance(count, int) or count < -1: raise ValueError("Count should be an integer in a range from -1 and up to +infinity") if self._mock is None and self._sock is None: return proto_failure("No connection. Ensure start is complete") try: if self._mock is not None: data = self._mock.pop(0) else: data = bytearray() if count < 1: recv_size = 1024 else: recv_size = count if timeout is not None: timeout = time.time() + timeout while len(data) < count or count == 0 or count == -1: try: received = self._sock.recv(recv_size) except TimeoutError: received = None except socket.timeout: received = None if received is None: if count < 0: break if count == 0 and len(data) > 0: break else: data += received if timeout is not None and time.time() > timeout: break if count > 0: recv_size = count - len(data) if decode is not None: data = data.decode(decode) except Exception as e: eprint("Platform {} failed to receive due to exception {}".format(self.name, e)) exprint() return proto_failure("Failed to receive due to exception {}".format(e), -2) tprint("rpyc_receive elapsed {}".format(time.time() - start_time)) if 0 < count != len(data): return proto_failure("Not all requested data were received") # TODO: need a way to return partially received data elif count == 0 and len(data) == 0: return proto_failure("No data were received") else: return proto_success(data)
def _stop(self, reply_contexts): """ 1. Sends exit sequence to app via stdin 2. Receives exit log if necessary 3. Closes connection to a Caller :return: """ stop_failed = None try: if self._connection is not None: # Check before proceeding as it can be emergency stop if self._exit_sequence is not None and len( self._exit_sequence) > 0: self.rpyc_send(self._exit_sequence) exit_time = time.time() + self._stop_timeout forced_stop = False else: self._connection.root.stop( "Stopped by intent of SoftwareRunner (_stop)", force=True) exit_time = time.time() + self._stop_timeout forced_stop = True while self._connection.root.running and ( not forced_stop or time.time() < exit_time): if not self._connection.root.running: break if time.time() >= exit_time and (not forced_stop): self._connection.root.stop( "Forced to stop by intent of SoftwareRunner (_stop)", force=True) exit_time = time.time() + self._stop_timeout forced_stop = True stop_failed = "Not stopped by stop sequence" if self._connection.root.running: if stop_failed is not None: stop_failed += ", " else: stop_failed = "" stop_failed += "App not stopped" # TODO: check whether isntance is running and call stop explicitly if self._display_log_on_stop: data = list(self._connection.root.log) vprint("Platform {}. Exit log: {}".format( self.name, '\n'.join(data))) self._connection.close() except Exception as e: eprint("Platform {} experienced exception on stop: {}".format( self.name, e)) exprint() self._reply_all( reply_contexts, PM.notify( "abnormal_stop: experienced exception on stop: {}".format( e))) if stop_failed is not None: stop_failed += ", " else: stop_failed = "" stop_failed += "Experienced exception: {}".format(e) if stop_failed is not None: eprint("Software runner {} failed to stop app properly: {}".format( self.name, stop_failed)) self._reply_all( reply_contexts, PM.notify("Software runner {} failed to stop app properly: {}". format(self.name, stop_failed))) self._connection = None super_result = super(SoftwareRunner, self)._stop(reply_contexts) if stop_failed: return proto_failure(stop_failed) else: return super_result
def _platformix_smoke_test(self, tests_list=None): env = self._env # Accepts following values in tests_list: # "success, fake, report" # "no fail check" # TODO: check if there is required amount of responses # Start # NOTE: startup is made by testenv itself now # assert env.transaction("#platforms", new_message("platformix", "start")) is True, "Failed to start platforms" # Test errors control if tests_list is not None: tests_list = list(tests_list) if tests_list is None or "[sanity check]" in tests_list: if tests_list is not None: del tests_list[tests_list.index("[sanity check]")] if env.transaction( "@platform_b", fake_op_message("platformix", proto_failure("Failed by intent (fake_next_op, " "Single unit error in [all success] condition)"), on_channel="#platforms", on_message=new_message("platformix", "report", "is_running") ), er.all_success + er.none_fail) is False: eprint("Failed to set fake_next_op on [sanity check]") return False # NOTE: both all and none in condition above are for testing purpose # No fail since it's another channel if env.transaction("@platform_b", new_message("platformix", "report", "is_running"), er.none_fail) is False: # NOTE: none in condition above is testing purpose eprint("Failed to pass [report is_running] or avoid faking on [sanity check]") return False # No fail since it's another command if env.transaction("#platforms", new_message("platformix", "report", "running")) is False: eprint("Failed to pass [report running] or avoid faking on [sanity check]") return False # Fail as we told platform_b to do so if env.transaction("#platforms", new_message("platformix", "report", "is_running")) is True: eprint("Failed to pass faking or [all success] fail check on [sanity check]") return False if tests_list is None or "[single fail] success check" in tests_list: if tests_list is not None: del tests_list[tests_list.index("[single fail] success check")] if env.transaction( "@platform_b", fake_op_message("platformix", proto_failure("Failed by intent (fake_next_op, Single unit error " "in [single fail] condition)"), on_channel="#platforms", execute=True )) is False: eprint("Failed to set fake_next_op on [single fail] success check") return False # No fail as we expecting that platform_b about to fail (NOTE: - no message filtering here) if env.transaction("#platforms", new_message("platformix", "report", "running"), er.fail("platform_b") + er.others_success) is False: eprint("Failed to pass [single fail] success check") return False if tests_list is None or "[any success] fail check" in tests_list: if tests_list is not None: del tests_list[tests_list.index("[any success] fail check")] if env.transaction( "#platforms", fake_op_message("platformix", proto_failure("Failed by intent (fake_next_op, All fail " "on [any success] condition)"), on_channel="#platforms" )) is False: eprint("Failed to set fake_next_op on [any success] fail check") return False # Should fail if env.transaction("#platforms", new_message("platformix", "report", "running"), er.any_success) is True: eprint("Failed to pass [any success] fail check") return False if tests_list is None or "[not(any success)] fail check" in tests_list: if tests_list is not None: del tests_list[tests_list.index("[not(any success)] fail check")] # Should pass as condition is negated if env.transaction("#platforms", new_message("platformix", "report", "running"), [("any", "success", True)]) is True: eprint("Failed to pass [not(any success)] fail check") return False if tests_list is None or "[any fail] success check" in tests_list: if tests_list is not None: del tests_list[tests_list.index("[any fail] success check")] if env.transaction( "@platform_b", fake_op_message("platformix", proto_failure("Failed by intent (fake_next_op, Should pass " "as condition is negated)"), on_channel="#platforms" )) is False: eprint("Failed to set fake_next_op on [any fail] success check") return False # Should pass as one item failed, as expected if env.transaction("#platforms", new_message("platformix", "report", "running"), er.any_fail) is False: eprint("Failed to pass [any fail] success check") return False # Check request without response (should fail) if tests_list is None or "[no response] fail check" in tests_list: if tests_list is not None: del tests_list[tests_list.index("[no response] fail check")] if env.transaction("__void__", new_message(None, "platformix", "report", "running"), er.all_fail) is True: eprint("Failed to pass [no response] fail check") return False # Stop # NOTE: stop is made by testenv itself now # assert env.transaction("#platforms", new_message("platformix", "stop")) is True, "Failed to stop platforms" if tests_list is not None and len(tests_list) > 0: eprint("There is unknown tests left in a list: {}".format(tests_list)) return False return True
def exception_test(self, *args, **kwargs): """ Checks testenv reaction to exception during test """ eprint("Exception is provoked as a part of test. Further testing should break and platforms shall be stopped") assert False, "Assertion error occurred AS EXPECTED. Ensure platforms are stopped correctly"
def run_tests(env, tests, start_stop=True, include=None, exclude=None, dry_run=False): """ A General Routine to run all Tests Returns dict with test name and it's results. True if test passed, False if test failed, None if test wasn't executed at all If exception is occurred on test then it's stored as a test's result :param env: Test environment in which test would run :param tests: List with tests descriptions to run. Each item exected to be a dict :param start_stop: If start_stop is True (default) then all platforms are stared before executing test and the all platforms are stopped after test has been executed :param include: List of test's names to be executed. Other test would be ignored. Set to None to include all :param exclude: List of test's names to be excluded from execution. Set to None to avoid exclusion. Exclude list takes priority over include :param dry_run: Don't run test actually. Just that tests would run :return: a dict with test name as key and test result (see run_test description) as value if dry_run specified then test result would be None """ result = {} if include is not None: includes_not_found = [] tests_names = [t['name'] for t in tests] for test in include: if test not in tests_names and test != "_scoreboards_": includes_not_found.append(test) if len(includes_not_found) != 0: raise ValueError("Tests {} were not found in:\n {}".format( ', '.join(includes_not_found), ' \n '.join([t["name"] for t in tests]))) # raise ValueError("Tests {} were not found".format(', '.join(includes_not_found))) for test in tests: k = test["name"] if exclude is not None and k in exclude: continue if include is not None and k not in include: continue result[k] = None for test in tests: k = test["name"] if exclude is not None and k in exclude: continue if include is not None and k not in include: continue try: if not dry_run: result[k] = TestRunner.run_test(env, test, start_stop, start_stop) else: result[k] = None except Exception as e: # TODO: make sure transaction ended (force it to end if necessary) result[k] = {"status": e, "elapsed": 0} eprint("Exception occurred on test '{}': {}".format(k, e)) exprint() if start_stop: # Make sure platforms are stopped try: env.stop_platforms() except Exception as e: # If it's impossible to stop platforms - break check result["stop_platforms"] = {"status": e, "elapsed": 0} eprint( "Unrecoverable exception occurred during test calc::{}!\n" "Aborting run_tests routine!\n" "Exception: {}".format(k, e)) exprint() env.emergency_stop() break return result
def check_responses(verbose=False): if len(conv_analyzer.in_progress) > 0: eprint( "Some platforms ({}) are not completed transaction {}::{}!" .format(conv_analyzer.in_progress, channel, message.serialize())) return False elif len(conv_analyzer.participants) == 0: eprint("No one acknowledged transaction {}::{}!".format( channel, message.serialize())) return False elif not isinstance(expected, (list, tuple)): raise ValueError( "'expected' argument should be a list or tuple of lists or tuples!" ) elif len(expected) == 0: eprint( "Warning! Transaction {}::{} is without expected result! Completed but no checks made" .format(channel, message.serialize())) return True else: participants = conv_analyzer.participants others = conv_analyzer.participants total = len(others) result = [False] * len(expected) idx = 0 for e in expected: if not isinstance(e, (list, tuple)) or len(e) < 2: raise ValueError( "'expected' items should be list or tuple with length 2 or more" ) negative = False if len(e) >= 3: negative = e[2] matched = sorted( conv_analyzer.partipiciants_by_result(e[1], negative)) if e[0] == "all": others = [] if len(matched) == total: result[idx] = True else: unmatched = [ p for p in participants if p not in matched ] # participants - matched eprint( "Platforms {} are failed to check against '{}{}' for 'all' " "on transaction {}::{}".format( unmatched, ["", "not "][negative], e[1], channel, message.serialize())) for p in unmatched: eprint(" {}: {}".format( p, conv_analyzer.replies[p].serialize())) elif e[0] == "any": others = [] if len(matched) > 0: result[idx] = True else: unmatched = participants eprint( "None of platforms {} succeeded check against '{}{}' for 'any' on transaction " "{}::{}".format(unmatched, ["", "not "][negative], e[1], channel, message.serialize())) for p in unmatched: eprint(" {}: {}".format( p, conv_analyzer.replies[p].serialize())) elif e[0] == "others": if len(others) == 0: eprint( "No other platforms left to check against '{}{}' on transaction {}::{}" .format(["", "not "][negative], e[1], channel, message.serialize())) result[idx] = True else: unmatched = [ p for p in others if p not in matched ] # others - matched others = [] if len(unmatched) > 0: eprint( "Platforms {} are failed to check against '{}{}' for 'others' on transaction" " {}::{}".format(unmatched, ["", "not "][negative], e[1], channel, message.serialize())) for p in unmatched: eprint(" {}: {}".format( p, conv_analyzer.replies[p].serialize())) else: result[idx] = True elif e[0] == "none": if len(matched) == 0: result[idx] = True else: unmatched = matched eprint( "Platforms {} are failed to check against '{}{}' for 'none' " "on transaction {}::{}".format( unmatched, ["", "not "][negative], e[1], channel, message.serialize())) for p in unmatched: eprint(" {}: {}".format( p, conv_analyzer.replies[p].serialize())) else: if not isinstance(e[0], (list, tuple)): el = [e[0]] else: el = e[0] others = [p for p in others if p not in el] # others - el unmatched = [p for p in el if p not in matched] # el - matched if len(unmatched) == 0: result[idx] = True else: eprint( "Platforms {} are failed to check against '{}{}' on transaction {}::{}" .format(unmatched, ["", "not "][negative], e[1], channel, message.serialize())) for p in unmatched: eprint(" {}: {}".format( p, conv_analyzer.replies[p].serialize())) idx += 1 result = all(r is True for r in result) if result: if verbose: vprint( "Success! Transaction {}::{} completed successfully". format(channel, message.serialize())) else: if verbose: eprint("Error! Transaction {}::{} failed on checks".format( channel, message.serialize())) return result
def _platformix_getset_test(self): env = self._env value = time.time() if env.transaction("@platform_b", new_message("platformix", "set", "tag", value), er.all_success) is False: eprint("Failed to set property") return False r = env.transaction("@platform_b", new_message("platformix", "get", "tag"), er.all_success, more_info=True) if r["result"] is False: eprint("Failed to get property") return False r = r["replies"]["platform_b"] if r.reply_data["tag"] != value: eprint("Value of property: {} is not that expected: {}!".format(r.reply_data["tag"], value)) return False if env.transaction("@platform_b", new_message("platformix", "set", "__shouldnt_exists__", value), er.all_fail) is False: eprint("Failed check of setting wrong property (expected to fail, but succeed)") return False return True # TODO: enable code below # NOTE: code below is unreachable yet since exception during transaction now breaks whole process try: if env.transaction("@platform_b", new_message("platformix", "set", "__shouldnt_exists__"), er.all_fail) is False: eprint("Failed check set without necessary arguments (expected exception, but succeed)") return False eprint("Failed check set without necessary arguments (expected exception, but failed)") return False except TypeError as e: pass try: if env.transaction("@platform_b", new_message("platformix", "get"), er.all_fail) is False: eprint("Failed check get without necessary arguments (expected exception, but succeed)") return False eprint("Failed check get without necessary arguments (expected exception, but failed)") return False except TypeError as e: pass return True
def _start(self, reply_contexts): """ 1. Update's host from parent platform if necessary 2. Connects to a Caller :return: True if started successfully, otherwise - False """ if self._port is None and self._service is None: eprint( "Platform {} failed to start - port or service should be specified" .format(self.name)) return proto_failure( "Platform {} failed to start - port or service should be specified" ) if self._port is not None and self._service is not None: eprint( "Platform {} failed to start - specify only port or service, but not both" .format(self.name)) return proto_failure( "Platform {} failed to start - port or service should be specified" ) if self._host is None and self.parent is not None: c = self.request( new_message("platformix", "get", "host"), self._generic_request_handler, [], { "on_success": lambda d: setattr(self, "_host", d["host"]), "on_failure": lambda d: eprint("Failed to get host due to {}:{}".format( d["errcode"], d["state"])) }) if self._host is None: eprint("Platform {} failed to start - can't get host".format( self.name)) return proto_failure("Failed to start - can't get host") try: self._connection = rpyc.connect(self._host, self._port) # TODO: use service instead of port if it's specified if self._args is not None: self._connection.root.set_args(copy.copy(self._args)) result = self._connection.root.run() except Exception as e: self._connection = None eprint("Platform {} failed to start due to exception {}".format( self.name, e)) exprint() return proto_failure("Failed to start due to exception {}", -2) if result is False: self._connection = None eprint("Platform {} failed to start: rpyc server returned False". format(self.name)) return proto_failure("RPYC server returned False") if result and not self._mock: if self._start_sequence is not None and len( self._start_sequence) > 0: self.rpyc_send(self._start_sequence) return super(SoftwareRunner, self)._start(reply_contexts)
if ve: simple_logging.eprint_worker = my_vprint simple_logging.tprint_worker = simple_logging.vprint_worker env = TestEnv(description=description, generics=generics, verbose=verbose) env.instantiate() farm_data = env.farm.expose_data() # NOTE: don't ever think about changing somethig in farm_data as it would break whole thing if verbose: vprint("Platforms: {}\nChannels: {}\nWaiting: {}".format( pprint.pformat(farm_data.platforms), pprint.pformat(farm_data.channels), pprint.pformat(farm_data.awaiting))) if len(farm_data.awaiting) > 0: eprint("Some items are still awaiting for creation!") for i in farm_data.awaiting: if farm_data.awaiting[i]["parent"] is not None \ and farm_data.awaiting[i]["parent"] not in farm_data.platforms \ and farm_data.awaiting[i]["parent"] not in farm_data.awaiting: eprint("\tNo parent with name {} is found for {}".format( farm_data.awaiting[i]["parent"], i)) else: eprint("{} is waiting someone: {}!".format( i, farm_data.awaiting[i]['wait'])) exit(-1) # Build device tree: if verbose: vprint("Device tree:") env.print_device_tree()
def _extrapolate_string(self, name, path, value, expr, types): if not isinstance(value[0], str): return False data = value[0] if not expr: if not types: m = re.findall( r"\$(s?){(\w+\.?\w+(?:(?:\[\d+\])|(?:\['.+?'\])|(?:\[\".+?\"\]))*)}", data) # TODO: avoid escaped chars else: m = re.findall(r"^\$([bfi]){(.+)}$", data) else: m = re.findall(r"\$(es?){(.+?)}", data) # TODO: avoid escaped chars if m is None or len(m) == 0: return False values = {} straight = False straight_value = None for t, k in list(m): if k in values: continue # NOTE: it's enough to extrapolate one time try: if not expr: if not types: tmp = self._get_extrapolated_value(k) # If specified to be string or it's part of value - convert value to string if t == 's' or len(r"$" + t + "{" + k + "}") != len(data) or name in ( "alias", ): values[k] = str(tmp) else: straight = True straight_value = tmp else: straight = True if len(k > 2) and (k[0] == "'" and k[-1] == "'" or k[0] == '"' and k[-1] == '"'): vv = k[1:-1] else: vv = k if t == 'b': if vv in ('True', 'False'): straight_value = vv == 'True' else: straight_value = bool(int(vv)) elif t == 'f': straight_value = float(vv) else: straight_value = int(vv) else: tmp = evaluate(k) if t == 'es' or len(r"$" + t + "{" + k + "}") != len(data) or name in ( "alias", ): values[k] = str(tmp) else: straight = True straight_value = tmp except Exception as e: eprint( "Exception occurred while extrapolating {} with value {}." "\n Failed on {}, exception: {}".format(path, data, k, e)) exprint() raise e if straight: value[0] = straight_value else: for t, k in list(m): kv = r"\$" + t + "{" + re.escape(k) + "}" vprint("Extrapolated {}: {}->{}".format( value[0], kv, values[k])) value[0] = re.sub(kv, values[k], value[0]) return True