def run_test(env, test, start, stop): """ A General Routine to run single Tests Returns dict with test status and time elapsed :param env: Test environment in which test would run :param test: A Test to run. Should be a dict with following structure "name" - Test's name "f" - A function to call "args" - Args to a function, list expected "kwargs" - Keywords args to a function, dict expected :param start: Boolean. Set to True to start all platforms before conducting a Test :param stop: Boolean. Set to True to stop all platforms after test has been conducted :return: a dict with fields "status" and "elapsed" """ vprint( "\n\n===================================\nStarting test {}...\n". format(test["name"])) elapsed = time.time() if start: env.start_platforms() stat = test["f"](*test["args"], **test["kwargs"]) if stop: env.stop_platforms() elapsed = time.time() - elapsed return {"status": stat, "elapsed": elapsed}
def _stop(self, reply_contexts): """ Stopping worker method. Does all necessary actions to stop platform after all stop conditions were met Derrived classes should call this in the end when overriding _stop method :return: True if successfully stopped - subject were stopped (whatever that means) and disconnected, otherwise - False """ self._running = False self._stopping = False vprint("platform {} has been stopped".format(self.name)) return proto_success(None)
def _start(self, reply_contexts): """ Startup worker method. Does all necessary actions to start platform instance after all start conditions were met Derrived classes should call this in the end when overriding _start method :return: True if successfully started (binded to subject and subject is in operational state), otherwise - False """ self._running = True self._start_in_progress = False self._starting = False vprint("platform {} just started".format(self.name)) return proto_success(None)
def _extrapolate_value(self, name, path, value, generics_only=False): if generics_only and name != 'generics': return value if path in self._extrapolated_values: return self._extrapolated_values[path] assert self._extrapolation_counter < 1000, "Looks like it's dead loop on values extrapolation of. " \ "Last item: {}. " \ "Extrapolated data so far: \n{}" \ "Extrapolation chain at this moment: \n {}" \ "".format(path, pprint.pformat(self._extrapolated_values), '\n '.join(self._extrapolation_chain)) assert path not in self._extrapolation_chain, "Looks like there is dependency loop in values extrapolation: " \ "Extrapolated data so far: \n{}" \ "Extrapolation chain: {}" \ "".format(pprint.pformat(self._extrapolated_values), '\n '.join(self._extrapolation_chain)) self._extrapolation_counter += 1 self._extrapolation_chain.append(path) expr = False count = 0 result = [value] while True: if self._extrapolate_string(name, path, result, expr=expr, types=False): expr = False elif expr is False: expr = True else: self._extrapolate_string(name, path, result, expr=False, types=True) break count += 1 assert count < 100, "Look's like it's extrapolation dead loop on extrapolation of {}".format( path) self._extrapolation_counter -= 1 self._extrapolation_chain.pop(-1) if value != result[0]: vprint("Extrapolated {}: from '{}' to '{}'".format( path, value, result[0])) self._extrapolated_values[path] = result[0] return result[0]
def call_test(self, *args): """ Method for testing purposes. Does nothing but prints args and returns them back If no args then exception is raised :param args: :return: args as list """ if len(args) == 0: raise ValueError("At least one arg expected") vprint(*args) r = [] for i in range(len(args) - 1, -1, -1): r.append(args[i]) return r
def _get_extrapolated_value(self, path): """ Extrapolates string value Replaces \w+ with corresponding generic value Replaces (\w+\).(\w+) with argument value (2) of specified platform (1) platform generics corresponds to generics also :param value: string to extrapolate :return: extrapolated string """ if path in self._extrapolated_values: return self._extrapolated_values[path] m = re.findall(r"^(\w+)\.?(\w+)?", path) if m is None or len(m) == 0: raise ValueError("Bad format for extrapolated value path") generic = False if m[0][1] == "": generic = True p = "generics" k = m[0][0] else: p = m[0][0] k = m[0][1] if generic: path = "generics." + path if path in self._extrapolated_values: return self._extrapolated_values[path] count = 0 while p in self._data["alias"]: if p in self.specials: break vprint("aliasing {} with {}".format(p, self._data["alias"])) p = self._get_extrapolated_value("alias." + p) count += 1 assert count < 100, "Look's like it's aliasing dead loop" if p not in self._data: raise ValueError("Not found {} within platforms".format(p)) if k not in self._data[p]: raise ValueError( "Not found {} within platform's {} generics".format(k, p)) self._extrapolate_data(p, p + "." + k, self._data[p][k], "[{}]") assert path in self._extrapolated_values, "No extrapolated data for {}".format( path) return self._extrapolated_values[path]
def _register_reply_handler(self, context, method, args, kwargs, timeout, send_message=True, force=False, store_state=True): assert force or context.str not in self._wait_reply_from, "Reply handler for {} of {} " \ "is already registered ({}({},{})!".format(context.str, self.name, *self._wait_reply_from[context.str]) if timeout is not None: timeout += time.time() self._wait_reply_from[context.str] = { "method": method, "args": args, "kwargs": kwargs, "send_message": send_message, "timeout": timeout, "store_state": store_state } vprint("{} is waiting for reply on {}:{}".format( self.name, context.channel, context.thread))
def instantiate(self): vprint( "Test environment at instances instantination start:\n{}".format( pprint.pformat(self._data))) # Other entries are treated as platform's instances def add_instance(instance_generics, base_platform=None): args = copy.deepcopy(instance_generics) if base_platform is not None: args["base_platform"] = base_platform assert "base_platform" in args, "Not found base_platform for instance with generics {}".format( args) pi = PlatformFactory(farm=self.farm, **args) # PI is for PLATFORM INSTANCE for p in self._data: # P is for PLATFORM if p in self._scheme.specials: continue if 'condition' not in self._data[p] or self._data[p][ 'condition'] == True: ig = copy.deepcopy(self._data[p]) if 'condition' in ig: del ig['condition'] add_instance(ig)
def _print_device_tree(self, node, level): vprint("{}{}({}):".format("\t" * level, node.name, node.base_platform)) level += 1 farm_data = self.farm.expose_data() depending_nodes = [ p.name for p in farm_data.platforms.values() if node in p.wait and node != p.parent ] if node.wait is not None: if node.parent is None: waits_for = node.wait else: waits_for = [w for w in node.wait if w != node.parent.name] if len(waits_for) > 0: vprint("{}Depends on {}:".format("\t" * level, waits_for)) if len(depending_nodes) > 0: vprint("{}Depending nodes {}:".format("\t" * level, depending_nodes)) for s in node.subplatforms: self._print_device_tree(s, level)
vf.close() vf = None simple_logging.vprint_worker = my_vprint_no_verbose else: simple_logging.vprint_worker = my_vprint if ve: simple_logging.eprint_worker = my_vprint simple_logging.tprint_worker = simple_logging.vprint_worker env = TestEnv(description=description, generics=generics, verbose=verbose) env.instantiate() farm_data = env.farm.expose_data() # NOTE: don't ever think about changing somethig in farm_data as it would break whole thing if verbose: vprint("Platforms: {}\nChannels: {}\nWaiting: {}".format( pprint.pformat(farm_data.platforms), pprint.pformat(farm_data.channels), pprint.pformat(farm_data.awaiting))) if len(farm_data.awaiting) > 0: eprint("Some items are still awaiting for creation!") for i in farm_data.awaiting: if farm_data.awaiting[i]["parent"] is not None \ and farm_data.awaiting[i]["parent"] not in farm_data.platforms \ and farm_data.awaiting[i]["parent"] not in farm_data.awaiting: eprint("\tNo parent with name {} is found for {}".format( farm_data.awaiting[i]["parent"], i)) else: eprint("{} is waiting someone: {}!".format( i, farm_data.awaiting[i]['wait'])) exit(-1)
def _stop(self, reply_contexts): """ 1. Sends exit sequence to app via stdin 2. Receives exit log if necessary 3. Closes connection to a Caller :return: """ stop_failed = None try: if self._connection is not None: # Check before proceeding as it can be emergency stop if self._exit_sequence is not None and len( self._exit_sequence) > 0: self.rpyc_send(self._exit_sequence) exit_time = time.time() + self._stop_timeout forced_stop = False else: self._connection.root.stop( "Stopped by intent of SoftwareRunner (_stop)", force=True) exit_time = time.time() + self._stop_timeout forced_stop = True while self._connection.root.running and ( not forced_stop or time.time() < exit_time): if not self._connection.root.running: break if time.time() >= exit_time and (not forced_stop): self._connection.root.stop( "Forced to stop by intent of SoftwareRunner (_stop)", force=True) exit_time = time.time() + self._stop_timeout forced_stop = True stop_failed = "Not stopped by stop sequence" if self._connection.root.running: if stop_failed is not None: stop_failed += ", " else: stop_failed = "" stop_failed += "App not stopped" # TODO: check whether isntance is running and call stop explicitly if self._display_log_on_stop: data = list(self._connection.root.log) vprint("Platform {}. Exit log: {}".format( self.name, '\n'.join(data))) self._connection.close() except Exception as e: eprint("Platform {} experienced exception on stop: {}".format( self.name, e)) exprint() self._reply_all( reply_contexts, PM.notify( "abnormal_stop: experienced exception on stop: {}".format( e))) if stop_failed is not None: stop_failed += ", " else: stop_failed = "" stop_failed += "Experienced exception: {}".format(e) if stop_failed is not None: eprint("Software runner {} failed to stop app properly: {}".format( self.name, stop_failed)) self._reply_all( reply_contexts, PM.notify("Software runner {} failed to stop app properly: {}". format(self.name, stop_failed))) self._connection = None super_result = super(SoftwareRunner, self)._stop(reply_contexts) if stop_failed: return proto_failure(stop_failed) else: return super_result
def _extrapolate_string(self, name, path, value, expr, types): if not isinstance(value[0], str): return False data = value[0] if not expr: if not types: m = re.findall( r"\$(s?){(\w+\.?\w+(?:(?:\[\d+\])|(?:\['.+?'\])|(?:\[\".+?\"\]))*)}", data) # TODO: avoid escaped chars else: m = re.findall(r"^\$([bfi]){(.+)}$", data) else: m = re.findall(r"\$(es?){(.+?)}", data) # TODO: avoid escaped chars if m is None or len(m) == 0: return False values = {} straight = False straight_value = None for t, k in list(m): if k in values: continue # NOTE: it's enough to extrapolate one time try: if not expr: if not types: tmp = self._get_extrapolated_value(k) # If specified to be string or it's part of value - convert value to string if t == 's' or len(r"$" + t + "{" + k + "}") != len(data) or name in ( "alias", ): values[k] = str(tmp) else: straight = True straight_value = tmp else: straight = True if len(k > 2) and (k[0] == "'" and k[-1] == "'" or k[0] == '"' and k[-1] == '"'): vv = k[1:-1] else: vv = k if t == 'b': if vv in ('True', 'False'): straight_value = vv == 'True' else: straight_value = bool(int(vv)) elif t == 'f': straight_value = float(vv) else: straight_value = int(vv) else: tmp = evaluate(k) if t == 'es' or len(r"$" + t + "{" + k + "}") != len(data) or name in ( "alias", ): values[k] = str(tmp) else: straight = True straight_value = tmp except Exception as e: eprint( "Exception occurred while extrapolating {} with value {}." "\n Failed on {}, exception: {}".format(path, data, k, e)) exprint() raise e if straight: value[0] = straight_value else: for t, k in list(m): kv = r"\$" + t + "{" + re.escape(k) + "}" vprint("Extrapolated {}: {}->{}".format( value[0], kv, values[k])) value[0] = re.sub(kv, values[k], value[0]) return True
def load_description(self, description, generics, verbose, extrapolate=True): # TODO: include could be at any level # TODO: include in specials should contain name self._sch_data = None if os.path.isfile(description): if verbose: vprint("openning {}".format(description)) with open(description, 'r') as stream: try: desc = yaml.safe_load(stream) if verbose: vprint("loaded data: {}".format(desc)) except yaml.YAMLError as exc: eprint(exc) desc = {} elif isinstance(description, str): description = re.sub(r"\\n", "\n", description) # TODO: unescape slashes desc = yaml.safe_load(description) else: desc = copy.deepcopy(description) assert isinstance( desc, dict), "Expecting a dict as scheme description! Got {}".format( type(desc)) if self._root_section is not None: assert self._root_section in desc, "Root section '{}' wasn't found in description".format( self._root_section) assert isinstance(desc[self._root_section], (dict, list, tuple)), "Root section should be a dict, list or tuple! " \ "Got {}".format(type(desc[self._root_section])) desc = desc[self._root_section] if isinstance(desc, (list, tuple)): assert len( desc ) == 1, "descriptions with more than 1 root section isn't supported yet!" desc = desc[0] # if "__metadata__" in desc: # raise ValueError("'__metadata__' section shouldn't be in testenv description") # if "__nodes__" in desc: # raise ValueError("'__nodes__' section shouldn't be in testenv description") # desc["__metadata__"] = {'name': None, 'description': None} # desc["__nodes__"] = [] if "name" not in desc: raise ValueError("No scheme name found in description!") self._name = desc["name"] for s in self.specials: # S is for SECTION if s in desc: assert isinstance( desc[s], dict ), "Specials like {} should be a dict only! Got: {} for {}".format( " ".join(self.specials), type(desc[s]), s) self._data[s] = copy.deepcopy(desc[s]) else: self._data[s] = {} if generics is not None: for g in generics: if g not in self._data["generics"]: raise ValueError( "Generic {} is not in environment".format(g)) else: if generics[g] in ("True", "False"): self._data["generics"][g] = generics[g] == "True" else: try: self._data["generics"][g] = int(generics[g]) except Exception as e: try: self._data["generics"][g] = float(generics[g]) except Exception as e: self._data["generics"][g] = generics[g] self._sch_data = desc errors = [] _sch_structure = self._sch_to_structured(errors, "", self._sch_data, 'root', None, None) if len(errors) > 0: message = "Errors occurred during description parsing:\n" + "\n".join( errors) raise ValueError(message) assert len(_sch_structure ) == 1, "Unexpectedly found multiple roots for scheme" self._sch_structure = _sch_structure[0] if extrapolate: self.extrapolate_description()
def check_responses(verbose=False): if len(conv_analyzer.in_progress) > 0: eprint( "Some platforms ({}) are not completed transaction {}::{}!" .format(conv_analyzer.in_progress, channel, message.serialize())) return False elif len(conv_analyzer.participants) == 0: eprint("No one acknowledged transaction {}::{}!".format( channel, message.serialize())) return False elif not isinstance(expected, (list, tuple)): raise ValueError( "'expected' argument should be a list or tuple of lists or tuples!" ) elif len(expected) == 0: eprint( "Warning! Transaction {}::{} is without expected result! Completed but no checks made" .format(channel, message.serialize())) return True else: participants = conv_analyzer.participants others = conv_analyzer.participants total = len(others) result = [False] * len(expected) idx = 0 for e in expected: if not isinstance(e, (list, tuple)) or len(e) < 2: raise ValueError( "'expected' items should be list or tuple with length 2 or more" ) negative = False if len(e) >= 3: negative = e[2] matched = sorted( conv_analyzer.partipiciants_by_result(e[1], negative)) if e[0] == "all": others = [] if len(matched) == total: result[idx] = True else: unmatched = [ p for p in participants if p not in matched ] # participants - matched eprint( "Platforms {} are failed to check against '{}{}' for 'all' " "on transaction {}::{}".format( unmatched, ["", "not "][negative], e[1], channel, message.serialize())) for p in unmatched: eprint(" {}: {}".format( p, conv_analyzer.replies[p].serialize())) elif e[0] == "any": others = [] if len(matched) > 0: result[idx] = True else: unmatched = participants eprint( "None of platforms {} succeeded check against '{}{}' for 'any' on transaction " "{}::{}".format(unmatched, ["", "not "][negative], e[1], channel, message.serialize())) for p in unmatched: eprint(" {}: {}".format( p, conv_analyzer.replies[p].serialize())) elif e[0] == "others": if len(others) == 0: eprint( "No other platforms left to check against '{}{}' on transaction {}::{}" .format(["", "not "][negative], e[1], channel, message.serialize())) result[idx] = True else: unmatched = [ p for p in others if p not in matched ] # others - matched others = [] if len(unmatched) > 0: eprint( "Platforms {} are failed to check against '{}{}' for 'others' on transaction" " {}::{}".format(unmatched, ["", "not "][negative], e[1], channel, message.serialize())) for p in unmatched: eprint(" {}: {}".format( p, conv_analyzer.replies[p].serialize())) else: result[idx] = True elif e[0] == "none": if len(matched) == 0: result[idx] = True else: unmatched = matched eprint( "Platforms {} are failed to check against '{}{}' for 'none' " "on transaction {}::{}".format( unmatched, ["", "not "][negative], e[1], channel, message.serialize())) for p in unmatched: eprint(" {}: {}".format( p, conv_analyzer.replies[p].serialize())) else: if not isinstance(e[0], (list, tuple)): el = [e[0]] else: el = e[0] others = [p for p in others if p not in el] # others - el unmatched = [p for p in el if p not in matched] # el - matched if len(unmatched) == 0: result[idx] = True else: eprint( "Platforms {} are failed to check against '{}{}' on transaction {}::{}" .format(unmatched, ["", "not "][negative], e[1], channel, message.serialize())) for p in unmatched: eprint(" {}: {}".format( p, conv_analyzer.replies[p].serialize())) idx += 1 result = all(r is True for r in result) if result: if verbose: vprint( "Success! Transaction {}::{} completed successfully". format(channel, message.serialize())) else: if verbose: eprint("Error! Transaction {}::{} failed on checks".format( channel, message.serialize())) return result
def transaction(self, channel, message, expected=None, ignore=None, more_info=False): """ Conducts transaction on specified channel and checks platform's results against expected value Transaction is started from starting new thread in a specified channel and sending message into it Then replies are monitored. If platform has replied into channel at least once it's treated to be a participant After conversation on channel is finished it's ensured that: * there were at least single participant * all participants has finished transaction * if expected result is specified then participants's results are checked against expected value * if all checks were successful then transaction assumed successful to, otherwise it's failed for some reason :param channel: string, messaging channel's name :param message: PlatformMessage, to send into channel to initiate transaction :param expected: list with expected results. List of pairs (as lists or tuples). 1st value in a pair is for participant names and 2nd is for expected value - "success"/"fail" or "s"/"f" also special keys supported: * all - all participants * any - any of participants * none - none of participants * others - should be last in a list. corresponds to all participants that were not covered by any of previous keys BE WARE! Order of items in list does matters if None then it's assumed that "all", "success" expected :param ignore: list, names of platforms that should be excluded from participants list :param more_info: If True then dict is returned with transaction result and all last replies, otherwise - True if transaction were successful, False if not :return: True/False or dict depending on more_info """ # TODO: Expected participants list if not isinstance(channel, str): raise ValueError( "channel should be a string! got {} with value {}".format( type(channel), channel)) if not isinstance(message, PlatformMessage): raise ValueError( "message should be a PlatformMessage! got {} with value {}". format(type(message), message)) message.sender = self.name if expected is None: expected = er.all_success if not isinstance(expected, (list, tuple)): raise ValueError( "expected should be a list or tuple! got {} with value {}". format(type(expected), expected)) for e in expected: if not isinstance(e, (list, tuple)) or len(e) < 2 or len(e) > 3: raise ValueError( "items of expected should be a list or tuple with length 2 ot 3! " "got {} with value {}".format(type(e), e)) if ignore is not None and not isinstance(ignore, list): raise ValueError( "ignore should be a list with length 2 ot 3! got {} with value {}" .format(type(ignore), ignore)) # TODO: check content of testing:fake_next_op if ignore is None: ignore = [] # TODO: ignore is not used yet if self.verbose: vprint("Starting transaction {}::{}".format( channel, message.serialize())) context = self.farm.start_thread(self, channel, message.interface) # NOTE: If previous transaction were interrupted due to exception (and messaging session were broken) # then can't proceed further assert self.farm.send_message_in_progress is False, "Can't start transaction" \ " if there is other messaging session is going" replies = self.farm.send_message(context, message) # NOTE: it may take some time to accomplish request (async usage for example) # No matter if multithreading is supported or not # but in the end everything should be settled down at this point conv_analyzer = ConversationAnalyzer(replies=replies, ignore=[self.name] + ignore) def check_responses(verbose=False): if len(conv_analyzer.in_progress) > 0: eprint( "Some platforms ({}) are not completed transaction {}::{}!" .format(conv_analyzer.in_progress, channel, message.serialize())) return False elif len(conv_analyzer.participants) == 0: eprint("No one acknowledged transaction {}::{}!".format( channel, message.serialize())) return False elif not isinstance(expected, (list, tuple)): raise ValueError( "'expected' argument should be a list or tuple of lists or tuples!" ) elif len(expected) == 0: eprint( "Warning! Transaction {}::{} is without expected result! Completed but no checks made" .format(channel, message.serialize())) return True else: participants = conv_analyzer.participants others = conv_analyzer.participants total = len(others) result = [False] * len(expected) idx = 0 for e in expected: if not isinstance(e, (list, tuple)) or len(e) < 2: raise ValueError( "'expected' items should be list or tuple with length 2 or more" ) negative = False if len(e) >= 3: negative = e[2] matched = sorted( conv_analyzer.partipiciants_by_result(e[1], negative)) if e[0] == "all": others = [] if len(matched) == total: result[idx] = True else: unmatched = [ p for p in participants if p not in matched ] # participants - matched eprint( "Platforms {} are failed to check against '{}{}' for 'all' " "on transaction {}::{}".format( unmatched, ["", "not "][negative], e[1], channel, message.serialize())) for p in unmatched: eprint(" {}: {}".format( p, conv_analyzer.replies[p].serialize())) elif e[0] == "any": others = [] if len(matched) > 0: result[idx] = True else: unmatched = participants eprint( "None of platforms {} succeeded check against '{}{}' for 'any' on transaction " "{}::{}".format(unmatched, ["", "not "][negative], e[1], channel, message.serialize())) for p in unmatched: eprint(" {}: {}".format( p, conv_analyzer.replies[p].serialize())) elif e[0] == "others": if len(others) == 0: eprint( "No other platforms left to check against '{}{}' on transaction {}::{}" .format(["", "not "][negative], e[1], channel, message.serialize())) result[idx] = True else: unmatched = [ p for p in others if p not in matched ] # others - matched others = [] if len(unmatched) > 0: eprint( "Platforms {} are failed to check against '{}{}' for 'others' on transaction" " {}::{}".format(unmatched, ["", "not "][negative], e[1], channel, message.serialize())) for p in unmatched: eprint(" {}: {}".format( p, conv_analyzer.replies[p].serialize())) else: result[idx] = True elif e[0] == "none": if len(matched) == 0: result[idx] = True else: unmatched = matched eprint( "Platforms {} are failed to check against '{}{}' for 'none' " "on transaction {}::{}".format( unmatched, ["", "not "][negative], e[1], channel, message.serialize())) for p in unmatched: eprint(" {}: {}".format( p, conv_analyzer.replies[p].serialize())) else: if not isinstance(e[0], (list, tuple)): el = [e[0]] else: el = e[0] others = [p for p in others if p not in el] # others - el unmatched = [p for p in el if p not in matched] # el - matched if len(unmatched) == 0: result[idx] = True else: eprint( "Platforms {} are failed to check against '{}{}' on transaction {}::{}" .format(unmatched, ["", "not "][negative], e[1], channel, message.serialize())) for p in unmatched: eprint(" {}: {}".format( p, conv_analyzer.replies[p].serialize())) idx += 1 result = all(r is True for r in result) if result: if verbose: vprint( "Success! Transaction {}::{} completed successfully". format(channel, message.serialize())) else: if verbose: eprint("Error! Transaction {}::{} failed on checks".format( channel, message.serialize())) return result result = check_responses(self.verbose) if more_info: result = {"result": result, "replies": conv_analyzer.replies} return result