def parse_testsets(base_url, test_structure, test_files = set(), working_directory = None): """ Convert a Python datastructure read from validated YAML to a set of structured testsets The data stucture is assumed to be a list of dictionaries, each of which describes: - a tests (test structure) - a simple test (just a URL, and a minimal test is created) - or overall test configuration for this testset - an import (load another set of tests into this one, from a separate file) - For imports, these are recursive, and will use the parent config if none is present Note: test_files is used to track tests that import other tests, to avoid recursive loops This returns a list of testsets, corresponding to imported testsets and in-line multi-document sets """ tests_out = list() test_config = TestConfig() testsets = list() benchmarks = list() if working_directory is None: working_directory = os.path.abspath(os.getcwd()) #returns a testconfig and collection of tests for node in test_structure: #Iterate through lists of test and configuration elements if isinstance(node,dict): #Each config element is a miniature key-value dictionary node = lowercase_keys(node) for key in node: if key == u'import': importfile = node[key] #import another file if importfile not in test_files: logger.debug("Importing test sets: " + importfile) test_files.add(importfile) import_test_structure = read_test_file(importfile) with cd(os.path.dirname(os.path.realpath(importfile))): import_testsets = parse_testsets(base_url, import_test_structure, test_files) testsets.extend(import_testsets) elif key == u'url': #Simple test, just a GET to a URL mytest = Test() val = node[key] assert isinstance(val,str) or isinstance(val,unicode) mytest.url = base_url + val tests_out.append(mytest) elif key == u'test': #Complex test with additional parameters with cd(working_directory): child = node[key] mytest = Test.parse_test(base_url, child) tests_out.append(mytest) elif key == u'benchmark': benchmark = parse_benchmark(base_url, node[key]) benchmarks.append(benchmark) elif key == u'config' or key == u'configuration': test_config = parse_configuration(node[key]) testset = TestSet() testset.tests = tests_out testset.config = test_config testset.benchmarks = benchmarks testsets.append(testset) return testsets
def parse_benchmark(base_url, node): """ Try building a benchmark configuration from deserialized configuration root node """ node = lowercase_keys(flatten_dictionaries(node)) # Make it usable benchmark = Benchmark() # Read & set basic test parameters benchmark = Test.parse_test(base_url, node, benchmark) # Complex parsing because of list/dictionary/singleton legal cases for key, value in node.items(): if key == u'warmup_runs': benchmark.warmup_runs = int(value) elif key == u'benchmark_runs': benchmark.benchmark_runs = int(value) elif key == u'output_format': format = value.lower() if format in OUTPUT_FORMATS: benchmark.output_format = format else: raise Exception('Invalid benchmark output format: ' + format) elif key == u'output_file': if not isinstance(value, basestring): raise Exception("Invalid output file format") benchmark.output_file = value elif key == u'metrics': if isinstance(value, unicode) or isinstance(value,str): # Single value benchmark.add_metric(unicode(value, 'UTF-8')) elif isinstance(value, list) or isinstance(value, set): # List of single values or list of {metric:aggregate, ...} for metric in value: if isinstance(metric, dict): for metricname, aggregate in metric.items(): if not isinstance(metricname, basestring): raise Exception("Invalid metric input: non-string metric name") if not isinstance(aggregate, basestring): raise Exception("Invalid aggregate input: non-string aggregate name") # TODO unicode-safe this benchmark.add_metric(unicode(metricname,'UTF-8'), unicode(aggregate,'UTF-8')) elif isinstance(metric, unicode) or isinstance(metric, str): benchmark.add_metric(unicode(metric,'UTF-8')) elif isinstance(value, dict): # Dictionary of metric-aggregate pairs for metricname, aggregate in value.items(): if not isinstance(metricname, basestring): raise Exception("Invalid metric input: non-string metric name") if not isinstance(aggregate, basestring): raise Exception("Invalid aggregate input: non-string aggregate name") benchmark.add_metric(unicode(metricname,'UTF-8'), unicode(aggregate,'UTF-8')) else: raise Exception("Invalid benchmark metric datatype: "+str(value)) return benchmark
def parse_benchmark(base_url, node): """ Try building a benchmark configuration from deserialized configuration root node """ node = lowercase_keys(flatten_dictionaries(node)) # Make it usable benchmark = Benchmark() # Read & set basic test parameters benchmark = Test.parse_test(base_url, node, benchmark) # Complex parsing because of list/dictionary/singleton legal cases for key, value in node.items(): if key == u'warmup_runs': benchmark.warmup_runs = int(value) elif key == u'benchmark_runs': benchmark.benchmark_runs = int(value) elif key == u'output_format': format = value.lower() if format in OUTPUT_FORMATS: benchmark.output_format = format else: raise Exception('Invalid benchmark output format: ' + format) elif key == u'output_file': if not isinstance(value, basestring): raise Exception("Invalid output file format") benchmark.output_file = value elif key == u'metrics': if isinstance(value, unicode) or isinstance(value, str): # Single value benchmark.add_metric(unicode(value, 'UTF-8')) elif isinstance(value, list) or isinstance(value, set): # List of single values or list of {metric:aggregate, ...} for metric in value: if isinstance(metric, dict): for metricname, aggregate in metric.items(): if not isinstance(metricname, basestring): raise Exception( "Invalid metric input: non-string metric name" ) if not isinstance(aggregate, basestring): raise Exception( "Invalid aggregate input: non-string aggregate name" ) # TODO unicode-safe this benchmark.add_metric(unicode(metricname, 'UTF-8'), unicode(aggregate, 'UTF-8')) elif isinstance(metric, unicode) or isinstance( metric, str): benchmark.add_metric(unicode(metric, 'UTF-8')) elif isinstance(value, dict): # Dictionary of metric-aggregate pairs for metricname, aggregate in value.items(): if not isinstance(metricname, basestring): raise Exception( "Invalid metric input: non-string metric name") if not isinstance(aggregate, basestring): raise Exception( "Invalid aggregate input: non-string aggregate name" ) benchmark.add_metric(unicode(metricname, 'UTF-8'), unicode(aggregate, 'UTF-8')) else: raise Exception("Invalid benchmark metric datatype: " + str(value)) return benchmark