def parse_testsets(base_url, test_structure, test_files = set(), working_directory = None):
""" Convert a Python datastructure read from validated YAML to a set of structured testsets
The data stucture is assumed to be a list of dictionaries, each of which describes:
- a tests (test structure)
- a simple test (just a URL, and a minimal test is created)
- or overall test configuration for this testset
- an import (load another set of tests into this one, from a separate file)
- For imports, these are recursive, and will use the parent config if none is present
Note: test_files is used to track tests that import other tests, to avoid recursive loops
This returns a list of testsets, corresponding to imported testsets and in-line multi-document sets
"""
tests_out = list()
test_config = TestConfig()
testsets = list()
benchmarks = list()
if working_directory is None:
working_directory = os.path.abspath(os.getcwd())
#returns a testconfig and collection of tests
for node in test_structure: #Iterate through lists of test and configuration elements
if isinstance(node,dict): #Each config element is a miniature key-value dictionary
node = lowercase_keys(node)
for key in node:
if key == u'import':
importfile = node[key] #import another file
if importfile not in test_files:
logger.debug("Importing test sets: " + importfile)
test_files.add(importfile)
import_test_structure = read_test_file(importfile)
with cd(os.path.dirname(os.path.realpath(importfile))):
import_testsets = parse_testsets(base_url, import_test_structure, test_files)
testsets.extend(import_testsets)
elif key == u'url': #Simple test, just a GET to a URL
mytest = Test()
val = node[key]
assert isinstance(val,str) or isinstance(val,unicode)
mytest.url = base_url + val
tests_out.append(mytest)
elif key == u'test': #Complex test with additional parameters
with cd(working_directory):
child = node[key]
mytest = Test.parse_test(base_url, child)
tests_out.append(mytest)
elif key == u'benchmark':
benchmark = parse_benchmark(base_url, node[key])
benchmarks.append(benchmark)
elif key == u'config' or key == u'configuration':
test_config = parse_configuration(node[key])
testset = TestSet()
testset.tests = tests_out
testset.config = test_config
testset.benchmarks = benchmarks
testsets.append(testset)
return testsets
def parse_testsets(base_url, test_structure, test_files = set(), working_directory = None):
""" Convert a Python datastructure read from validated YAML to a set of structured testsets
The data stucture is assumed to be a list of dictionaries, each of which describes:
- a tests (test structure)
- a simple test (just a URL, and a minimal test is created)
- or overall test configuration for this testset
- an import (load another set of tests into this one, from a separate file)
- For imports, these are recursive, and will use the parent config if none is present
Note: test_files is used to track tests that import other tests, to avoid recursive loops
This returns a list of testsets, corresponding to imported testsets and in-line multi-document sets
"""
tests_out = list()
test_config = TestConfig()
testsets = list()
benchmarks = list()
if working_directory is None:
working_directory = os.path.abspath(os.getcwd())
#returns a testconfig and collection of tests
for node in test_structure: #Iterate through lists of test and configuration elements
if isinstance(node,dict): #Each config element is a miniature key-value dictionary
node = lowercase_keys(node)
for key in node:
if key == u'import':
importfile = node[key] #import another file
if importfile not in test_files:
logger.debug("Importing test sets: " + importfile)
test_files.add(importfile)
import_test_structure = read_test_file(importfile)
with cd(os.path.dirname(os.path.realpath(importfile))):
import_testsets = parse_testsets(base_url, import_test_structure, test_files)
testsets.extend(import_testsets)
elif key == u'url': #Simple test, just a GET to a URL
mytest = Test()
val = node[key]
assert isinstance(val,str) or isinstance(val,unicode)
mytest.url = base_url + val
tests_out.append(mytest)
elif key == u'test': #Complex test with additional parameters
with cd(working_directory):
child = node[key]
mytest = Test.parse_test(base_url, child)
tests_out.append(mytest)
elif key == u'benchmark':
benchmark = parse_benchmark(base_url, node[key])
benchmarks.append(benchmark)
elif key == u'config' or key == u'configuration':
test_config = parse_configuration(node[key])
testset = TestSet()
testset.tests = tests_out
testset.config = test_config
testset.benchmarks = benchmarks
testsets.append(testset)
return testsets
def parse_benchmark(base_url, node):
""" Try building a benchmark configuration from deserialized configuration root node """
node = lowercase_keys(flatten_dictionaries(node)) # Make it usable
benchmark = Benchmark()
# Read & set basic test parameters
benchmark = Test.parse_test(base_url, node, benchmark)
# Complex parsing because of list/dictionary/singleton legal cases
for key, value in node.items():
if key == u'warmup_runs':
benchmark.warmup_runs = int(value)
elif key == u'benchmark_runs':
benchmark.benchmark_runs = int(value)
elif key == u'output_format':
format = value.lower()
if format in OUTPUT_FORMATS:
benchmark.output_format = format
else:
raise Exception('Invalid benchmark output format: ' + format)
elif key == u'output_file':
if not isinstance(value, basestring):
raise Exception("Invalid output file format")
benchmark.output_file = value
elif key == u'metrics':
if isinstance(value, unicode) or isinstance(value,str):
# Single value
benchmark.add_metric(unicode(value, 'UTF-8'))
elif isinstance(value, list) or isinstance(value, set):
# List of single values or list of {metric:aggregate, ...}
for metric in value:
if isinstance(metric, dict):
for metricname, aggregate in metric.items():
if not isinstance(metricname, basestring):
raise Exception("Invalid metric input: non-string metric name")
if not isinstance(aggregate, basestring):
raise Exception("Invalid aggregate input: non-string aggregate name")
# TODO unicode-safe this
benchmark.add_metric(unicode(metricname,'UTF-8'), unicode(aggregate,'UTF-8'))
elif isinstance(metric, unicode) or isinstance(metric, str):
benchmark.add_metric(unicode(metric,'UTF-8'))
elif isinstance(value, dict):
# Dictionary of metric-aggregate pairs
for metricname, aggregate in value.items():
if not isinstance(metricname, basestring):
raise Exception("Invalid metric input: non-string metric name")
if not isinstance(aggregate, basestring):
raise Exception("Invalid aggregate input: non-string aggregate name")
benchmark.add_metric(unicode(metricname,'UTF-8'), unicode(aggregate,'UTF-8'))
else:
raise Exception("Invalid benchmark metric datatype: "+str(value))
return benchmark
def parse_benchmark(base_url, node):
""" Try building a benchmark configuration from deserialized configuration root node """
node = lowercase_keys(flatten_dictionaries(node)) # Make it usable
benchmark = Benchmark()
# Read & set basic test parameters
benchmark = Test.parse_test(base_url, node, benchmark)
# Complex parsing because of list/dictionary/singleton legal cases
for key, value in node.items():
if key == u'warmup_runs':
benchmark.warmup_runs = int(value)
elif key == u'benchmark_runs':
benchmark.benchmark_runs = int(value)
elif key == u'output_format':
format = value.lower()
if format in OUTPUT_FORMATS:
benchmark.output_format = format
else:
raise Exception('Invalid benchmark output format: ' + format)
elif key == u'output_file':
if not isinstance(value, basestring):
raise Exception("Invalid output file format")
benchmark.output_file = value
elif key == u'metrics':
if isinstance(value, unicode) or isinstance(value, str):
# Single value
benchmark.add_metric(unicode(value, 'UTF-8'))
elif isinstance(value, list) or isinstance(value, set):
# List of single values or list of {metric:aggregate, ...}
for metric in value:
if isinstance(metric, dict):
for metricname, aggregate in metric.items():
if not isinstance(metricname, basestring):
raise Exception(
"Invalid metric input: non-string metric name"
)
if not isinstance(aggregate, basestring):
raise Exception(
"Invalid aggregate input: non-string aggregate name"
)
# TODO unicode-safe this
benchmark.add_metric(unicode(metricname, 'UTF-8'),
unicode(aggregate, 'UTF-8'))
elif isinstance(metric, unicode) or isinstance(
metric, str):
benchmark.add_metric(unicode(metric, 'UTF-8'))
elif isinstance(value, dict):
# Dictionary of metric-aggregate pairs
for metricname, aggregate in value.items():
if not isinstance(metricname, basestring):
raise Exception(
"Invalid metric input: non-string metric name")
if not isinstance(aggregate, basestring):
raise Exception(
"Invalid aggregate input: non-string aggregate name"
)
benchmark.add_metric(unicode(metricname, 'UTF-8'),
unicode(aggregate, 'UTF-8'))
else:
raise Exception("Invalid benchmark metric datatype: " +
str(value))
return benchmark
