def test_3parameters_sparse(self):
experiment = read_jsonlines_file('data/jsonlines/matrix_3p.jsonl')
modeler = MultiParameterModeler()
measurements = experiment.measurements[(Callpath('<root>'), Metric('metr'))]
for _ in range(len(measurements)):
shuffle(measurements)
f_msm = modeler.find_best_measurement_points(measurements)
self.assertEqual(len(f_msm), 3)
self.assertSetEqual(set(m.coordinate for m in f_msm[0]), {
Coordinate(1),
Coordinate(2),
Coordinate(3),
Coordinate(4),
Coordinate(5)
})
self.assertListEqual([1] * 5, [m.mean for m in f_msm[0]])
self.assertSetEqual(set(m.coordinate for m in f_msm[1]), {
Coordinate(1),
Coordinate(2),
Coordinate(3),
Coordinate(4),
Coordinate(5)
})
self.assertListEqual([1] * 5, [m.mean for m in f_msm[1]])
self.assertSetEqual(set(m.coordinate for m in f_msm[2]), {
Coordinate(1),
Coordinate(2),
Coordinate(3),
Coordinate(4),
Coordinate(5)
})
self.assertListEqual([1] * 5, [m.mean for m in f_msm[2]])
def test_3parameters_bands(self):
experiment = read_jsonlines_file(
'data/jsonlines/matrix_3p_bands.jsonl')
modeler = MultiParameterModeler()
measurements = experiment.measurements[(Callpath('<root>'),
Metric('metr'))]
f_msm = modeler.find_best_measurement_points(measurements)
self.assertEqual(len(f_msm), 3)
self.assertListEqual([m.coordinate for m in f_msm[0]], [
Coordinate(1),
Coordinate(2),
Coordinate(3),
Coordinate(4),
Coordinate(5)
])
self.assertListEqual([0] + [1] * 4, [m.mean for m in f_msm[0]])
self.assertListEqual([m.coordinate for m in f_msm[1]], [
Coordinate(1),
Coordinate(2),
Coordinate(3),
Coordinate(4),
Coordinate(5)
])
self.assertListEqual([0.5] + [2.5] * 4, [m.mean for m in f_msm[1]])
self.assertListEqual([m.coordinate for m in f_msm[2]], [
Coordinate(1),
Coordinate(2),
Coordinate(3),
Coordinate(4),
Coordinate(5)
])
self.assertListEqual([0] + [4] * 4, [m.mean for m in f_msm[2]])
def read_json_file(path, progress_bar=DUMMY_PROGRESS):
# read lines from json file
with open(path, "r") as inputfile:
try:
json_data = json.load(inputfile)
except JSONDecodeError as error:
inputfile.seek(0)
is_jsonlines = any(line.strip().startswith('{') for line in inputfile) and \
all(line.strip().startswith('{') or line.strip() == "" for line in inputfile)
if is_jsonlines:
return read_jsonlines_file(path, progress_bar=DUMMY_PROGRESS)
else:
raise FileFormatError(str(error)) from error
# create an experiment object to save the date loaded from the text file
experiment = Experiment()
if "callpaths" not in json_data:
try:
_read_new_json_file(experiment, json_data, progress_bar)
except KeyError as err:
raise FileFormatError(str(err)) from err
else:
try:
_read_legacy_json_file(experiment, json_data, progress_bar)
except KeyError as err:
raise FileFormatError(str(err)) from err
call_tree = create_call_tree(experiment.callpaths, progress_bar)
experiment.call_tree = call_tree
io_helper.validate_experiment(experiment, progress_bar)
return experiment
def test_matrix_3p(self):
experiment = read_jsonlines_file('data/jsonlines/matrix_3p.jsonl')
modeler = MultiParameterModeler()
modeler.single_parameter_point_selection = 'all'
# initialize model generator
model_generator = ModelGenerator(experiment, modeler)
# create models from data
self.assertWarns(UserWarning, model_generator.model_all)
def test_complete_matrix_2p(self):
experiment = read_jsonlines_file('data/jsonlines/complete_matrix_2p.jsonl')
modeler = MultiParameterModeler()
modeler.single_parameter_point_selection = 'all'
# initialize model generator
model_generator = ModelGenerator(experiment, modeler)
# create models from data
model_generator.model_all()
def test_read_1(self):
Parameter.ID_COUNTER = itertools.count()
experiment = read_jsonlines_file("data/jsonlines/test1.jsonl")
self.assertListEqual([Parameter('x'), Parameter('y')],
experiment.parameters)
self.assertListEqual([0, 1], [p.id for p in experiment.parameters])
self.assertListEqual([
Coordinate(x, y) for x in range(1, 5 + 1) for y in range(1, 5 + 1)
], experiment.coordinates)
self.assertListEqual([Metric('metr')], experiment.metrics)
self.assertListEqual([Callpath('<root>')], experiment.callpaths)
def test_read_2(self):
Parameter.ID_COUNTER = itertools.count()
experiment = read_jsonlines_file("data/jsonlines/test2.jsonl")
self.assertListEqual([Parameter('p'), Parameter('n')],
experiment.parameters)
self.assertListEqual([0, 1], [p.id for p in experiment.parameters])
self.assertListEqual([
Coordinate(x, y) for x in [16, 32, 64, 128, 256]
for y in [100, 200, 300, 400, 500]
], experiment.coordinates)
self.assertListEqual([Metric('metr')], experiment.metrics)
self.assertListEqual([Callpath('<root>')], experiment.callpaths)
def test_matrix4p(self):
Parameter.ID_COUNTER = itertools.count()
experiment = read_jsonlines_file("data/jsonlines/matrix_4p.jsonl")
self.assertListEqual(
[Parameter('a'),
Parameter('b'),
Parameter('c'),
Parameter('d')], experiment.parameters)
self.assertListEqual([0, 1, 2, 3],
[p.id for p in experiment.parameters])
self.assertListEqual([Coordinate(x, 1, 1, 1)
for x in range(1, 5 + 1)] +
[Coordinate(1, x, 1, 1)
for x in range(2, 5 + 1)] +
[Coordinate(1, 1, x, 1)
for x in range(2, 5 + 1)] +
[Coordinate(1, 1, 1, x) for x in range(2, 5 + 1)],
experiment.coordinates)
self.assertListEqual([Metric('metr')], experiment.metrics)
self.assertListEqual([Callpath('<root>')], experiment.callpaths)
def test_3parameters_bands_incomplete(self):
experiment = read_jsonlines_file('data/jsonlines/matrix_3p_bands_incomplete.jsonl')
modeler = MultiParameterModeler()
measurements = experiment.measurements[(Callpath('<root>'), Metric('metr'))]
f_msm = modeler.find_best_measurement_points(measurements)
self.assertEqual(len(f_msm), 3)
self.assertListEqual([m.coordinate for m in f_msm[0]], [
Coordinate(c) for c in [1, 3, 4, 5, 6]
])
self.assertListEqual([0] + [1] * 4, [m.mean for m in f_msm[0]])
self.assertListEqual([m.coordinate for m in f_msm[1]], [
Coordinate(c) for c in range(1, 5 + 1)
])
self.assertListEqual([0] + [2] * 4, [m.mean for m in f_msm[1]])
self.assertListEqual([m.coordinate for m in f_msm[2]], [
Coordinate(c) for c in range(1, 5 + 1)
])
self.assertListEqual([0] + [4] * 4, [m.mean for m in f_msm[2]])
measurements.reverse()
f_msm = modeler.find_best_measurement_points(measurements)
self.assertEqual(len(f_msm), 3)
self.assertListEqual([m.coordinate for m in f_msm[0]], [
Coordinate(c) for c in reversed([1, 3, 4, 5, 6])
])
self.assertListEqual([1] * 4 + [0], [m.mean for m in f_msm[0]])
self.assertListEqual([m.coordinate for m in f_msm[1]], [
Coordinate(c) for c in [6, 5, 4, 3, 2]
])
self.assertListEqual([3] * 5, [m.mean for m in f_msm[1]])
self.assertListEqual([m.coordinate for m in f_msm[2]], [
Coordinate(c) for c in reversed(range(1, 5 + 1))
])
self.assertListEqual([4] * 4 + [0], [m.mean for m in f_msm[2]])
def test_sparse_matrix2p(self):
Parameter.ID_COUNTER = itertools.count()
experiment = read_jsonlines_file(
"data/jsonlines/sparse_matrix_2p.jsonl")
self.assertListEqual([Parameter('x'), Parameter('y')],
experiment.parameters)
self.assertListEqual([0, 1], [p.id for p in experiment.parameters])
self.assertListEqual([
Coordinate(20, 1),
Coordinate(30, 1),
Coordinate(30, 2),
Coordinate(40, 1),
Coordinate(40, 2),
Coordinate(40, 3),
Coordinate(50, 1),
Coordinate(50, 2),
Coordinate(50, 3),
Coordinate(50, 4),
Coordinate(60, 1),
Coordinate(60, 2),
Coordinate(60, 3),
Coordinate(60, 4),
Coordinate(60, 5),
Coordinate(70, 2),
Coordinate(70, 3),
Coordinate(70, 4),
Coordinate(70, 5),
Coordinate(80, 3),
Coordinate(80, 4),
Coordinate(80, 5),
Coordinate(90, 4),
Coordinate(90, 5),
Coordinate(100, 5)
], experiment.coordinates)
self.assertListEqual([Metric('metr')], experiment.metrics)
self.assertListEqual([Callpath('<root>')], experiment.callpaths)
def test_complete_matrix_2p(self):
experiment = read_jsonlines_file(
"data/jsonlines/complete_matrix_2p.jsonl")
def test_input_1(self):
experiment = read_jsonlines_file("data/jsonlines/input_1.jsonl")
def test_input_1(self):
experiment = read_jsonlines_file('data/jsonlines/input_1.jsonl')
# initialize model generator
model_generator = ModelGenerator(experiment)
# create models from data
model_generator.model_all()
