def make_measurement(c, ms: Sequence[Measurement]):
if len(ms) == 1:
measurement = copy.copy(ms[0])
measurement.coordinate = Coordinate(c)
return measurement
measurement = Measurement(Coordinate(c), ms[0].callpath, ms[0].metric, None)
if self.use_median:
value = np.mean([m.median for m in ms])
else:
value = np.mean([m.mean for m in ms])
measurement.mean = value
measurement.median = value
if measurement.mean == 0:
measurement.maximum = np.mean([m.maximum for m in ms])
measurement.minimum = np.mean([m.minimum for m in ms])
measurement.std = np.mean([m.std for m in ms])
else:
measurement.maximum = np.nanmean([m.maximum / m.mean for m in ms]) * measurement.mean
measurement.minimum = np.nanmean([m.minimum / m.mean for m in ms]) * measurement.mean
measurement.std = np.nanmean([m.std / m.mean for m in ms]) * measurement.mean
return measurement
def test_2parameters_reversed(self):
experiment = read_text_file('data/text/two_parameter_1.txt')
modeler = MultiParameterModeler()
measurements = experiment.measurements[(Callpath('reg'),
Metric('metr'))]
measurements = list(reversed(measurements))
f_msm = modeler.find_best_measurement_points(measurements)
self.assertEqual(len(f_msm), 2)
self.assertListEqual([m.coordinate for m in f_msm[0]], [
Coordinate((60, )),
Coordinate((50, )),
Coordinate((40, )),
Coordinate((30, )),
Coordinate((20, ))
])
self.assertListEqual([m.coordinate for m in f_msm[1]], [
Coordinate((5, )),
Coordinate((4, )),
Coordinate((3, )),
Coordinate((2, )),
Coordinate((1, ))
])
def test_2parameters_random(self):
experiment = read_text_file('data/text/two_parameter_1.txt')
modeler = MultiParameterModeler()
measurements = experiment.measurements[(Callpath('reg'),
Metric('metr'))]
for _ in range(len(measurements)):
shuffle(measurements)
f_msm = modeler.find_first_measurement_points(measurements)
self.assertEqual(len(f_msm), 2)
self.assertSetEqual(
set(m.coordinate for m in f_msm[0]), {
Coordinate(20),
Coordinate(30),
Coordinate(40),
Coordinate(50),
Coordinate(60)
})
self.assertSetEqual(
set(m.coordinate for m in f_msm[1]), {
Coordinate(1),
Coordinate(2),
Coordinate(3),
Coordinate(4),
Coordinate(5)
})
def find_first_measurement_points(measurements: Sequence[Measurement]):
"""
This method returns the smallest possible measurements that should be used for creating
the single-parameter models.
"""
dimensions = measurements[0].coordinate.dimensions
min_coordinate = [
Coordinate(float('Inf') for _ in range(dimensions))
for _ in range(dimensions)
]
candidate_list = [[] for _ in range(dimensions)]
for m in measurements:
for p in range(dimensions):
if m.coordinate.is_mostly_equal(min_coordinate[p], p):
m_sp = copy.copy(m)
m_sp.coordinate = Coordinate(m.coordinate[p])
candidate_list[p].append(m_sp)
elif m.coordinate.is_mostly_lower(min_coordinate[p], p):
candidate_list[p].clear()
m_sp = copy.copy(m)
m_sp.coordinate = Coordinate(m.coordinate[p])
candidate_list[p].append(m_sp)
min_coordinate[p] = m.coordinate
return candidate_list
def test_single_parameter(self):
experiment = read_cube_file('data/cubeset/single_parameter', 'weak')
self.assertListEqual([Parameter('x')], experiment.parameters)
self.assertSetEqual(
{
Coordinate(1),
Coordinate(10),
Coordinate(25),
Coordinate(50),
Coordinate(100),
Coordinate(250),
Coordinate(500),
Coordinate(1000),
Coordinate(2000)
}, set(experiment.coordinates))
self.assertSetEqual(
{
Callpath('main'),
Callpath('main->init_mat'),
Callpath('main->zero_mat'),
Callpath('main->mat_mul')
}, set(experiment.callpaths))
self.assertSetEqual(
{
Metric('visits'),
Metric('time'),
Metric('min_time'),
Metric('max_time'),
Metric('PAPI_FP_OPS'),
Metric('PAPI_L3_TCM'),
Metric('PAPI_L2_TCM')
}, set(experiment.metrics))
read_cube_file('data/cubeset/single_parameter', 'strong')
def test_matrix3p(self):
Parameter.ID_COUNTER = itertools.count()
experiment = read_jsonlines_file("data/jsonlines/matrix_3p.jsonl")
self.assertListEqual(
[Parameter('x'), Parameter('y'),
Parameter('z')], experiment.parameters)
self.assertListEqual([0, 1, 2], [p.id for p in experiment.parameters])
self.assertListEqual([Coordinate(x, 1, 1) for x in range(1, 5 + 1)] +
[Coordinate(1, x, 1) for x in range(2, 5 + 1)] +
[Coordinate(1, 1, x) for x in range(2, 5 + 1)],
experiment.coordinates)
self.assertListEqual([Metric('metr')], experiment.metrics)
self.assertListEqual([Callpath('<root>')], experiment.callpaths)
def test_read_1(self):
Parameter.ID_COUNTER = itertools.count()
experiment = read_json_file("data/json/input_1.JSON")
self.assertListEqual(experiment.parameters, [Parameter('x')])
self.assertListEqual([p.id for p in experiment.parameters], [0])
self.assertListEqual(experiment.coordinates, [
Coordinate(4),
Coordinate(8),
Coordinate(16),
Coordinate(32),
Coordinate(64)
])
self.assertListEqual(experiment.metrics, [Metric('time')])
self.assertListEqual(experiment.callpaths, [Callpath('sweep')])
def test_get_matching_hypotheses(self):
modeler = SingleParameterModeler()
modeler.hypotheses_building_blocks.append(CompoundTerm.create(1, 1, 1))
for bb in modeler.get_matching_hypotheses(
[Measurement(Coordinate(15), None, None, 15),
Measurement(Coordinate(0.1), None, None, 0.1)]):
self.assertEqual(len(bb.simple_terms), 1)
self.assertNotEqual(bb.simple_terms[0].term_type, 'logarithm')
hbb = modeler.get_matching_hypotheses(
[Measurement(Coordinate(31), None, None, 31),
Measurement(Coordinate(1), None, None, 1)])
self.assertIn(2, (len(bb.simple_terms) for bb in hbb))
self.assertIn('logarithm', (bb.simple_terms[0].term_type for bb in hbb))
def test_modeling_4p(self):
exponents = [(0, 1, 1), (0, 1, 2), (1, 4, 0), (1, 3, 0), (1, 4, 1),
(1, 3, 1), (1, 4, 2), (1, 3, 2), (1, 2, 0), (1, 2, 1),
(1, 2, 2), (2, 3, 0), (3, 4, 0), (2, 3, 1), (3, 4, 1),
(4, 5, 0), (2, 3, 2), (3, 4, 2), (1, 1, 0), (1, 1, 1),
(1, 1, 2), (5, 4, 0), (5, 4, 1), (4, 3, 0), (4, 3, 1),
(3, 2, 0), (3, 2, 1), (3, 2, 2), (5, 3, 0), (7, 4, 0),
(2, 1, 0), (2, 1, 1), (2, 1, 2), (9, 4, 0), (7, 3, 0),
(5, 2, 0), (5, 2, 1), (5, 2, 2), (8, 3, 0), (11, 4, 0),
(3, 1, 0), (3, 1, 1)]
points = np.array(
list(zip(*itertools.product([2, 4, 8, 10, 12], repeat=4))))
for expo1, expo2, expo3, expo4 in zip(exponents, exponents[1:],
exponents[2:], exponents[3:]):
termX = CompoundTerm.create(*expo1)
termY = CompoundTerm.create(*expo2)
termZ = CompoundTerm.create(*expo3)
termW = CompoundTerm.create(*expo4)
term = MultiParameterTerm((0, termX), (1, termY), (2, termZ),
(3, termW))
term.coefficient = 10
function = MultiParameterFunction(term)
function.constant_coefficient = 20000
values = function.evaluate(points)
measurements = [
Measurement(Coordinate(p), None, None, v)
for p, v in zip(zip(*points), values)
]
modeler = MultiParameterModeler()
models = modeler.model([measurements])
self.assertEqual(1, len(models))
self.assertApproxFunction(function, models[0].hypothesis.function)
def test_modeling(self):
for exponents in [
(0, 1, 1), (0, 1, 2), (1, 4, 0), (1, 3, 0), (1, 4, 1), (1, 3, 1),
(1, 4, 2), (1, 3, 2), (1, 2, 0), (1, 2, 1), (1, 2, 2), (2, 3, 0),
(3, 4, 0), (2, 3, 1), (3, 4, 1), (4, 5, 0), (2, 3, 2), (3, 4, 2),
(1, 1, 0), (1, 1, 1), (1, 1, 2), (5, 4, 0), (5, 4, 1), (4, 3, 0),
(4, 3, 1), (3, 2, 0), (3, 2, 1), (3, 2, 2), (5, 3, 0), (7, 4, 0),
(2, 1, 0), (2, 1, 1), (2, 1, 2), (9, 4, 0), (7, 3, 0), (5, 2, 0),
(5, 2, 1), (5, 2, 2), (8, 3, 0), (11, 4, 0), (3, 1, 0), (3, 1, 1)
]:
term = CompoundTerm.create(*exponents)
term.coefficient = 10
function = SingleParameterFunction(term)
function.constant_coefficient = 200
points = [2, 4, 8, 16, 32]
values = function.evaluate(np.array(points))
measurements = [
Measurement(Coordinate(p), None, None, v)
for p, v in zip(points, values)
]
modeler = SingleParameterModeler()
models = modeler.model([measurements])
self.assertEqual(1, len(models))
self.assertApproxFunction(function, models[0].hypothesis.function)
def test_modeling_plus(self):
exponents = [(0, 1, 1), (0, 1, 2), (1, 4, 0), (1, 3, 0), (1, 4, 1), (1, 3, 1), (1, 4, 2), (1, 3, 2),
(1, 2, 0), (1, 2, 1), (1, 2, 2), (2, 3, 0), (3, 4, 0), (2, 3, 1), (3, 4, 1), (4, 5, 0),
(2, 3, 2), (3, 4, 2), (1, 1, 0), (1, 1, 1), (1, 1, 2), (5, 4, 0), (5, 4, 1), (4, 3, 0),
(4, 3, 1), (3, 2, 0), (3, 2, 1), (3, 2, 2), (5, 3, 0), (7, 4, 0), (2, 1, 0), (2, 1, 1),
(2, 1, 2), (9, 4, 0), (7, 3, 0), (5, 2, 0), (5, 2, 1), (5, 2, 2), (8, 3, 0), (11, 4, 0),
(3, 1, 0), (3, 1, 1)]
for expo1, expo2 in zip(exponents, exponents[1:]):
termX = CompoundTerm.create(*expo1)
termY = CompoundTerm.create(*expo2)
term1 = MultiParameterTerm((0, termX))
term1.coefficient = 10
term2 = MultiParameterTerm((1, termY))
term2.coefficient = 20
function = MultiParameterFunction(term1, term2)
function.constant_coefficient = 200
points = [np.array([2, 4, 8, 16, 32, 2, 4, 8, 16, 32, 2, 4, 8, 16, 32, 2, 4, 8, 16, 32, 2, 4, 8, 16, 32]),
np.array([2, 2, 2, 2, 2, 4, 4, 4, 4, 4, 8, 8, 8, 8, 8, 16, 16, 16, 16, 16, 32, 32, 32, 32, 32])]
values = function.evaluate(np.array(points))
measurements = [Measurement(Coordinate(p), None, None, v) for p, v in zip(zip(*points), values)]
modeler = MultiParameterModeler()
models = modeler.model([measurements])
self.assertEqual(1, len(models))
self.assertApproxFunction(function, models[0].hypothesis.function)
def test_read_2(self):
Parameter.ID_COUNTER = itertools.count()
experiment = read_json_file("data/json/new/input2.json")
x = Parameter('x')
y = Parameter('y')
self.assertListEqual(experiment.parameters, [x, y])
self.assertListEqual(experiment.coordinates, [
Coordinate(4, 10),
Coordinate(8, 20),
Coordinate(16, 30),
Coordinate(32, 40),
Coordinate(64, 50)
])
self.assertListEqual(experiment.metrics,
[Metric('time'), Metric('visits')])
self.assertListEqual(
experiment.callpaths,
[Callpath('sweep'), Callpath('sweep2')])
def test_read_1_json(self):
Parameter.ID_COUNTER = itertools.count()
experiment = read_json_file("data/jsonlines/test1.jsonl")
self.assertListEqual([Parameter('x'), Parameter('y')],
experiment.parameters)
self.assertListEqual([0, 1], [p.id for p in experiment.parameters])
self.assertListEqual([
Coordinate(x, y) for x in range(1, 5 + 1) for y in range(1, 5 + 1)
], experiment.coordinates)
self.assertListEqual([Metric('metr')], experiment.metrics)
self.assertListEqual([Callpath('<root>')], experiment.callpaths)
def read_talpas_file(path, progress_bar=DUMMY_PROGRESS):
# create an experiment object to save the date loaded from the text file
experiment = Experiment()
complete_data = {}
parameters = None
progress_bar.total += os.path.getsize(path)
# read talpas file into complete_data
with open(path) as file:
progress_bar.step('Reading file')
for ln, line in enumerate(file):
progress_bar.update(len(line))
if line.isspace():
continue
line = line.replace(';', ',')
try:
data = json.loads(line)
except JSONDecodeError as error:
raise FileFormatError(
f'Decoding of line {ln} failed: {str(error).replace(",", ";")}. Line: "{line}"'
)
try:
key = Callpath(data['callpath']), Metric(data['metric'])
if parameters is None:
parameters = [
Parameter(p) for p in data['parameters'].keys()
]
coordinate = Coordinate(data['parameters'][p.name]
for p in parameters)
io_helper.append_to_repetition_dict(complete_data, key,
coordinate, data['value'],
progress_bar)
except KeyError as error:
raise FileFormatError(
f'Missing property in line {ln}: {str(error)}. Line: "{line}"'
)
# create experiment
io_helper.repetition_dict_to_experiment(complete_data, experiment,
progress_bar)
for p in parameters:
experiment.add_parameter(p)
call_tree = create_call_tree(experiment.callpaths, progress_bar)
experiment.call_tree = call_tree
io_helper.validate_experiment(experiment, progress_bar)
return experiment
def test_read_2(self):
Parameter.ID_COUNTER = itertools.count()
experiment = read_jsonlines_file("data/jsonlines/test2.jsonl")
self.assertListEqual([Parameter('p'), Parameter('n')],
experiment.parameters)
self.assertListEqual([0, 1], [p.id for p in experiment.parameters])
self.assertListEqual([
Coordinate(x, y) for x in [16, 32, 64, 128, 256]
for y in [100, 200, 300, 400, 500]
], experiment.coordinates)
self.assertListEqual([Metric('metr')], experiment.metrics)
self.assertListEqual([Callpath('<root>')], experiment.callpaths)
def deserialize_coordinate(exp, id_mapping, ioHelper):
id = ioHelper.readId()
length = ioHelper.readInt()
coordinate_parts = [None] * length
for i in range(length):
param = Parameter(ioHelper.readString())
paramIdx = exp.parameters.index(param)
val = ioHelper.readValue()
coordinate_parts[paramIdx] = val
coordinate = Coordinate(*coordinate_parts)
id_mapping.coordinate_mapping[id] = coordinate
return coordinate
def _read_new_json_file(experiment, json_data, progress_bar):
parameter_data = json_data["parameters"]
for p in parameter_data:
parameter = Parameter(p)
experiment.add_parameter(parameter)
measurements_data = json_data["measurements"]
for callpath_name, data in progress_bar(measurements_data.items()):
for metric_name, measurements in data.items():
for measurement in measurements:
coordinate = Coordinate(measurement['point'])
experiment.add_coordinate(coordinate)
callpath = Callpath(callpath_name)
experiment.add_callpath(callpath)
metric = Metric(metric_name)
experiment.add_metric(metric)
measurement = Measurement(coordinate, callpath, metric,
measurement['values'])
experiment.add_measurement(measurement)
def test_3parameters_bands_incomplete(self):
experiment = read_jsonlines_file('data/jsonlines/matrix_3p_bands_incomplete.jsonl')
modeler = MultiParameterModeler()
measurements = experiment.measurements[(Callpath('<root>'), Metric('metr'))]
f_msm = modeler.find_best_measurement_points(measurements)
self.assertEqual(len(f_msm), 3)
self.assertListEqual([m.coordinate for m in f_msm[0]], [
Coordinate(c) for c in [1, 3, 4, 5, 6]
])
self.assertListEqual([0] + [1] * 4, [m.mean for m in f_msm[0]])
self.assertListEqual([m.coordinate for m in f_msm[1]], [
Coordinate(c) for c in range(1, 5 + 1)
])
self.assertListEqual([0] + [2] * 4, [m.mean for m in f_msm[1]])
self.assertListEqual([m.coordinate for m in f_msm[2]], [
Coordinate(c) for c in range(1, 5 + 1)
])
self.assertListEqual([0] + [4] * 4, [m.mean for m in f_msm[2]])
measurements.reverse()
f_msm = modeler.find_best_measurement_points(measurements)
self.assertEqual(len(f_msm), 3)
self.assertListEqual([m.coordinate for m in f_msm[0]], [
Coordinate(c) for c in reversed([1, 3, 4, 5, 6])
])
self.assertListEqual([1] * 4 + [0], [m.mean for m in f_msm[0]])
self.assertListEqual([m.coordinate for m in f_msm[1]], [
Coordinate(c) for c in [6, 5, 4, 3, 2]
])
self.assertListEqual([3] * 5, [m.mean for m in f_msm[1]])
self.assertListEqual([m.coordinate for m in f_msm[2]], [
Coordinate(c) for c in reversed(range(1, 5 + 1))
])
self.assertListEqual([4] * 4 + [0], [m.mean for m in f_msm[2]])
def test_3parameters_sparse(self):
experiment = read_jsonlines_file('data/jsonlines/matrix_3p.jsonl')
modeler = MultiParameterModeler()
measurements = experiment.measurements[(Callpath('<root>'),
Metric('metr'))]
for _ in range(len(measurements)):
shuffle(measurements)
f_msm = modeler.find_best_measurement_points(measurements)
self.assertEqual(len(f_msm), 3)
self.assertSetEqual(
set(m.coordinate for m in f_msm[0]), {
Coordinate(1),
Coordinate(2),
Coordinate(3),
Coordinate(4),
Coordinate(5)
})
self.assertListEqual([1] * 5, [m.mean for m in f_msm[0]])
self.assertSetEqual(
set(m.coordinate for m in f_msm[1]), {
Coordinate(1),
Coordinate(2),
Coordinate(3),
Coordinate(4),
Coordinate(5)
})
self.assertListEqual([1] * 5, [m.mean for m in f_msm[1]])
self.assertSetEqual(
set(m.coordinate for m in f_msm[2]), {
Coordinate(1),
Coordinate(2),
Coordinate(3),
Coordinate(4),
Coordinate(5)
})
self.assertListEqual([1] * 5, [m.mean for m in f_msm[2]])
def test_multi_parameter(self):
experiment = read_cube_file('data/cubeset/multi_parameter', 'weak')
self.assertListEqual(
[Parameter('x'), Parameter('y'),
Parameter('z')], experiment.parameters)
self.assertSetEqual(
{
Coordinate(1, 1, 1),
Coordinate(1, 1, 10),
Coordinate(1, 1, 25),
Coordinate(1, 10, 1),
Coordinate(1, 10, 10),
Coordinate(1, 10, 25),
Coordinate(1, 25, 1),
Coordinate(1, 25, 10),
Coordinate(1, 25, 25),
Coordinate(10, 1, 1),
Coordinate(10, 1, 10),
Coordinate(10, 1, 25),
Coordinate(10, 10, 1),
Coordinate(10, 10, 10),
Coordinate(10, 10, 25),
Coordinate(10, 25, 1),
Coordinate(10, 25, 10),
Coordinate(10, 25, 25),
Coordinate(25, 1, 1),
Coordinate(25, 1, 10),
Coordinate(25, 1, 25),
Coordinate(25, 10, 1),
Coordinate(25, 10, 10),
Coordinate(25, 10, 25),
Coordinate(25, 25, 1),
Coordinate(25, 25, 10),
Coordinate(25, 25, 25)
}, set(experiment.coordinates))
self.assertSetEqual(
{
Callpath('main'),
Callpath('main->init_mat'),
Callpath('main->zero_mat'),
Callpath('main->mat_mul')
}, set(experiment.callpaths))
call_tree = CallTree()
main = Node('main', Callpath('main'))
call_tree.add_child_node(main)
init_mat = Node('init_mat', Callpath('main->init_mat'))
main.add_child_node(init_mat)
zero_mat = Node('zero_mat', Callpath('main->zero_mat'))
main.add_child_node(zero_mat)
mat_mul = Node('mat_mul', Callpath('main->mat_mul'))
main.add_child_node(mat_mul)
self.assertEqual(call_tree, experiment.call_tree)
self.assertSetEqual(
{
Metric('visits'),
Metric('time'),
Metric('min_time'),
Metric('max_time'),
Metric('PAPI_FP_OPS'),
Metric('PAPI_L3_TCM'),
Metric('PAPI_L2_TCM')
}, set(experiment.metrics))
read_cube_file('data/cubeset/multi_parameter', 'strong')
def read_cube_file(dir_name,
scaling_type,
pbar=DUMMY_PROGRESS,
selected_metrics=None):
# read the paths of the cube files in the given directory with dir_name
path = Path(dir_name)
if not path.is_dir():
raise FileFormatError(
f'Cube file path must point to a directory: {dir_name}')
cubex_files = list(path.glob('*/[!.]*.cubex'))
if not cubex_files:
raise FileFormatError(f'No cube files were found in: {dir_name}')
pbar.total += len(cubex_files) + 6
# iterate over all folders and read the cube profiles in them
experiment = Experiment()
pbar.step("Reading cube files")
parameter_names_initial = []
parameter_names = []
parameter_values = []
parameter_dict = defaultdict(set)
progress_step_size = 5 / len(cubex_files)
for path_id, path in enumerate(cubex_files):
pbar.update(progress_step_size)
folder_name = path.parent.name
logging.debug(f"Cube file: {path} Folder: {folder_name}")
# create the parameters
par_start = folder_name.find(".") + 1
par_end = folder_name.find(".r")
par_end = None if par_end == -1 else par_end
parameters = folder_name[par_start:par_end]
# parameters = folder_name.split(".")
# set scaling flag for experiment
if path_id == 0:
if scaling_type == "weak" or scaling_type == "strong":
experiment.scaling = scaling_type
param_list = re.split('([0-9.,]+)', parameters)
param_list.remove("")
parameter_names = [n for i, n in enumerate(param_list) if i % 2 == 0]
parameter_value = [
float(n.replace(',', '.').rstrip('.'))
for i, n in enumerate(param_list) if i % 2 == 1
]
# check if parameter already exists
if path_id == 0:
parameter_names_initial = parameter_names
elif parameter_names != parameter_names_initial:
raise FileFormatError(
f"Parameters must be the same and in the same order: {parameter_names} is not {parameter_names_initial}."
)
for n, v in zip(parameter_names, parameter_value):
parameter_dict[n].add(v)
parameter_values.append(parameter_value)
# determine non-constant parameters and add them to experiment
parameter_selection_mask = []
for i, p in enumerate(parameter_names):
if len(parameter_dict[p]) > 1:
experiment.add_parameter(Parameter(p))
parameter_selection_mask.append(i)
# check number of parameters, if > 1 use weak scaling instead
# since sum values for strong scaling does not work for more than 1 parameter
if scaling_type == 'strong' and len(experiment.parameters) > 1:
warnings.warn(
"Strong scaling only works for one parameter. Using weak scaling instead."
)
scaling_type = 'weak'
experiment.scaling = scaling_type
pbar.step("Reading cube files")
show_warning_skipped_metrics = set()
aggregated_values = defaultdict(list)
# import data from cube files
# optimize import memory usage by reordering files and grouping by coordinate
num_points = 0
reordered_files = sorted(zip(cubex_files, parameter_values),
key=itemgetter(1))
for parameter_value, point_group in groupby(reordered_files,
key=itemgetter(1)):
num_points += 1
# create coordinate
coordinate = Coordinate(parameter_value[i]
for i in parameter_selection_mask)
experiment.add_coordinate(coordinate)
aggregated_values.clear()
for path, _ in point_group:
pbar.update()
with CubexParser(str(path)) as parsed:
callpaths = make_callpath_mapping(parsed.get_root_cnodes())
# iterate over all metrics
for cube_metric in parsed.get_metrics():
pbar.update(0)
# NOTE: here we could choose which metrics to extract
if selected_metrics and cube_metric.name not in selected_metrics:
continue
try:
metric_values = parsed.get_metric_values(
metric=cube_metric, cache=False)
# create the metrics
metric = Metric(cube_metric.name)
for cnode_id in metric_values.cnode_indices:
pbar.update(0)
cnode = parsed.get_cnode(cnode_id)
callpath = callpaths[cnode_id]
# NOTE: here we can use clustering algorithm to select only certain node level values
# create the measurements
cnode_values = metric_values.cnode_values(
cnode, convert_to_exclusive=True)
# in case of weak scaling calculate mean and median over all mpi process values
if scaling_type == "weak":
# do NOT use generator it is slower
aggregated_values[(callpath, metric)].extend(
map(float, cnode_values))
# in case of strong scaling calculate the sum over all mpi process values
elif scaling_type == "strong":
aggregated_values[(callpath, metric)].append(
float(sum(cnode_values)))
# Take care of missing metrics
except MissingMetricError as e: # @UnusedVariable
show_warning_skipped_metrics.add(e.metric.name)
logging.info(
f'The cubex file {Path(*path.parts[-2:])} does not contain data for the metric "{e.metric.name}"'
)
# add measurements to experiment
for (callpath, metric), values in aggregated_values.items():
pbar.update(0)
experiment.add_measurement(
Measurement(coordinate, callpath, metric, values))
pbar.step("Unify calltrees")
to_delete = []
# determine common callpaths for common calltree
# add common callpaths and metrics to experiment
for key, value in pbar(experiment.measurements.items(),
len(experiment.measurements),
scale=0.1):
if len(value) < num_points:
to_delete.append(key)
else:
(callpath, metric) = key
experiment.add_callpath(callpath)
experiment.add_metric(metric)
for key in to_delete:
pbar.update(0)
del experiment.measurements[key]
# determine calltree
call_tree = io_helper.create_call_tree(experiment.callpaths,
pbar,
progress_scale=0.1)
experiment.call_tree = call_tree
if show_warning_skipped_metrics:
warnings.warn(
"The following metrics were skipped because they contained no data: "
f"{', '.join(show_warning_skipped_metrics)}. For more details see log."
)
io_helper.validate_experiment(experiment, pbar)
pbar.update()
return experiment
def read_jsonlines_file(path, progress_bar=DUMMY_PROGRESS):
# create an experiment object to save the date loaded from the text file
experiment = Experiment()
complete_data = {}
parameters = None
default_callpath = Callpath('<root>')
default_metric = Metric('<default>')
progress_bar.total += os.path.getsize(path)
# read jsonlines file into complete_data
with open(path) as file:
progress_bar.step('Reading file')
for ln, line in enumerate(file):
progress_bar.update(len(line))
if line.isspace():
continue
try:
data = json.loads(line)
except JSONDecodeError as error:
raise FileFormatError(
f'Decoding of line {ln} failed: {str(error)}. Line: "{line}"'
)
try:
if 'callpath' in data:
callpath = Callpath(data['callpath'])
else:
callpath = default_callpath
if 'metric' in data:
metric = Metric(data['metric'])
else:
metric = default_metric
key = callpath, metric
if parameters is None: # ensures uniform order of paremeters
parameters = [Parameter(p) for p in data['params'].keys()]
coordinate = Coordinate(data['params'][p.name]
for p in parameters)
io_helper.append_to_repetition_dict(complete_data, key,
coordinate, data['value'],
progress_bar)
except KeyError as error:
raise FileFormatError(
f'Missing property in line {ln}: {str(error)}. Line: "{line}"'
)
# create experiment
io_helper.repetition_dict_to_experiment(complete_data, experiment,
progress_bar)
for p in parameters:
experiment.add_parameter(p)
callpaths = experiment.callpaths
experiment.call_tree = create_call_tree(callpaths, progress_bar)
io_helper.validate_experiment(experiment, progress_bar)
return experiment
def test_compare(self):
points = [4, 8, 16, 32, 64, 128]
data = [
((None, (12.279235119728051 + 112.3997486813747, 0)), [
124.67898380110276, 124.67898380110276, 124.67898380110276,
124.67898380110276, 124.67898380110276, 124.67898380110276
], (None, (124.679, 0.0))),
(((0, Fraction(1, 1)), (392.837968713381, 683.8645895889935)), [
1760.5671478913678, 2444.4317374803613, 3128.296327069355,
3812.1609166583485, 4496.025506247342, 5179.890095836336
], ((0.0, 1.0), (392.838, 683.865))),
(((0, Fraction(2, 1)), (138.69179452369758, 112.44445041582443)), [
588.4695961869953, 1150.6918482661176, 1937.8030011768885,
2949.803054919308, 4186.692009493378, 5648.469864899094
], ((0.0, 2.0), (138.692, 112.444))),
(((Fraction(1, 4), 0), (231.8031252715932, 757.5927278025262)), [
1303.2010356851542, 1505.917143334448, 1746.9885808766455,
2033.672449625761, 2374.5989460987153, 2780.0311613973026
], ((0.25, 0.0), (231.803, 757.593))),
(((Fraction(1, 3), 0), (147.40207355905747, 740.6554582848072)), [
1323.1193271863492, 1628.712990128672, 2013.7368787841829,
2498.836580813641, 3110.023906698286, 3880.071684009308
], ((0.333333, 0.0), (147.402, 740.655))),
(((Fraction(1, 4), Fraction(1, 1)),
(662.1669933486077, 136.57938776640577)), [
1048.4718383883378, 1351.2616987705137, 1754.802095479854,
2286.378790293861, 2979.9960635869884, 3877.9422853175015
], ((0.25, 1.0), (662.167, 136.579))),
(((Fraction(1, 3), Fraction(1, 1)),
(535.6622118860412, 447.75148218635366)), [
1957.1845595719178, 3222.171105004163, 5048.714352111772,
7643.273950315424, 11281.697784358528, 16331.344702676097
], ((0.333333, 1.0), (535.662, 447.751))),
(((Fraction(1, 4), Fraction(2, 1)),
(412.5706706079675, 996.343695251814)), [
6048.741737048133, 15493.363821169984, 32295.568918666017,
59655.521239685964, 101863.64986653095, 164625.65164339435
], ((0.25, 2.0), (412.571, 996.344))),
(((Fraction(1, 3), Fraction(2, 1)),
(93.11229615417925, 20.367438006670188)), [
222.43746622492063, 459.72618027424267, 914.2759402192239,
1709.6769220384463, 3026.0233691146855, 5122.739616052577
], ((0.333333, 2.0), (93.1123, 20.3674))),
(((Fraction(1, 2), 0), (939.8019758412179, 402.94640866510485)), [
1745.6947931714276, 2079.5065279286637, 2551.5876105016373,
3219.2110800161095, 4163.373245162056, 5498.620184191001
], ((0.5, 0.0), (939.802, 402.946))),
(((Fraction(1, 2), Fraction(1, 1)),
(198.49843369241415, 330.31007853365884)), [
1519.7387478270496, 3001.272390797349, 5483.459690230956,
9541.078290708863, 16053.382203308038, 26357.722033338472
], ((0.5, 1.0), (198.498, 330.31))),
(((Fraction(1, 2), Fraction(2, 1)),
(364.8953574839538, 955.112891429775)), [
8005.798488922153, 24678.100241316602, 61492.120408989555,
135438.25582322088, 275437.4080892591, 529852.4683831728
], ((0.5, 2.0), (364.895, 955.113))),
(((Fraction(2, 3), 0), (210.3330694987003, 216.92681699057178)), [
756.9543955249287, 1078.0403374609873, 1587.732499462487,
2396.8183736036135, 3681.1621413478483, 5719.930789353846
], ((0.666667, 0.0), (210.333, 216.927))),
(((Fraction(3, 4), 0), (584.9013580111865, 547.3819137326248)), [
2133.1312104080216, 3188.7032237497583, 4963.956667872185,
7949.565182530901, 12970.740177185866, 21415.31628391976
], ((0.75, 0.0), (584.901, 547.382))),
(((Fraction(2, 3), Fraction(1, 1)),
(953.7431095545323, 838.6830078923111)), [
5180.440612885216, 11017.939204262264, 22254.963733492266,
43220.718142861355, 81467.31186721638, 150062.28747711863
], ((0.666667, 1.0), (953.743, 838.683))),
(((Fraction(3, 4), Fraction(1, 1)),
(355.50475595529707, 203.8586065472728)), [
1508.7031806978325, 3264.666020281983, 6878.980165468027,
14069.422473092825, 28032.266949776153, 54659.84835672009
], ((0.75, 1.0), (355.505, 203.859))),
(((Fraction(4, 5), 0), (836.4136945625079, 988.9778707606993)), [
3834.433979810851, 6056.270190754812, 9924.711720732794,
16660.0596267337, 28386.981453862616, 48804.738258536
], ((0.8, 0.0), (836.414, 988.978))),
(((Fraction(2, 3), Fraction(2, 1)),
(30.370684174349353, 735.0460670350777)), [
7439.17078417381, 26492.029097437142, 74706.39628779484,
185250.37318416082, 423416.90529637906, 914811.6843285251
], ((0.666667, 2.0), (30.3707, 735.046))),
(((Fraction(3, 4), Fraction(2, 1)),
(868.3557741916711, 651.1510359543278)), [
8235.288783790882, 28745.079790529722, 84215.68837634563,
219888.5845432864, 531287.5324653349, 1215054.5573746935
], ((0.75, 2.0), (868.356, 651.151))),
(((Fraction(1, 1), 0), (218.35982887307853, 796.5944762009765)), [
3404.7377336769846, 6591.11563848089, 12963.871448088703,
25709.383067304327, 51200.406305735574, 102182.45278259806
], ((1.0, 0.0), (218.36, 796.594))),
(((Fraction(1, 1), Fraction(1, 1)),
(729.8185276288646, 193.81268721358396)), [
2280.320025337536, 5381.323020754879, 13133.830509298237,
31739.8484818023, 75153.8904176451, 174385.9862710001
], ((1.0, 1.0), (729.819, 193.813))),
(((Fraction(1, 1), Fraction(2, 1)),
(640.8857481060144, 219.18401331861853)), [
4147.829961203911, 16422.13470704655, 56751.993157672354,
175988.09640300085, 505640.8524342031, 1375363.0172824815
], ((1.0, 2.0), (640.886, 219.184))),
(((Fraction(5, 4), 0), (41.41439439883205, 336.0107050284518)), [
1942.1779790139603, 4562.217551923606, 10793.75695530929,
25614.93894716142, 60865.84910208294, 144707.1154351916
], ((1.25, 0.0), (41.4144, 336.011))),
(((Fraction(5, 4), Fraction(1, 1)),
(334.34344019665406, 396.1666489172391)), [
4816.457423065935, 16324.82895624065, 51043.67450160326,
151094.0866783297, 430617.2857956476, 1194290.5953048149
], ((1.25, 1.0), (334.343, 396.167))),
(((Fraction(4, 3), 0), (646.7639733950962, 836.1733802023176)), [
5956.133986838953, 14025.538056632176, 34359.162151911856,
85596.68418849679, 214707.14930518836, 540045.1348296632
], ((1.33333, 0.0), (646.764, 836.173))),
(((Fraction(4, 3), Fraction(1, 1)),
(961.3235324936308, 976.4308028867101)), [
13361.221801905767, 47830.002071055715, 158430.21598980148,
496957.254308979, 1500759.03676648, 4410090.312337113
], ((1.33333, 1.0), (961.324, 976.431))),
(((Fraction(3, 2), 0), (993.5060588040174, 789.8477910359313)), [
7312.288387091468, 18865.72139149913, 51543.76468510362,
143971.22872036492, 405395.57506920083, 1144815.2873512912
], ((1.5, 0.0), (993.506, 789.848))),
(((Fraction(3, 2), Fraction(1, 1)),
(306.3138276450713, 176.2989470011036)), [
3127.096979662729, 12273.883197935773, 45438.844259927595,
159873.90543152104, 541896.6790150353, 1787463.339791056
], ((1.5, 1.0), (306.314, 176.299))),
(((Fraction(3, 2), Fraction(2, 1)),
(623.7521800036756, 545.819243769916)), [
18089.967980640988, 111778.0688886881, 559542.6578003977,
2470719.679039657, 10061164.053347098, 38731727.88533937
], ((1.5, 2.0), (623.752, 545.819))),
(((Fraction(5, 3), 0), (674.515344060474, 93.52470814254)), [
1617.1853318529588, 3667.3060046217547, 10176.033429851634,
30839.954953419994, 96443.81648202146, 304723.0940893776
], ((1.66667, 0.0), (674.515, 93.5247))),
(((Fraction(7, 4), 0), (192.79185213528302, 921.0172026961501)), [
10612.912005989885, 35241.758587692566, 118082.9937972425,
396726.5846888438, 1333968.1715455244, 4486460.534003467
], ((1.75, 0.0), (192.792, 921.017))),
(((Fraction(2, 1), 0), (601.3899361738712, 695.3677746959734)), [
11727.274331309445, 45104.92751671617, 178615.54025834304,
712657.9912248506, 2848827.7950908807, 11393507.010555001
], ((2.0, 0.0), (601.39, 695.368))),
(((Fraction(2, 1), Fraction(1, 1)),
(95.64610607936808, 399.1728717576563)), [
12869.17800232437, 76736.83748354937, 408848.66678591946,
2043860.7495052798, 9810168.14242224, 45780433.96224817
], ((2.0, 1.0), (95.6461, 399.173))),
(((Fraction(2, 1), Fraction(2, 1)),
(910.0933475245264, 649.3303470713025)), [
42467.23556008789, 374924.3732605948, 2660567.19495158,
16623766.97837287, 95748565.7510935, 521293702.00774235
], ((2.0, 2.0), (910.093, 649.33))),
(((Fraction(9, 4), 0), (991.6538373014305, 196.89404893784854)), [
5446.857587036748, 22184.29382918959, 101801.40689347989,
480526.35622160026, 2282055.9737017835, 10851623.32968404
], ((2.25, 0.0), (991.654, 196.894))),
(((Fraction(7, 3), 0), (454.80058623925424, 800.545149328823)), [
20787.379981321064, 102924.57970032863, 516870.1273219162,
2603024.963156712, 13116586.527189683, 66101616.62275291
], ((2.33333, 0.0), (454.801, 800.545))),
(((Fraction(5, 2), 0), (444.6771885244102, 788.914623707213)), [
25689.945147155224, 143253.478519879, 808293.2518647105,
4570326.31979187, 25851599.066826478, 146236657.2404956
], ((2.5, 0.0), (444.677, 788.915))),
(((Fraction(5, 2), Fraction(1, 1)),
(561.6799261002398, 118.03818227888634)), [
8116.123591948965, 64663.26005666098, 484046.0745404187,
3419312.620222673, 23207812.621413387, 153160603.80521256
], ((2.5, 1.0), (561.68, 118.038))),
(((Fraction(5, 2), Fraction(2, 1)),
(98.86684880610103, 982.9958000733947)), [
125922.32925820061, 1601570.0898857696, 16105502.055251304,
142353096.47013444, 1159589128.4318285, 8928380108.544924
], ((2.5, 2.0), (0.0, 982.996))),
(((Fraction(8, 3), 0), (119.32012560773262, 194.108049319692)), [
7945.26627894892, 49810.980751448864, 315641.6975316356,
2003561.5353809511, 12721184.440340932, 80773847.93606871
], ((2.66667, 0.0), (119.32, 194.108))),
(((Fraction(11, 4), 0), (335.21389505653997, 713.4600389668701)), [
32622.72952123845, 217538.8630750938, 1461501.3736992066,
9829850.300849823, 66125167.21631561, 444833408.7346114
], ((2.75, 0.0), (335.214, 713.46))),
(((Fraction(3, 1), 0), (854.0891091206599, 475.68703018220896)), [
31298.059040782035, 244405.84856241164, 1949268.1647354485,
15588166.694119744, 124699354.92919411, 997588860.809789
], ((3.0, 0.0), (854.089, 475.687))),
(((Fraction(3, 1), Fraction(1, 1)),
(498.14812816021515, 788.4374477528575)), [
101418.14144052597, 1211538.0678765494, 12918257.292110976,
129178089.58795632, 1240105375.9704788, 11574312691.156733
], ((3.0, 1.0), (498.148, 788.437))),
]
modeler = SingleParameterModeler()
modeler.use_crossvalidation = False
for orig, values, (exponents, coeff) in data:
if exponents:
term = CompoundTerm.create(*exponents)
term.coefficient = coeff[1]
function = SingleParameterFunction(term)
else:
function = SingleParameterFunction()
function.constant_coefficient = coeff[0]
measurements = [
Measurement(Coordinate(p), None, None, v)
for p, v in zip(points, values)
]
models = modeler.model([measurements])
self.assertEqual(1, len(models))
self.assertApproxFunction(function,
models[0].hypothesis.function,
places=3)
def test_sparse_matrix2p(self):
Parameter.ID_COUNTER = itertools.count()
experiment = read_jsonlines_file(
"data/jsonlines/sparse_matrix_2p.jsonl")
self.assertListEqual([Parameter('x'), Parameter('y')],
experiment.parameters)
self.assertListEqual([0, 1], [p.id for p in experiment.parameters])
self.assertListEqual([
Coordinate(20, 1),
Coordinate(30, 1),
Coordinate(30, 2),
Coordinate(40, 1),
Coordinate(40, 2),
Coordinate(40, 3),
Coordinate(50, 1),
Coordinate(50, 2),
Coordinate(50, 3),
Coordinate(50, 4),
Coordinate(60, 1),
Coordinate(60, 2),
Coordinate(60, 3),
Coordinate(60, 4),
Coordinate(60, 5),
Coordinate(70, 2),
Coordinate(70, 3),
Coordinate(70, 4),
Coordinate(70, 5),
Coordinate(80, 3),
Coordinate(80, 4),
Coordinate(80, 5),
Coordinate(90, 4),
Coordinate(90, 5),
Coordinate(100, 5)
], experiment.coordinates)
self.assertListEqual([Metric('metr')], experiment.metrics)
self.assertListEqual([Callpath('<root>')], experiment.callpaths)
def read_text_file(path, progress_bar=DUMMY_PROGRESS):
# read text file into list
with open(path) as file:
lines = file.readlines()
# remove empty lines
lines_no_space = [l for l in lines if not l.isspace()]
# remove line breaks
lines_no_space = [l.replace("\n", "") for l in lines_no_space]
# create an experiment object to save the date loaded from the text file
experiment = Experiment()
# variables for parsing
number_parameters = 0
last_metric = None
last_callpath = Callpath("")
coordinate_id = 0
if len(lines_no_space) == 0:
raise FileFormatError(f'File contains no data: "{path}"')
# parse text to extrap objects
for i, line in enumerate(progress_bar(lines)):
if line.isspace() or line.startswith('#'):
continue # allow comments
line = re_whitespace.sub(' ', line)
# get field name
field_separator_idx = line.find(" ")
field_name = line[:field_separator_idx]
field_value = line[field_separator_idx + 1:].strip()
if field_name == "METRIC":
# create a new metric if not already exists
metric_name = field_value
test_metric = Metric(metric_name)
if test_metric not in experiment.metrics:
metric = test_metric
experiment.add_metric(metric)
last_metric = metric
else:
last_metric = metric
# reset the coordinate id, since moving to a new region
coordinate_id = 0
elif field_name == "REGION":
# create a new region if not already exists
callpath_name = field_value
callpath = Callpath(callpath_name)
experiment.add_callpath(callpath)
last_callpath = callpath
# reset the coordinate id, since moving to a new region
coordinate_id = 0
elif field_name == "DATA":
if last_metric is None:
last_metric = Metric("")
# create a new data set
data_string = field_value
data_list = data_string.split(" ")
values = [float(d) for d in data_list]
if 1 <= number_parameters <= 4:
# create one measurement per repetition
if coordinate_id >= len(experiment.coordinates):
raise FileFormatError(
f'To many DATA lines ({coordinate_id}) for the number of POINTS '
f'({len(experiment.coordinates)}) in line {i}.')
measurement = Measurement(
experiment.coordinates[coordinate_id], last_callpath,
last_metric, values)
experiment.add_measurement(measurement)
coordinate_id += 1
elif number_parameters >= 5:
raise FileFormatError(
"This input format supports a maximum of 4 parameters.")
else:
raise FileFormatError("This file has no parameters.")
elif field_name == "PARAMETER":
# create a new parameter
parameters = field_value.split(' ')
experiment.parameters += [Parameter(p) for p in parameters]
number_parameters = len(experiment.parameters)
elif field_name == "POINTS":
coordinate_string = field_value.strip()
if '(' in coordinate_string:
coordinate_string = coordinate_string.replace(") (", ")(")
coordinate_string = coordinate_string[1:-1]
coordinate_strings = coordinate_string.split(')(')
else:
coordinate_strings = coordinate_string.split(' ')
# create a new point
if number_parameters == 1:
coordinates = [
Coordinate(float(c)) for c in coordinate_strings
]
experiment.coordinates.extend(coordinates)
elif 1 < number_parameters < 5:
for coordinate_string in coordinate_strings:
coordinate_string = coordinate_string.strip()
values = coordinate_string.split(" ")
coordinate = Coordinate(float(v) for v in values)
experiment.coordinates.append(coordinate)
elif number_parameters >= 5:
raise FileFormatError(
"This input format supports a maximum of 4 parameters.")
else:
raise FileFormatError("This file has no parameters.")
else:
raise FileFormatError(
f'Encountered wrong field: "{field_name}" in line {i}: {line}')
if last_metric == Metric(''):
experiment.metrics.append(last_metric)
if last_metric == Callpath(''):
experiment.callpaths.append(last_callpath)
# create the call tree and add it to the experiment
call_tree = create_call_tree(experiment.callpaths,
progress_bar,
progress_scale=10)
experiment.call_tree = call_tree
io_helper.validate_experiment(experiment, progress_bar)
return experiment
def test_read_12(self):
experiment = read_json_file("data/json/input_12.JSON")
self.assertListEqual(experiment.parameters,
[Parameter('x'), Parameter('y')])
self.assertListEqual(experiment.coordinates, [
Coordinate(4.0, 10.0),
Coordinate(4.0, 20.0),
Coordinate(4.0, 30.0),
Coordinate(4.0, 40.0),
Coordinate(4.0, 50.0),
Coordinate(8.0, 10.0),
Coordinate(8.0, 20.0),
Coordinate(8.0, 30.0),
Coordinate(8.0, 40.0),
Coordinate(8.0, 50.0),
Coordinate(16.0, 10.0),
Coordinate(16.0, 20.0),
Coordinate(16.0, 30.0),
Coordinate(16.0, 40.0),
Coordinate(16.0, 50.0),
Coordinate(32.0, 10.0),
Coordinate(32.0, 20.0),
Coordinate(32.0, 30.0),
Coordinate(32.0, 40.0),
Coordinate(32.0, 50.0),
Coordinate(64.0, 10.0),
Coordinate(64.0, 20.0),
Coordinate(64.0, 30.0),
Coordinate(64.0, 40.0),
Coordinate(64.0, 50.0)
])
def test_3parameters_reversed(self):
experiment = read_text_file('data/text/three_parameter_1.txt')
modeler = MultiParameterModeler()
measurements = experiment.measurements[(Callpath('reg'),
Metric('metr'))]
measurements = list(reversed(measurements))
f_msm = modeler.find_best_measurement_points(measurements)
self.assertEqual(len(f_msm), 3)
self.assertListEqual([m.coordinate for m in f_msm[0]], [
Coordinate(60),
Coordinate(50),
Coordinate(40),
Coordinate(30),
Coordinate(20)
])
self.assertListEqual([m.coordinate for m in f_msm[1]], [
Coordinate(5),
Coordinate(4),
Coordinate(3),
Coordinate(2),
Coordinate(1)
])
self.assertListEqual([m.coordinate for m in f_msm[2]], [
Coordinate(500),
Coordinate(400),
Coordinate(300),
Coordinate(200),
Coordinate(100)
])
def _read_legacy_json_file(experiment, json_data, progress_bar):
# read parameters
parameter_data = json_data["parameters"]
parameter_data = sorted(parameter_data, key=lambda x: x["id"])
logging.debug("Number of parameters: " + str(len(parameter_data)))
for i, p_data in enumerate(progress_bar(parameter_data)):
parameter_name = p_data["name"]
parameter = Parameter(parameter_name)
experiment.add_parameter(parameter)
logging.debug("Parameter " + str(i + 1) + ": " + parameter_name)
# read callpaths
callpath_data = json_data["callpaths"]
callpath_data = sorted(callpath_data, key=lambda x: x["id"])
logging.debug("Number of callpaths: " + str(len(callpath_data)))
for i, c_data in enumerate(progress_bar(callpath_data)):
callpath_name = c_data["name"]
callpath = Callpath(callpath_name)
experiment.add_callpath(callpath)
logging.debug("Callpath " + str(i + 1) + ": " + callpath_name)
# read metrics
metric_data = json_data["metrics"]
metric_data = sorted(metric_data, key=lambda x: x["id"])
logging.debug("Number of metrics: " + str(len(metric_data)))
for i, m_data in enumerate(progress_bar(metric_data)):
metric_name = m_data["name"]
metric = Metric(metric_name)
experiment.add_metric(metric)
logging.debug("Metric " + str(i + 1) + ": " + metric_name)
# read coordinates
coordinate_data = json_data["coordinates"]
coordinate_data = sorted(coordinate_data, key=lambda x: x["id"])
logging.debug("Number of coordinates: " + str(len(coordinate_data)))
for i, c_data in enumerate(progress_bar(coordinate_data)):
parameter_value_pairs = c_data["parameter_value_pairs"]
parameter_value_pairs = sorted(parameter_value_pairs,
key=lambda x: x["parameter_id"])
coordinate = Coordinate(
float(p["parameter_value"]) for p in parameter_value_pairs)
experiment.add_coordinate(coordinate)
logging.debug(f"Coordinate {i + 1}: {coordinate}")
aggregate_data = {}
# read measurements
measurements_data = json_data["measurements"]
logging.debug("Number of measurements: " + str(len(measurements_data)))
for i, m_data in enumerate(progress_bar(measurements_data)):
coordinate_id = int(m_data["coordinate_id"]) - 1
callpath_id = int(m_data["callpath_id"]) - 1
metric_id = int(m_data["metric_id"]) - 1
value = float(m_data["value"])
key = coordinate_id, callpath_id, metric_id
if key in aggregate_data:
aggregate_data[key].append(value)
else:
aggregate_data[key] = [value]
for key in progress_bar(aggregate_data):
coordinate_id, callpath_id, metric_id = key
coordinate = experiment.coordinates[coordinate_id]
callpath = experiment.callpaths[callpath_id]
metric = experiment.metrics[metric_id]
values = aggregate_data[key]
measurement = Measurement(coordinate, callpath, metric, values)
experiment.add_measurement(measurement)
def test_sparse_experiment(self):
experiment = read_extrap3_experiment('data/input/experiment_3_sparse')
self.assertListEqual([Parameter('x'), Parameter('y'), Parameter('z')], experiment.parameters)
self.assertSetEqual({Coordinate(1, 1, 1), Coordinate(1, 1, 10), Coordinate(1, 1, 25),
Coordinate(1, 10, 1), Coordinate(1, 10, 10), Coordinate(1, 10, 25),
Coordinate(1, 25, 1), Coordinate(1, 25, 10), Coordinate(1, 25, 25),
Coordinate(10, 1, 1), Coordinate(10, 1, 10), Coordinate(10, 1, 25),
Coordinate(10, 10, 1), Coordinate(10, 10, 10), Coordinate(10, 10, 25),
Coordinate(10, 25, 1), Coordinate(10, 25, 10), Coordinate(10, 25, 25),
Coordinate(25, 1, 1), Coordinate(25, 1, 10), Coordinate(25, 1, 25),
Coordinate(25, 10, 1), Coordinate(25, 10, 10), Coordinate(25, 10, 25),
Coordinate(25, 25, 1), Coordinate(25, 25, 10), Coordinate(25, 25, 25)
}, set(experiment.coordinates))
self.assertSetEqual({Callpath('main'), Callpath('main->init_mat'), Callpath('main->zero_mat'),
Callpath('main->mat_mul')}, set(experiment.callpaths))
call_tree = CallTree()
main = Node('main', Callpath('main'))
call_tree.add_child_node(main)
init_mat = Node('init_mat', Callpath('main->init_mat'))
main.add_child_node(init_mat)
zero_mat = Node('zero_mat', Callpath('main->zero_mat'))
main.add_child_node(zero_mat)
mat_mul = Node('mat_mul', Callpath('main->mat_mul'))
main.add_child_node(mat_mul)
def test_3parameters_bands(self):
experiment = read_jsonlines_file(
'data/jsonlines/matrix_3p_bands.jsonl')
modeler = MultiParameterModeler()
measurements = experiment.measurements[(Callpath('<root>'),
Metric('metr'))]
f_msm = modeler.find_best_measurement_points(measurements)
self.assertEqual(len(f_msm), 3)
self.assertListEqual([m.coordinate for m in f_msm[0]], [
Coordinate(1),
Coordinate(2),
Coordinate(3),
Coordinate(4),
Coordinate(5)
])
self.assertListEqual([0] + [1] * 4, [m.mean for m in f_msm[0]])
self.assertListEqual([m.coordinate for m in f_msm[1]], [
Coordinate(1),
Coordinate(2),
Coordinate(3),
Coordinate(4),
Coordinate(5)
])
self.assertListEqual([0.5] + [2.5] * 4, [m.mean for m in f_msm[1]])
self.assertListEqual([m.coordinate for m in f_msm[2]], [
Coordinate(1),
Coordinate(2),
Coordinate(3),
Coordinate(4),
Coordinate(5)
])
self.assertListEqual([0] + [4] * 4, [m.mean for m in f_msm[2]])
