def _prepare_data(data, reduce_rows=1000):
durations = []
idle_durations = []
atomic_durations = {}
num_errors = 0
for i in data["result"]:
# TODO(maretskiy): store error value and scenario output
if i["error"]:
num_errors += 1
durations.append(i["duration"])
idle_durations.append(i["idle_duration"])
for met, duration in i["atomic_actions"].items():
try:
atomic_durations[met].append(duration)
except KeyError:
atomic_durations[met] = [duration]
for k, v in atomic_durations.items():
atomic_durations[k] = utils.compress(v, limit=reduce_rows)
return {
"total_durations": {
"duration": utils.compress(durations, limit=reduce_rows),
"idle_duration": utils.compress(idle_durations,
limit=reduce_rows)},
"atomic_durations": atomic_durations,
"num_errors": num_errors,
}
def test_compress(self):
data64 = range(64)
data4 = [4, 2, 1, 3]
mixed = [2, "5", None, 0.5]
alt_normalize = str
alt_merge = lambda a, b: str(a) + str(b)
compress = lambda lst: [(k + 1, float(v)) for k, v in enumerate(lst)]
# Long list
self.assertEqual(utils.compress(data64), compress(data64))
self._compare_items_lists(utils.compress(data64, limit=4),
[(17, 15.0), (33, 31.01), (49, 47.0),
(64, 62.0)])
self.assertEqual(
utils.compress(data64,
limit=4,
normalize=alt_normalize,
merge=alt_merge),
[(17, '012345678910111213141516'),
(33, '17181920212223242526272829303132'),
(49, '33343536373839404142434445464748'),
(64, '495051525354555657585960616263')])
# Short list
self.assertEqual(utils.compress(data4, limit=2), [(3, 2.0), (4, 3.0)])
self.assertEqual(utils.compress(data4, normalize=alt_normalize),
[(1, '4'), (2, '2'), (3, '1'), (4, '3')])
# List with mixed data types
self.assertEqual(utils.compress(mixed), [(1, 2.0), (2, 5.0), (3, 0.0),
(4, 0.5)])
self.assertEqual(utils.compress(mixed, normalize=str), [(1, '2'),
(2, '5'),
(3, 'None'),
(4, '0.5')])
self.assertRaises(TypeError, utils.compress, mixed, normalize=int)
self.assertEqual(
utils.compress(mixed, normalize=alt_normalize, merge=alt_merge),
[(1, '2'), (2, '5'), (3, 'None'), (4, '0.5')])
def test_compress(self):
data64 = range(64)
data4 = [4, 2, 1, 3]
mixed = [2, "5", None, 0.5]
alt_normalize = str
alt_merge = lambda a, b: str(a) + str(b)
compress = lambda lst: [(k + 1, float(v)) for k, v in enumerate(lst)]
# Long list
self.assertEqual(utils.compress(data64), compress(data64))
self._compare_items_lists(
utils.compress(data64, limit=4),
[(17, 15.0), (33, 31.01), (49, 47.0), (64, 62.0)])
self.assertEqual(
utils.compress(data64, limit=4,
normalize=alt_normalize, merge=alt_merge),
[(17, "012345678910111213141516"),
(33, "17181920212223242526272829303132"),
(49, "33343536373839404142434445464748"),
(64, "495051525354555657585960616263")])
# Short list
self.assertEqual(utils.compress(data4, limit=2),
[(3, 2.0), (4, 3.0)])
self.assertEqual(utils.compress(data4, normalize=alt_normalize),
[(1, "4"), (2, "2"), (3, "1"), (4, "3")])
# List with mixed data types
self.assertEqual(utils.compress(mixed),
[(1, 2.0), (2, 5.0), (3, 0.0), (4, 0.5)])
self.assertEqual(utils.compress(mixed, normalize=str),
[(1, "2"), (2, "5"), (3, "None"), (4, "0.5")])
self.assertRaises(TypeError, utils.compress, mixed, normalize=int)
self.assertEqual(
utils.compress(mixed, normalize=alt_normalize, merge=alt_merge),
[(1, "2"), (2, "5"), (3, "None"), (4, "0.5")])
def _prepare_data(data):
durations = []
idle_durations = []
atomic_durations = {}
output = {}
output_errors = []
output_stacked = []
errors = []
# NOTE(maretskiy): We need this extra iteration
# to determine something that we should know about the data
# before starting its processing.
atomic_names = set()
output_names = set()
for r in data["result"]:
atomic_names.update(r["atomic_actions"].keys())
output_names.update(r["scenario_output"]["data"].keys())
for idx, r in enumerate(data["result"]):
# NOTE(maretskiy): Sometimes we miss iteration data.
# So we care about data integrity by setting zero values
if len(r["atomic_actions"]) < len(atomic_names):
for atomic_name in atomic_names:
r["atomic_actions"].setdefault(atomic_name, 0)
if len(r["scenario_output"]["data"]) < len(output_names):
for output_name in output_names:
r["scenario_output"]["data"].setdefault(output_name, 0)
if r["scenario_output"]["errors"]:
output_errors.append((idx, r["scenario_output"]["errors"]))
for param, value in r["scenario_output"]["data"].items():
try:
output[param].append(value)
except KeyError:
output[param] = [value]
if r["error"]:
type_, message, traceback = r["error"]
errors.append({"iteration": idx,
"type": type_,
"message": message,
"traceback": traceback})
# NOTE(maretskiy): Reset failed durations (no sense to display)
r["duration"] = 0
r["idle_duration"] = 0
durations.append(r["duration"])
idle_durations.append(r["idle_duration"])
for met, duration in r["atomic_actions"].items():
try:
atomic_durations[met].append(duration)
except KeyError:
atomic_durations[met] = [duration]
for k, v in six.iteritems(output):
output_stacked.append({"key": k, "values": utils.compress(v)})
for k, v in six.iteritems(atomic_durations):
atomic_durations[k] = utils.compress(v)
return {
"total_durations": {
"duration": utils.compress(durations),
"idle_duration": utils.compress(idle_durations)},
"atomic_durations": atomic_durations,
"output": output_stacked,
"output_errors": output_errors,
"errors": errors,
"sla": data["sla"],
"load_duration": data["load_duration"],
"full_duration": data["full_duration"],
}
def _prepare_data(data):
durations = []
idle_durations = []
atomic_durations = {}
output = {}
output_errors = []
output_stacked = []
errors = []
# NOTE(maretskiy): We need this extra iteration
# to determine something that we should know about the data
# before starting its processing.
atomic_names = set()
output_names = set()
for r in data["result"]:
atomic_names.update(r["atomic_actions"].keys())
output_names.update(r["scenario_output"]["data"].keys())
for idx, r in enumerate(data["result"]):
# NOTE(maretskiy): Sometimes we miss iteration data.
# So we care about data integrity by setting zero values
if len(r["atomic_actions"]) < len(atomic_names):
for atomic_name in atomic_names:
r["atomic_actions"].setdefault(atomic_name, 0)
if len(r["scenario_output"]["data"]) < len(output_names):
for output_name in output_names:
r["scenario_output"]["data"].setdefault(output_name, 0)
if r["scenario_output"]["errors"]:
output_errors.append((idx, r["scenario_output"]["errors"]))
for param, value in r["scenario_output"]["data"].items():
try:
output[param].append(value)
except KeyError:
output[param] = [value]
if r["error"]:
type_, message, traceback = r["error"]
errors.append({
"iteration": idx,
"type": type_,
"message": message,
"traceback": traceback
})
# NOTE(maretskiy): Reset failed durations (no sense to display)
r["duration"] = 0
r["idle_duration"] = 0
durations.append(r["duration"])
idle_durations.append(r["idle_duration"])
for met, duration in r["atomic_actions"].items():
try:
atomic_durations[met].append(duration)
except KeyError:
atomic_durations[met] = [duration]
for k, v in six.iteritems(output):
output_stacked.append({"key": k, "values": utils.compress(v)})
for k, v in six.iteritems(atomic_durations):
atomic_durations[k] = utils.compress(v)
return {
"total_durations": {
"duration": utils.compress(durations),
"idle_duration": utils.compress(idle_durations)
},
"atomic_durations": atomic_durations,
"output": output_stacked,
"output_errors": output_errors,
"errors": errors,
"sla": data["sla"],
"load_duration": data["load_duration"],
"full_duration": data["full_duration"],
}
