def start_factorial_tests(wf):
experiment_id = wf.id
key = wf.analysis["data_type"]
# key = "overhead"
if key is not None:
try:
stage_ids, samples, knobs = get_tuples(experiment_id=experiment_id, step_no=wf.step_no, key=key)
test = FactorialAnova(stage_ids=stage_ids, y_key=key, knob_keys=None, stages_count=len(stage_ids))
aov_table, aov_table_sqr = test.run(data=samples, knobs=knobs)
# before saving and merging tables, extract useful information
aov_table = delete_combination_notation(aov_table)
aov_table_sqr = delete_combination_notation(aov_table_sqr)
# type(dd) is DefaultOrderedDict
# keys = [exploration_percentage, route_random_sigma, exploration_percentage,route_random_sigma...]
# resultDict e.g. {'PR(>F)': 0.0949496951695454, 'F': 2.8232330924997346 ...
dod = iterate_anova_tables(aov_table=aov_table, aov_table_sqr=aov_table_sqr)
# from now on, caller functions should fetch result from DB
db().save_analysis(experiment_id=experiment_id, step_no=wf.step_no, analysis_name=test.name, anova_result=dod, knobs=knobs)
return True
except Exception as e:
print("error in factorial tests, while performing anova")
tb = traceback.format_exc()
error(tb)
return False
else:
error("data type for anova is not properly provided")
return False
def get_all_stage_data(experiment_id, timestamp=None):
steps_and_stages = sc.StageController.get(experiment_id=experiment_id)
new_tuples = dict()
# we get step_no as keys, stage_numbers as inner keys
for step_no, step in steps_and_stages.items():
new_tuples[step_no] = dict()
for stage in step:
if stage['number'] != "best":
new_tuples[step_no][stage['number']] = dict()
if timestamp:
data_points = db().get_data_points_after(
experiment_id=experiment_id,
step_no=step_no,
stage_no=stage['number'],
timestamp=timestamp)
else:
data_points = db().get_data_points(
experiment_id=experiment_id,
step_no=step_no,
stage_no=stage['number'])
# wrap the stage data with stage number if there are some data points
if len(data_points) != 0:
stage_and_data = {
"knobs": stage["knobs"],
"number": stage["number"],
"step_name": stage["step_name"],
"values": data_points
}
# also append stage_result if there are any
if "stage_result" in stage:
stage_and_data["stage_result"] = stage["stage_result"]
new_tuples[step_no][stage['number']] = stage_and_data
return new_tuples
def start_two_sample_tests(wf):
experiment_id = wf.id
alpha = wf.analysis["tTestAlpha"]
key = wf.analysis["data_type"]
mean_diff = 0.1 # as in crowdnav-elastic-ttest-sample-size/definition.py # TODO: get it from user ??
# this part will test this way of t-test: Default --> Best configuration
stage_ids, samples, knobs = get_tuples(experiment_id=experiment_id, step_no=wf.step_no, key=key)
test1 = Ttest(stage_ids=stage_ids, y_key=key, alpha=alpha)
result = test1.run(data=samples, knobs=knobs)
# prepare default & optimal knobs to save to analysis
stage_no = 1
knobs = {}
for tpl in wf.execution_strategy["knobs"]:
knobs[stage_no] = tpl
stage_no += 1
db().save_analysis(experiment_id=experiment_id, step_no=wf.step_no, analysis_name=test1.name, result=result, knobs=knobs)
# if we want to integrate following tests, they should be saved as another step_no, just increment it before saving
# x1 = samples[0]
# x2 = samples[1]
# pooled_std = sqrt((np.var(x1) + np.var(x2)) / 2)
# effect_size = mean_diff / pooled_std
# effect_size = wf.analysis["tTestEffectSize"]
# test2 = TtestPower(stage_ids=stage_ids, y_key=key, effect_size=effect_size)
# result2 = test2.run(data=samples, knobs=knobs)
# db().save_analysis(experiment_id=experiment_id, step_no=wf.step_no, analysis_name=test2.name, result=result2)
#
# test3 = TtestSampleSizeEstimation(stage_ids=stage_ids, y_key=key, effect_size=None, mean_diff=mean_diff)
# result3 = test3.run(data=samples, knobs=knobs)
# db().save_analysis(experiment_id=experiment_id, step_no=wf.step_no, analysis_name=test3.name, result=result3)
return result
def start_n_sample_tests(wf):
id = wf.id
alpha = wf.analysis["alpha"]
key = wf.analysis["data_type"]
stage_ids, samples, knobs = get_tuples(experiment_id=id, step_no=wf.step_no, key=key)
test = OneWayAnova(stage_ids=stage_ids, y_key=key, alpha=alpha)
result = test.run(data=samples, knobs=knobs)
db().save_analysis(experiment_id=id, step_no=wf.step_no, analysis_name=test.name, result=result)
test = KruskalWallis(stage_ids=stage_ids, y_key=key, alpha=alpha)
result = test.run(data=samples, knobs=knobs)
db().save_analysis(experiment_id=id, step_no=wf.step_no, analysis_name=test.name, result=result)
test = Levene(stage_ids=stage_ids, y_key=key, alpha=alpha)
result = test.run(data=samples, knobs=knobs)
db().save_analysis(experiment_id=id, step_no=wf.step_no, analysis_name=test.name, result=result)
test = Bartlett(stage_ids=stage_ids, y_key=key, alpha=alpha)
result = test.run(data=samples, knobs=knobs)
db().save_analysis(experiment_id=id, step_no=wf.step_no, analysis_name=test.name, result=result)
test = FlignerKilleen(stage_ids=stage_ids, y_key=key, alpha=alpha)
result = test.run(data=samples, knobs=knobs)
db().save_analysis(experiment_id=id, step_no=wf.step_no, analysis_name=test.name, result=result)
return
def get_tuples(experiment_id, step_no, key):
stage_ids = db().get_stages(experiment_id, step_no)[0]
data, knobs = db().get_data_for_analysis(experiment_id, step_no)
extract_inner_values(key=key, stage_ids=stage_ids, data=data)
# parse data & knobs (k-v pairs) to a proper array of values
samples = [data[stage_id] for stage_id in stage_ids]
knobs = [knobs[stage_id] for stage_id in stage_ids]
return stage_ids, samples, knobs
def test_i_create_target_system(self):
target_system = create_target_system(
data_providers=IntegrationTest.data_providers,
default_variables=IntegrationTest.knobs,
ignore_first_n_samples=30)
self.assertTrue(target_system)
db().save_target(target_system)
IntegrationTest.target_system = target_system
def get_best_configurations(target_system, sample_size, y_key):
passed_experiments = []
not_passed_experiments = []
anova_passed = 0
anova_failed = 0
total = 0
experiment_ids = db().get_experiments()[0]
# experiment_ids = [experiment_ids[0]]
for exp_id in experiment_ids:
experiment = db().get_experiment(exp_id)
steps_and_stages = sc.get(experiment_id=exp_id)
last_step_number = steps_and_stages.keys()[-1]
originalEffectSize = experiment["analysis"]["tTestEffectSize"]
considered_data_type = experiment["considered_data_types"][0]
direction = considered_data_type["criteria"]
targetSystemId = experiment["targetSystemId"]
ts = db().get_target(targetSystemId)
if str(target_system
) in ts["name"] and experiment["executionStrategy"][
"sample_size"] == sample_size and considered_data_type[
"name"] == y_key:
try:
anova = db().get_analysis(exp_id, 1, "two-way-anova")
anova_eligible_for_bogp = anova["eligible_for_next_step"]
if anova_eligible_for_bogp:
anova_passed += 1
t_test = db().get_analysis(exp_id, last_step_number,
"t-test")
t_test_result = t_test["result"]
different_averages = t_test_result["different_averages"]
effect_size = t_test_result["effect_size"]
if direction == 'Maximize':
# significant and effect size is enough
if effect_size < -1.0 * originalEffectSize and different_averages:
passed_experiments.append(
(experiment, t_test, last_step_number))
else:
not_passed_experiments.append(
(experiment, t_test, last_step_number))
else:
if effect_size > originalEffectSize and different_averages:
passed_experiments.append(
(experiment, t_test, last_step_number))
else:
not_passed_experiments.append(
(experiment, t_test, last_step_number))
total += 1
except:
anova_failed += 1
if target_system == "CrowdNav":
calculate_results_and_flush(not_passed_experiments, target_system,
y_key, sample_size)
else:
calculate_results_and_flush(passed_experiments, target_system, y_key,
sample_size)
def secondary_data_reducer(new_data, wf, idx):
db().save_data_point(experiment_id=wf.id,
step_no=wf.step_no,
stage_no=wf.stage_counter,
data_point_count=wf.secondary_data_counter,
secondary_data_provider_index=idx,
payload=new_data)
wf.secondary_data_counter += 1
return wf
def get():
try:
if db() is not None:
db().clear_db()
return {"message": "Database is cleared successfully"}, 200
except Exception as e:
tb = traceback.format_exc()
print(tb)
return {"error": e.message}, 404
def test_r_FlignerKilleen(self):
stage_ids, samples, knobs = AnalysisTest.get_data_for_tests()
test = FlignerKilleen(stage_ids=stage_ids, y_key=AnalysisTest.key)
result = test.run(data=samples, knobs=knobs)
self.assertTrue(result)
for key in result:
self.assertTrue(result[key] is not None)
db().save_analysis(AnalysisTest.experiment_id, AnalysisTest.stage_ids,
test.name, result)
retrieved = db().get_analysis(AnalysisTest.experiment_id,
AnalysisTest.stage_ids, test.name)
self.assertTrue(retrieved)
def evaluator(wf):
# do the actual calculation of output variable (y)
result = RTXDefinition.calculate_result(wf)
info("---------------- result")
info(result)
db().update_stage(experiment_id=wf.id,
step_no=wf.step_no,
stage_no=wf.stage_counter,
field="stage_result",
value=result)
wf.stage_counter += 1 # this must be done after all calculations of a single stage
return result
def primary_data_reducer(new_data, wf):
db().save_data_point(experiment_id=wf.id,
step_no=wf.step_no,
stage_no=wf.stage_counter,
data_point_count=wf.primary_data_counter,
secondary_data_provider_index=None,
payload=new_data)
wf.primary_data_counter += 1
if wf._oeda_stop_request.isSet():
raise RuntimeError(
"Experiment interrupted from OEDA while reducing primary data."
)
return wf
def post(experiment_id):
content = request.get_json()
new_knobs = {}
for knob in content["executionStrategy"]["knobs"]:
if content["executionStrategy"]["type"] == 'step_explorer':
new_knobs[knob[0]] = [knob[1], knob[2], knob[3]]
else:
new_knobs[knob[0]] = [knob[1], knob[2]]
content["executionStrategy"]["knobs"] = new_knobs
db().save_experiment(experiment_id, content)
# here we refresh the status of oeda callback, too
set_dict(None, experiment_id)
return {}, 200
def put(target_id):
try:
if target_id is None:
return {"message": "target_id should not be null"}, 404
content = request.get_json()
status = content["status"]
db().update_target_system_status(target_id, status)
resp = jsonify({"message": "Target system status is updated"})
resp.status_code = 200
return resp
except Exception as e:
tb = traceback.format_exc()
print(tb)
return {"message": e.message}, 404
def test_j_shapiro(self):
test = ShapiroWilk(AnalysisTest.experiment_id,
AnalysisTest.key,
alpha=0.05)
result = test.run(data=AnalysisTest.data[AnalysisTest.stage_id],
knobs=AnalysisTest.knobs[AnalysisTest.stage_id])
self.assertTrue(result)
for key in result:
self.assertTrue(result[key] is not None)
db().save_analysis(AnalysisTest.experiment_id, AnalysisTest.stage_id,
test.name, result)
retrieved = db().get_analysis(AnalysisTest.experiment_id,
AnalysisTest.stage_id, test.name)
self.assertTrue(retrieved)
def calculate_result(wf):
data_type = wf.considered_data_types[0]
data_type_name = data_type["name"]
data_type_aggregate_function = str(data_type['aggregateFunction'])
if data_type["scale"] == "Boolean":
# now data_type_aggregate_function is either count-True or count-False
field_value = data_type_aggregate_function.split("-")[
1] # fetch value
# we store them in binary, not in True/False
if field_value == 'True':
field_value = 1
else:
field_value = 0
count = db().get_count(wf.id, wf.step_no, wf.stage_counter,
data_type_name, field_value)
total = db().get_aggregation(wf.id, wf.step_no, wf.stage_counter,
"stats", data_type_name)["count"]
value = float(count) / total
else:
if 'percentiles' in data_type_aggregate_function:
# we have percentiles-25, percentiles-50 etc and parse it to use percentiles as outer aggregate_function
aggregate_function, percentile_number = data_type_aggregate_function.split(
"-")
values = db().get_aggregation(wf.id, wf.step_no,
wf.stage_counter,
aggregate_function,
data_type_name)
value = values[str(float(percentile_number))]
else:
aggregate_function = "stats"
if data_type_aggregate_function in [
'sum_of_squares', 'variance', 'std_deviation'
]:
aggregate_function = "extended_stats"
values = db().get_aggregation(wf.id, wf.step_no,
wf.stage_counter,
aggregate_function,
data_type_name)
value = values[
data_type_aggregate_function] # retrieve exact value from response
if value is not None and isnan(float(value)) is False:
# maximization criteria before calculating the result
if data_type["criteria"] == "Maximize":
value = take_inverse(value)
info("data_type_name: " + data_type_name + " value: " + str(value))
return value
else:
error("data_type_name: " + data_type_name + " value: 0")
return 0
def test_m_TtestSampleSizeEstimation(self):
stage_ids, samples, knobs = AnalysisTest.get_data_for_tests()
test = TtestSampleSizeEstimation(stage_ids=stage_ids,
y_key=AnalysisTest.key,
effect_size=None,
mean_diff=AnalysisTest.mean_diff)
result = test.run(data=samples, knobs=knobs)
self.assertTrue(result)
for key in result:
self.assertTrue(result[key] is not None)
db().save_analysis(AnalysisTest.experiment_id, AnalysisTest.stage_ids,
test.name, result)
retrieved = db().get_analysis(AnalysisTest.experiment_id,
AnalysisTest.stage_ids, test.name)
self.assertTrue(retrieved)
def get(experiment_id):
numberOfSteps = db().get_experiment(
experiment_id=experiment_id)["numberOfSteps"]
steps_and_stages = {}
# step numbers always start from 1, not 0. and we should pass numberOfSteps + 1 to range fcn
for step_no in range(1, numberOfSteps + 1):
stage_ids, stages = db().get_stages(experiment_id=experiment_id,
step_no=step_no)
new_stages = stages
i = 0
for _ in stages:
new_stages[i]["id"] = stage_ids[i]
i += 1
steps_and_stages[step_no] = new_stages
return steps_and_stages
def test_g_anderson(self):
test = AndersonDarling(AnalysisTest.experiment_id,
AnalysisTest.key,
alpha=0.05)
result = test.run(data=AnalysisTest.data[AnalysisTest.stage_id],
knobs=AnalysisTest.knobs[AnalysisTest.stage_id])
self.assertTrue(result)
for key in result:
self.assertTrue(
result[key] is not None
) # we used this instead of assertTrue(test[k]) because value can be False
db().save_analysis(AnalysisTest.experiment_id, AnalysisTest.stage_id,
test.name, result)
retrieved = db().get_analysis(AnalysisTest.experiment_id,
AnalysisTest.stage_id, test.name)
self.assertTrue(retrieved)
def test_c_data_points(self):
data, knobs = db().get_data_for_analysis(AnalysisTest.experiment_id,
AnalysisTest.stage_no)
self.assertTrue(data)
self.assertTrue(knobs)
AnalysisTest.data = data
AnalysisTest.knobs = knobs
def test_k_execution(self):
executed_workflow = rtx_execution(
experiment=IntegrationTest.experiment,
target=IntegrationTest.target_system)
self.assertTrue(executed_workflow)
target_status = db().get_target(
IntegrationTest.target_system["id"])["status"]
self.assertEqual(target_status, "READY")
experiment_status = db().get_experiment(
IntegrationTest.experiment["id"])["status"]
self.assertEqual(experiment_status, "SUCCESS")
self.stage_test()
self.data_point_test()
self.analysis_test()
if IntegrationTest.analysis_tests_included is False:
self.delete_index_test()
def test_j_create_experiment(self):
experiment = create_experiment_with_mlr_mbo(
"mlr_mbo",
sample_size=20,
knobs=IntegrationTest.knobs,
considered_data_types=IntegrationTest.considered_data_types,
analysis=IntegrationTest.analysis,
optimizer_iterations_in_design=len(IntegrationTest.knobs) * 4,
acquisition_method="ei",
optimizer_iterations=5)
self.assertTrue(experiment)
self.assertTrue(experiment["id"])
experiment["targetSystemId"] = IntegrationTest.target_system["id"]
db().save_experiment(experiment)
saved_experiment = db().get_experiment(experiment["id"])
self.assertTrue(saved_experiment)
IntegrationTest.experiment = experiment
def get():
ids, experiments = db().get_experiments()
new_experiments = experiments
i = 0
for _ in experiments:
new_experiments[i]["id"] = ids[i]
i += 1
return new_experiments
def data_point_test(self):
for stage_id in IntegrationTest.stage_ids:
self.assertTrue(stage_id)
data_points = db().get_data_points(
IntegrationTest.experiment["id"], stage_id)
for point in data_points:
self.assertTrue(point["payload"])
self.assertTrue(point["createdDate"])
def get():
ids, targets = db().get_targets()
new_targets = targets
i = 0
for _ in targets:
new_targets[i]["id"] = ids[i]
i += 1
new_targets = sorted(new_targets, key=lambda x: x["name"])
return new_targets
def get(experiment_id):
stage_ids, stages = db().get_stages(experiment_id)
new_stages = stages
i = 0
for _ in stages:
new_stages[i]["id"] = stage_ids[i]
new_stages[i]["knobs"] = _["knobs"]
i += 1
return new_stages
def test_l_TtestPower(self):
stage_ids, samples, knobs = AnalysisTest.get_data_for_tests()
x1 = samples[0]
x2 = samples[1]
pooled_std = sqrt((np.var(x1) + np.var(x2)) / 2)
effect_size = AnalysisTest.mean_diff / pooled_std
test = TtestPower(stage_ids=stage_ids,
y_key=AnalysisTest.key,
effect_size=effect_size)
result = test.run(data=samples, knobs=knobs)
self.assertTrue(result)
for key in result:
self.assertTrue(result[key] is not None)
db().save_analysis(AnalysisTest.experiment_id, AnalysisTest.stage_ids,
test.name, result)
retrieved = db().get_analysis(AnalysisTest.experiment_id,
AnalysisTest.stage_ids, test.name)
self.assertTrue(retrieved)
def post(target_id):
try:
content = request.get_json()
# check if given name is unique
ids, targets = db().get_targets()
for target in targets:
if target['name'] == content['name']:
return {"message": "Duplicate target system names"}, 409
content["status"] = "READY"
new_target = db().save_target(target_id, content)
new_target["id"] = target_id
return new_target
except Exception as e:
tb = traceback.format_exc()
print(tb)
return {"error": e.message}, 404
def get():
ids, targets = db().get_targets()
new_targets = targets
i = 0
for _ in targets:
new_targets[i]["id"] = ids[i]
i += 1
return new_targets
def primary_data_reducer(state, new_data, wf):
cnt = state["data_points"]
db().save_data_point(new_data, cnt, wf.id, wf.stage_counter)
state["overhead"] = (state["overhead"] * cnt +
new_data["overhead"]) / (cnt + 1)
state["data_points"] += 1
# for data_type in incomingDataTypes:
# data_type_name = data_type["name"]
# data_type_count = data_type_name + "_cnt"
# cnt = state[data_type_count]
# db().save_data_point(new_data, cnt, wf.id, wf.stage_counter)
# state[data_type_name] = (state[data_type_name] * cnt + new_data[data_type_name]) / (cnt + 1)
# state[data_type_count] += 1
if wf._oeda_stop_request.isSet():
raise RuntimeError(
"Experiment interrupted from OEDA while gathering data.")
return state
