def start_factorial_tests(wf): experiment_id = wf.id key = wf.analysis["data_type"] # key = "overhead" if key is not None: try: stage_ids, samples, knobs = get_tuples(experiment_id=experiment_id, step_no=wf.step_no, key=key) test = FactorialAnova(stage_ids=stage_ids, y_key=key, knob_keys=None, stages_count=len(stage_ids)) aov_table, aov_table_sqr = test.run(data=samples, knobs=knobs) # before saving and merging tables, extract useful information aov_table = delete_combination_notation(aov_table) aov_table_sqr = delete_combination_notation(aov_table_sqr) # type(dd) is DefaultOrderedDict # keys = [exploration_percentage, route_random_sigma, exploration_percentage,route_random_sigma...] # resultDict e.g. {'PR(>F)': 0.0949496951695454, 'F': 2.8232330924997346 ... dod = iterate_anova_tables(aov_table=aov_table, aov_table_sqr=aov_table_sqr) # from now on, caller functions should fetch result from DB db().save_analysis(experiment_id=experiment_id, step_no=wf.step_no, analysis_name=test.name, anova_result=dod, knobs=knobs) return True except Exception as e: print("error in factorial tests, while performing anova") tb = traceback.format_exc() error(tb) return False else: error("data type for anova is not properly provided") return False
def get_all_stage_data(experiment_id, timestamp=None): steps_and_stages = sc.StageController.get(experiment_id=experiment_id) new_tuples = dict() # we get step_no as keys, stage_numbers as inner keys for step_no, step in steps_and_stages.items(): new_tuples[step_no] = dict() for stage in step: if stage['number'] != "best": new_tuples[step_no][stage['number']] = dict() if timestamp: data_points = db().get_data_points_after( experiment_id=experiment_id, step_no=step_no, stage_no=stage['number'], timestamp=timestamp) else: data_points = db().get_data_points( experiment_id=experiment_id, step_no=step_no, stage_no=stage['number']) # wrap the stage data with stage number if there are some data points if len(data_points) != 0: stage_and_data = { "knobs": stage["knobs"], "number": stage["number"], "step_name": stage["step_name"], "values": data_points } # also append stage_result if there are any if "stage_result" in stage: stage_and_data["stage_result"] = stage["stage_result"] new_tuples[step_no][stage['number']] = stage_and_data return new_tuples
def start_two_sample_tests(wf): experiment_id = wf.id alpha = wf.analysis["tTestAlpha"] key = wf.analysis["data_type"] mean_diff = 0.1 # as in crowdnav-elastic-ttest-sample-size/definition.py # TODO: get it from user ?? # this part will test this way of t-test: Default --> Best configuration stage_ids, samples, knobs = get_tuples(experiment_id=experiment_id, step_no=wf.step_no, key=key) test1 = Ttest(stage_ids=stage_ids, y_key=key, alpha=alpha) result = test1.run(data=samples, knobs=knobs) # prepare default & optimal knobs to save to analysis stage_no = 1 knobs = {} for tpl in wf.execution_strategy["knobs"]: knobs[stage_no] = tpl stage_no += 1 db().save_analysis(experiment_id=experiment_id, step_no=wf.step_no, analysis_name=test1.name, result=result, knobs=knobs) # if we want to integrate following tests, they should be saved as another step_no, just increment it before saving # x1 = samples[0] # x2 = samples[1] # pooled_std = sqrt((np.var(x1) + np.var(x2)) / 2) # effect_size = mean_diff / pooled_std # effect_size = wf.analysis["tTestEffectSize"] # test2 = TtestPower(stage_ids=stage_ids, y_key=key, effect_size=effect_size) # result2 = test2.run(data=samples, knobs=knobs) # db().save_analysis(experiment_id=experiment_id, step_no=wf.step_no, analysis_name=test2.name, result=result2) # # test3 = TtestSampleSizeEstimation(stage_ids=stage_ids, y_key=key, effect_size=None, mean_diff=mean_diff) # result3 = test3.run(data=samples, knobs=knobs) # db().save_analysis(experiment_id=experiment_id, step_no=wf.step_no, analysis_name=test3.name, result=result3) return result
def start_n_sample_tests(wf): id = wf.id alpha = wf.analysis["alpha"] key = wf.analysis["data_type"] stage_ids, samples, knobs = get_tuples(experiment_id=id, step_no=wf.step_no, key=key) test = OneWayAnova(stage_ids=stage_ids, y_key=key, alpha=alpha) result = test.run(data=samples, knobs=knobs) db().save_analysis(experiment_id=id, step_no=wf.step_no, analysis_name=test.name, result=result) test = KruskalWallis(stage_ids=stage_ids, y_key=key, alpha=alpha) result = test.run(data=samples, knobs=knobs) db().save_analysis(experiment_id=id, step_no=wf.step_no, analysis_name=test.name, result=result) test = Levene(stage_ids=stage_ids, y_key=key, alpha=alpha) result = test.run(data=samples, knobs=knobs) db().save_analysis(experiment_id=id, step_no=wf.step_no, analysis_name=test.name, result=result) test = Bartlett(stage_ids=stage_ids, y_key=key, alpha=alpha) result = test.run(data=samples, knobs=knobs) db().save_analysis(experiment_id=id, step_no=wf.step_no, analysis_name=test.name, result=result) test = FlignerKilleen(stage_ids=stage_ids, y_key=key, alpha=alpha) result = test.run(data=samples, knobs=knobs) db().save_analysis(experiment_id=id, step_no=wf.step_no, analysis_name=test.name, result=result) return
def get_tuples(experiment_id, step_no, key): stage_ids = db().get_stages(experiment_id, step_no)[0] data, knobs = db().get_data_for_analysis(experiment_id, step_no) extract_inner_values(key=key, stage_ids=stage_ids, data=data) # parse data & knobs (k-v pairs) to a proper array of values samples = [data[stage_id] for stage_id in stage_ids] knobs = [knobs[stage_id] for stage_id in stage_ids] return stage_ids, samples, knobs
def test_i_create_target_system(self): target_system = create_target_system( data_providers=IntegrationTest.data_providers, default_variables=IntegrationTest.knobs, ignore_first_n_samples=30) self.assertTrue(target_system) db().save_target(target_system) IntegrationTest.target_system = target_system
def get_best_configurations(target_system, sample_size, y_key): passed_experiments = [] not_passed_experiments = [] anova_passed = 0 anova_failed = 0 total = 0 experiment_ids = db().get_experiments()[0] # experiment_ids = [experiment_ids[0]] for exp_id in experiment_ids: experiment = db().get_experiment(exp_id) steps_and_stages = sc.get(experiment_id=exp_id) last_step_number = steps_and_stages.keys()[-1] originalEffectSize = experiment["analysis"]["tTestEffectSize"] considered_data_type = experiment["considered_data_types"][0] direction = considered_data_type["criteria"] targetSystemId = experiment["targetSystemId"] ts = db().get_target(targetSystemId) if str(target_system ) in ts["name"] and experiment["executionStrategy"][ "sample_size"] == sample_size and considered_data_type[ "name"] == y_key: try: anova = db().get_analysis(exp_id, 1, "two-way-anova") anova_eligible_for_bogp = anova["eligible_for_next_step"] if anova_eligible_for_bogp: anova_passed += 1 t_test = db().get_analysis(exp_id, last_step_number, "t-test") t_test_result = t_test["result"] different_averages = t_test_result["different_averages"] effect_size = t_test_result["effect_size"] if direction == 'Maximize': # significant and effect size is enough if effect_size < -1.0 * originalEffectSize and different_averages: passed_experiments.append( (experiment, t_test, last_step_number)) else: not_passed_experiments.append( (experiment, t_test, last_step_number)) else: if effect_size > originalEffectSize and different_averages: passed_experiments.append( (experiment, t_test, last_step_number)) else: not_passed_experiments.append( (experiment, t_test, last_step_number)) total += 1 except: anova_failed += 1 if target_system == "CrowdNav": calculate_results_and_flush(not_passed_experiments, target_system, y_key, sample_size) else: calculate_results_and_flush(passed_experiments, target_system, y_key, sample_size)
def secondary_data_reducer(new_data, wf, idx): db().save_data_point(experiment_id=wf.id, step_no=wf.step_no, stage_no=wf.stage_counter, data_point_count=wf.secondary_data_counter, secondary_data_provider_index=idx, payload=new_data) wf.secondary_data_counter += 1 return wf
def get(): try: if db() is not None: db().clear_db() return {"message": "Database is cleared successfully"}, 200 except Exception as e: tb = traceback.format_exc() print(tb) return {"error": e.message}, 404
def test_r_FlignerKilleen(self): stage_ids, samples, knobs = AnalysisTest.get_data_for_tests() test = FlignerKilleen(stage_ids=stage_ids, y_key=AnalysisTest.key) result = test.run(data=samples, knobs=knobs) self.assertTrue(result) for key in result: self.assertTrue(result[key] is not None) db().save_analysis(AnalysisTest.experiment_id, AnalysisTest.stage_ids, test.name, result) retrieved = db().get_analysis(AnalysisTest.experiment_id, AnalysisTest.stage_ids, test.name) self.assertTrue(retrieved)
def evaluator(wf): # do the actual calculation of output variable (y) result = RTXDefinition.calculate_result(wf) info("---------------- result") info(result) db().update_stage(experiment_id=wf.id, step_no=wf.step_no, stage_no=wf.stage_counter, field="stage_result", value=result) wf.stage_counter += 1 # this must be done after all calculations of a single stage return result
def primary_data_reducer(new_data, wf): db().save_data_point(experiment_id=wf.id, step_no=wf.step_no, stage_no=wf.stage_counter, data_point_count=wf.primary_data_counter, secondary_data_provider_index=None, payload=new_data) wf.primary_data_counter += 1 if wf._oeda_stop_request.isSet(): raise RuntimeError( "Experiment interrupted from OEDA while reducing primary data." ) return wf
def post(experiment_id): content = request.get_json() new_knobs = {} for knob in content["executionStrategy"]["knobs"]: if content["executionStrategy"]["type"] == 'step_explorer': new_knobs[knob[0]] = [knob[1], knob[2], knob[3]] else: new_knobs[knob[0]] = [knob[1], knob[2]] content["executionStrategy"]["knobs"] = new_knobs db().save_experiment(experiment_id, content) # here we refresh the status of oeda callback, too set_dict(None, experiment_id) return {}, 200
def put(target_id): try: if target_id is None: return {"message": "target_id should not be null"}, 404 content = request.get_json() status = content["status"] db().update_target_system_status(target_id, status) resp = jsonify({"message": "Target system status is updated"}) resp.status_code = 200 return resp except Exception as e: tb = traceback.format_exc() print(tb) return {"message": e.message}, 404
def test_j_shapiro(self): test = ShapiroWilk(AnalysisTest.experiment_id, AnalysisTest.key, alpha=0.05) result = test.run(data=AnalysisTest.data[AnalysisTest.stage_id], knobs=AnalysisTest.knobs[AnalysisTest.stage_id]) self.assertTrue(result) for key in result: self.assertTrue(result[key] is not None) db().save_analysis(AnalysisTest.experiment_id, AnalysisTest.stage_id, test.name, result) retrieved = db().get_analysis(AnalysisTest.experiment_id, AnalysisTest.stage_id, test.name) self.assertTrue(retrieved)
def calculate_result(wf): data_type = wf.considered_data_types[0] data_type_name = data_type["name"] data_type_aggregate_function = str(data_type['aggregateFunction']) if data_type["scale"] == "Boolean": # now data_type_aggregate_function is either count-True or count-False field_value = data_type_aggregate_function.split("-")[ 1] # fetch value # we store them in binary, not in True/False if field_value == 'True': field_value = 1 else: field_value = 0 count = db().get_count(wf.id, wf.step_no, wf.stage_counter, data_type_name, field_value) total = db().get_aggregation(wf.id, wf.step_no, wf.stage_counter, "stats", data_type_name)["count"] value = float(count) / total else: if 'percentiles' in data_type_aggregate_function: # we have percentiles-25, percentiles-50 etc and parse it to use percentiles as outer aggregate_function aggregate_function, percentile_number = data_type_aggregate_function.split( "-") values = db().get_aggregation(wf.id, wf.step_no, wf.stage_counter, aggregate_function, data_type_name) value = values[str(float(percentile_number))] else: aggregate_function = "stats" if data_type_aggregate_function in [ 'sum_of_squares', 'variance', 'std_deviation' ]: aggregate_function = "extended_stats" values = db().get_aggregation(wf.id, wf.step_no, wf.stage_counter, aggregate_function, data_type_name) value = values[ data_type_aggregate_function] # retrieve exact value from response if value is not None and isnan(float(value)) is False: # maximization criteria before calculating the result if data_type["criteria"] == "Maximize": value = take_inverse(value) info("data_type_name: " + data_type_name + " value: " + str(value)) return value else: error("data_type_name: " + data_type_name + " value: 0") return 0
def test_m_TtestSampleSizeEstimation(self): stage_ids, samples, knobs = AnalysisTest.get_data_for_tests() test = TtestSampleSizeEstimation(stage_ids=stage_ids, y_key=AnalysisTest.key, effect_size=None, mean_diff=AnalysisTest.mean_diff) result = test.run(data=samples, knobs=knobs) self.assertTrue(result) for key in result: self.assertTrue(result[key] is not None) db().save_analysis(AnalysisTest.experiment_id, AnalysisTest.stage_ids, test.name, result) retrieved = db().get_analysis(AnalysisTest.experiment_id, AnalysisTest.stage_ids, test.name) self.assertTrue(retrieved)
def get(experiment_id): numberOfSteps = db().get_experiment( experiment_id=experiment_id)["numberOfSteps"] steps_and_stages = {} # step numbers always start from 1, not 0. and we should pass numberOfSteps + 1 to range fcn for step_no in range(1, numberOfSteps + 1): stage_ids, stages = db().get_stages(experiment_id=experiment_id, step_no=step_no) new_stages = stages i = 0 for _ in stages: new_stages[i]["id"] = stage_ids[i] i += 1 steps_and_stages[step_no] = new_stages return steps_and_stages
def test_g_anderson(self): test = AndersonDarling(AnalysisTest.experiment_id, AnalysisTest.key, alpha=0.05) result = test.run(data=AnalysisTest.data[AnalysisTest.stage_id], knobs=AnalysisTest.knobs[AnalysisTest.stage_id]) self.assertTrue(result) for key in result: self.assertTrue( result[key] is not None ) # we used this instead of assertTrue(test[k]) because value can be False db().save_analysis(AnalysisTest.experiment_id, AnalysisTest.stage_id, test.name, result) retrieved = db().get_analysis(AnalysisTest.experiment_id, AnalysisTest.stage_id, test.name) self.assertTrue(retrieved)
def test_c_data_points(self): data, knobs = db().get_data_for_analysis(AnalysisTest.experiment_id, AnalysisTest.stage_no) self.assertTrue(data) self.assertTrue(knobs) AnalysisTest.data = data AnalysisTest.knobs = knobs
def test_k_execution(self): executed_workflow = rtx_execution( experiment=IntegrationTest.experiment, target=IntegrationTest.target_system) self.assertTrue(executed_workflow) target_status = db().get_target( IntegrationTest.target_system["id"])["status"] self.assertEqual(target_status, "READY") experiment_status = db().get_experiment( IntegrationTest.experiment["id"])["status"] self.assertEqual(experiment_status, "SUCCESS") self.stage_test() self.data_point_test() self.analysis_test() if IntegrationTest.analysis_tests_included is False: self.delete_index_test()
def test_j_create_experiment(self): experiment = create_experiment_with_mlr_mbo( "mlr_mbo", sample_size=20, knobs=IntegrationTest.knobs, considered_data_types=IntegrationTest.considered_data_types, analysis=IntegrationTest.analysis, optimizer_iterations_in_design=len(IntegrationTest.knobs) * 4, acquisition_method="ei", optimizer_iterations=5) self.assertTrue(experiment) self.assertTrue(experiment["id"]) experiment["targetSystemId"] = IntegrationTest.target_system["id"] db().save_experiment(experiment) saved_experiment = db().get_experiment(experiment["id"]) self.assertTrue(saved_experiment) IntegrationTest.experiment = experiment
def get(): ids, experiments = db().get_experiments() new_experiments = experiments i = 0 for _ in experiments: new_experiments[i]["id"] = ids[i] i += 1 return new_experiments
def data_point_test(self): for stage_id in IntegrationTest.stage_ids: self.assertTrue(stage_id) data_points = db().get_data_points( IntegrationTest.experiment["id"], stage_id) for point in data_points: self.assertTrue(point["payload"]) self.assertTrue(point["createdDate"])
def get(): ids, targets = db().get_targets() new_targets = targets i = 0 for _ in targets: new_targets[i]["id"] = ids[i] i += 1 new_targets = sorted(new_targets, key=lambda x: x["name"]) return new_targets
def get(experiment_id): stage_ids, stages = db().get_stages(experiment_id) new_stages = stages i = 0 for _ in stages: new_stages[i]["id"] = stage_ids[i] new_stages[i]["knobs"] = _["knobs"] i += 1 return new_stages
def test_l_TtestPower(self): stage_ids, samples, knobs = AnalysisTest.get_data_for_tests() x1 = samples[0] x2 = samples[1] pooled_std = sqrt((np.var(x1) + np.var(x2)) / 2) effect_size = AnalysisTest.mean_diff / pooled_std test = TtestPower(stage_ids=stage_ids, y_key=AnalysisTest.key, effect_size=effect_size) result = test.run(data=samples, knobs=knobs) self.assertTrue(result) for key in result: self.assertTrue(result[key] is not None) db().save_analysis(AnalysisTest.experiment_id, AnalysisTest.stage_ids, test.name, result) retrieved = db().get_analysis(AnalysisTest.experiment_id, AnalysisTest.stage_ids, test.name) self.assertTrue(retrieved)
def post(target_id): try: content = request.get_json() # check if given name is unique ids, targets = db().get_targets() for target in targets: if target['name'] == content['name']: return {"message": "Duplicate target system names"}, 409 content["status"] = "READY" new_target = db().save_target(target_id, content) new_target["id"] = target_id return new_target except Exception as e: tb = traceback.format_exc() print(tb) return {"error": e.message}, 404
def get(): ids, targets = db().get_targets() new_targets = targets i = 0 for _ in targets: new_targets[i]["id"] = ids[i] i += 1 return new_targets
def primary_data_reducer(state, new_data, wf): cnt = state["data_points"] db().save_data_point(new_data, cnt, wf.id, wf.stage_counter) state["overhead"] = (state["overhead"] * cnt + new_data["overhead"]) / (cnt + 1) state["data_points"] += 1 # for data_type in incomingDataTypes: # data_type_name = data_type["name"] # data_type_count = data_type_name + "_cnt" # cnt = state[data_type_count] # db().save_data_point(new_data, cnt, wf.id, wf.stage_counter) # state[data_type_name] = (state[data_type_name] * cnt + new_data[data_type_name]) / (cnt + 1) # state[data_type_count] += 1 if wf._oeda_stop_request.isSet(): raise RuntimeError( "Experiment interrupted from OEDA while gathering data.") return state