def get_weights(experiment_resource: ExperimentResource): """ Get weights using experiment resource. All weight values in the output will be integers. """ if experiment_resource.spec.strategy.testingPattern == TestingPattern.CONFORMANCE: return WeightsAnalysis(data = [], \ message = "weight computation is not applicable to a conformance experiment") versions = [experiment_resource.spec.versionInfo.baseline] versions += experiment_resource.spec.versionInfo.candidates messages = [] # create exploration weights; in fraction # if there are three versions: # exploration_weights = [1/3, 1/3, 1/3] exploration_weights = np.full((len(versions), ), 1.0 / len(versions)) def get_exploitation_weights(): """Create exploitation weights; in fraction if there are three versions: if there are no best versions: exploitation_weights = [1.0, 0, 0], i.e., baseline gets to be exploited if there is a single best version, say, the 2nd version: exploitation_weights = [0, 1.0, 0], i.e., the best version gets exploited if there are two best versions, say, the 2nd and 3rd versions: exploitation_weights = [0, 0.5, 0.5], i.e., best versions get exploited evenly """ exploitation_weights = np.full((len(versions), ), 0.0) try: bvs = experiment_resource.status.analysis.winner_assessment.data.bestVersions assert len(bvs) > 0 messages.append(Message(MessageLevel.INFO, "found best version(s)")) for i, version in enumerate(versions): if version.name in bvs: exploitation_weights[i] = 1 / len(bvs) except (KeyError, AssertionError): exploitation_weights = np.full((len(versions), ), 0.0) exploitation_weights[0] = 1.0 messages.append( Message(MessageLevel.INFO, "no best version(s) found")) return exploitation_weights exploitation_weights = get_exploitation_weights() def get_constrained_weights(input_weights): """ Take input weights in percentage. Apply weight constraints and return modified weights. Example illustrating the inner workings of this function: old_weights = [20, 40, 40] input_weights = [20, 30, 50] maxCandidateWeightIncrement = 10 maxCandidateWeight = 40 after i = 0, constrained_weights = [20, 30, 50] during i = 1 increase = -10 excess = max(0, -10 - 10, 30 - 40) = max(0, -20, -10) = 0 after i = 1, constrained_weights = [20, 30, 50] during i = 2 increase = 10 excess = max(0, 10 - 10, 50 - 40) = 10 after i = 2, constrained_weights = [30, 30, 40] """ # Suppose there are 3 versions. old_weights initialized to [100, 0, 0] old_weights = [100] + ([0] * (len(versions) - 1)) # and then, old_weights are updated to currentWeightDistribution, e.g., [5, 25, 70] if experiment_resource.status.currentWeightDistribution is not None: old_weights = list(map(lambda x: x.value, \ experiment_resource.status.currentWeightDistribution)) logger.debug("Old weights: %s", old_weights) logger.debug("Input weights: %s", input_weights) constrained_weights = input_weights.copy() if experiment_resource.spec.strategy.weights is not None: for i in range(len(versions)): if i == 0: continue # for each candidate, compute excess increase = input_weights[i] - old_weights[i] excess = max(0, \ increase - \ experiment_resource.spec.strategy.weights.maxCandidateWeightIncrement, \ input_weights[i] - experiment_resource.spec.strategy.weights.maxCandidateWeight) # cap candidate weight and add the excess to baseline constrained_weights[i] -= excess constrained_weights[0] += excess logger.debug("Constrained weights: %s", constrained_weights) return constrained_weights # create mix-weight: in fraction ewf = AdvancedParameters.exploration_traffic_percentage / 100.0 # Suppose, ewf = 0.1 (i.e., exploration_traffic_percentage = 10%) # Let exploration_weights = [1/3, 1/3, 1/3] # Let exploitation_weights = [0, 0.5, 0.5] # Then, mix_weights = 0.1 * exploration_weights + 0.9 * exploitation_weights # = 0.1 * [1/3, 1/3, 1/3] + 0.9 * [0, 0.5, 0.5] # = [0.033333, 0.033333, 0.033333] + [0.0, 0.45, 0.45] # = [0.033333, 0.483333, 0.483333] mix_weights = (exploration_weights * ewf) + (exploitation_weights * (1 - ewf)) # create mix-weight: in percent # in the above example, we have mix_weights (in percent) = [3.3333, 48.3333, 48.3333] mix_weights *= 100.0 # apply weight constraints constrained_weights = get_constrained_weights(mix_weights) # perform rounding of weights, so that they sum up to 100 integral_weights = gen_round(constrained_weights, 100) data = [] for version in versions: data.append( VersionWeight(name=version.name, value=next(integral_weights))) _weights = WeightsAnalysis(data=data) _weights.message = Message.join_messages( [Message(MessageLevel.INFO, "all ok")]) logger.debug("weights: %s", pprint.PrettyPrinter().pformat(_weights)) return _weights
def test_experiment_abn_response_objects(): AggregatedMetricsAnalysis(**abn_am_response) VersionAssessmentsAnalysis(**abn_va_response) WinnerAssessmentAnalysis(**abn_wa_response) WeightsAnalysis(**abn_w_response)