def _handle_exception(mtm, e, test_name, child_revision, test_run_id, msg_type, debug): msg = u"{0}\nTest type: {1}\n Exception Name: {2}: {3}".format( traceback.format_exc(), test_name, e.__class__.__name__, unicode(e)) if debug: println(msg, debug) mtm.log_msg(child_revision, test_run_id, msg_type, msg)
def _handle_exception( mtm, e, test_name, child_revision, test_run_id, msg_type, debug ): msg = u"{0}\nTest type: {1}\n Exception Name: {2}: {3}".format( traceback.format_exc(), test_name, e.__class__.__name__, unicode(e) ) if debug: println(msg, debug) mtm.log_msg( child_revision, test_run_id, msg_type, msg )
def check_run_conditions(test_name, rep_count, push_node, branch, replicate_min, debug): """ Test a set of conditions that have to be met in order to run the set of metrics tests. """ #Confirm test name is in the safe test list if not run_test(test_name): println(u"Cannot run {0}".format(test_name), debug) return False if rep_count < replicate_min: #If we don't have more than one replicate we cannot #run any of the existing metric tests println( u"Not enough replicates: {0} rep_count {1}".format( test_name, rep_count), debug) return False if not push_node: #No push node found for this test run #we cannot proceed, need to log this println(u"No push node found", debug) return False if branch in BRANCHES_NOT_ALLOWED: return False return True
def check_run_conditions( test_name, rep_count, push_node, branch, replicate_min, debug ): """ Test a set of conditions that have to be met in order to run the set of metrics tests. """ #Confirm test name is in the safe test list if not run_test(test_name): println(u"Cannot run {0}".format(test_name), debug) return False if rep_count < replicate_min: #If we don't have more than one replicate we cannot #run any of the existing metric tests println( u"Not enough replicates: {0} rep_count {1}".format( test_name, rep_count ), debug ) return False if not push_node: #No push node found for this test run #we cannot proceed, need to log this println(u"No push node found", debug) return False if branch in BRANCHES_NOT_ALLOWED: return False return True
def _run_summary(test_run_id, mtm, plm, child_revision, child_test_data, stored_metric_keys, push_node, debug): println(u"\tStarting _run_summary()", debug) for mkey in stored_metric_keys: child_metrics_data = mtm.get_metrics_data_from_ref_data( child_test_data[mkey]['ref_data'], test_run_id) if not child_metrics_data: println( u"\t\tNo metrics data found for {0} {1}".format( mkey, str(test_run_id)), debug) continue #### #Add push log data to the ref_data structure in child_metric_data #for storage. ### extend_ref_data(child_metrics_data, mkey, push_node) if debug: println( u"\t\tValues passed to metric summary:{0}".format( str(child_metrics_data[mkey]['values'])), debug) summary_results = mtm.run_metric_summary( child_metrics_data[mkey]['ref_data'], child_metrics_data[mkey]['values']) if debug: println(u"\t\tSummary results:{0}".format(summary_results), debug) parent_metrics_data = mtm.get_metrics_data_from_ref_data( child_metrics_data[mkey]['ref_data'], child_metrics_data[mkey]['ref_data']['threshold_test_run_id']) if debug: println( u"\t\tStoring child summary values:{0}".format( str(child_metrics_data[mkey]['values'])), debug) if mkey in parent_metrics_data: if debug: println(u"\t\tMetric datum key found in parent metrics data", debug) println( u"\t\tparent data stored:{0}".format( str(parent_metrics_data[mkey]['values'])), debug) mtm.store_metric_summary_results( child_revision, child_metrics_data[mkey]['ref_data'], summary_results, child_metrics_data[mkey]['values'], child_metrics_data[mkey]['ref_data']['threshold_test_run_id'], parent_metrics_data[mkey]['values']) else: println(u"\t\tMetric datum key NOT found in parent metrics data", debug) mtm.store_metric_summary_results( child_revision, child_metrics_data[mkey]['ref_data'], summary_results, child_metrics_data[mkey]['values'], child_metrics_data[mkey]['ref_data']['threshold_test_run_id'])
def _run_metrics(test_run_id, mtm, plm, child_test_data, pushlog, pushlog_lookup, child_revision, push_node, branch, test_name, debug): """ Run all metrics tests on the test_run_id provided. """ println(u"\tStarting _run_metrics()", debug) stored_metric_keys = [] for mkey in child_test_data: #### #Add push log data to the ref_data structure in child_metric_data #for storage. ### extend_ref_data(child_test_data, mkey, push_node) #Get the threshold data threshold_data = mtm.get_threshold_data( child_test_data[mkey]['ref_data']) if threshold_data: if debug: println(u"\tThreshold data found for metric datum", debug) println(u"\t\tCalling run_metric_method() with:", debug) println( u"\t\tchild values:{0}".format( str(child_test_data[mkey]['values'])), debug) println( u"\t\tthreshold values:{0}".format( str(threshold_data[mkey]['values'])), debug) println( u"\t\tthreshold metric values:{0}".format( str(threshold_data[mkey]['metric_values'])), debug) #Run the metric method try: test_result = mtm.run_metric_method( child_test_data[mkey]['ref_data'], child_test_data[mkey]['values'], threshold_data[mkey]['values'], threshold_data[mkey]['metric_values'], ) except MetricMethodError as e: ### #If we get an exception here, skip the test_run_id ### _handle_exception(mtm, e, test_name, child_revision, test_run_id, compute_test_run_metrics.__name__, debug) continue if debug: #avoid formatting data if possible println(u"\t\tStoring results:{0}".format(str(test_result)), debug) #Store the results mtm.store_metric_results( child_revision, child_test_data[mkey]['ref_data'], test_result, threshold_data[mkey]['ref_data']['test_run_id']) stored_metric_keys.append(mkey) else: #No threshold for the metrics datum #find a parent and initialize. This is #either the first or second time this metrics #datum has been received. # # If it's the first, no parent will be found and # no metric data will be calculated. # # If it's the second, a parent will be found, # a full set of metrics data will be computed, and # the threshold will be set. branch_id = push_node['branch_id'] if branch_id not in pushlog: #Get the last five days of the pushlog for #this branch. pushlog[branch_id] = plm.get_branch_pushlog(branch_id, 5, 0) _build_push_lookup(mtm, branch_id, pushlog, pushlog_lookup) if child_revision in pushlog_lookup[branch_id]: index = pushlog_lookup[branch_id][child_revision] else: #revision is not found in the pushlog, we cannot #proceed without an index at this point println( u"\t\tRevision, {0}, not found in push log skipping". format(child_revision), debug) break if debug: println( u"\t\tSearching pushlog for child: metric datum:{0}". format(mkey), debug) msg = u"\t\tPush log: branch:{0},".format(branch) msg = u"{0} index:{1} push log length:{2}".format( msg, str(index), str(len(pushlog[branch_id]))) println(msg, debug) println( u"\t\tChild values provided:{0}".format( str(child_test_data[mkey]['values'])), debug) #### #Walk the push log looking for a parent that passes the test #for the metric datum. #### parent_data, test_result = mtm.get_parent_test_data( pushlog[branch_id], index, mkey, child_test_data[mkey]['ref_data'], child_test_data[mkey]['values']) if parent_data and test_result: if debug: #avoid formatting data if possible println(u"\t\tParent found in push log", debug) println(u"\t\ttest result:{0}".format(str(test_result)), debug) ##Store the child data## mtm.store_metric_results( child_revision, child_test_data[mkey]['ref_data'], test_result, parent_data['ref_data']['test_run_id']) stored_metric_keys.append(mkey) #The parent metric data needs to also be explicitly #stored, adapt the parent data from the test result parent_test_result = test_result.copy() parent_test_result['stddev1'] = test_result['stddev2'] parent_test_result['mean1'] = test_result['mean2'] parent_test_result['h0_rejected'] = \ test_result['h0_rejected'] parent_test_result['p'] = test_result['p'] parent_test_result['trend_stddev'] = test_result['stddev2'] parent_test_result['trend_mean'] = test_result['mean2'] if debug: #avoid formatting data if possible println( u"\t\tStoring parent data: {0}".format( str(parent_test_result)), debug) mtm.store_metric_results( parent_data['ref_data']['revision'], parent_data['ref_data'], parent_test_result, parent_data['ref_data']['test_run_id']) else: println(u"\t\tNo parent found", debug) return stored_metric_keys
def compute_test_run_metrics(project, pushlog_project, debug, replicate_min, test_run_ids=[]): """ Runs all metric tests and associated summaries on a list of test run ids """ ### #Insure that test_run_ids is iterable, if process objects generates #an error it's possible that test_run_ids will be explicitly set to #None ### test_run_ids = test_run_ids or [] ## #Get test data for test run ids ## plm = PushLogModel(pushlog_project) mtm = MetricsTestModel(project) ##### #We don't know if we need the pushlog, or for what branches #it will be required. Make sure to only retrieve once for each #branch encountered and only when we need it. #### pushlog = {} ##### #This data structure is used to lookup up the index position #of a revision in the push log to start walking from ##### pushlog_lookup = {} for test_run_id in test_run_ids: child_test_data = mtm.get_test_values_by_test_run_id(test_run_id) if not child_test_data: msg = u"No test data available for test run id {0}".format( test_run_id) println(msg, debug) continue first_key = _get_first_mkey(child_test_data) rep_count = len(child_test_data[first_key]['values']) test_name = child_test_data[first_key]['ref_data']['test_name'] child_revision, push_node, branch = _get_revision_and_push_node( plm, child_test_data, first_key) base_message = u"{0} {1}".format(child_revision, str(test_run_id)) if not check_run_conditions(test_name, rep_count, push_node, branch, replicate_min, debug): println(u"Not able to run {0}\n".format(base_message), debug) continue #The test and its replicates pass the run conditions println(u"Running {0}".format(base_message), debug) stored_metric_keys = [] try: stored_metric_keys = _run_metrics(test_run_id, mtm, plm, child_test_data, pushlog, pushlog_lookup, child_revision, push_node, branch, test_name, debug) except Exception as e: _handle_exception(mtm, e, test_name, child_revision, test_run_id, compute_test_run_metrics.__name__, debug) try: _run_summary(test_run_id, mtm, plm, child_revision, child_test_data, stored_metric_keys, push_node, debug) except Exception as e: _handle_exception(mtm, e, test_name, child_revision, test_run_id, compute_test_run_metrics.__name__, debug) println(u"\tProcessing complete for {0}\n".format(base_message), debug) plm.disconnect() mtm.disconnect()
def _run_summary( test_run_id, mtm, plm, child_revision, child_test_data, stored_metric_keys, push_node, debug ): println(u"\tStarting _run_summary()", debug) for mkey in stored_metric_keys: child_metrics_data = mtm.get_metrics_data_from_ref_data( child_test_data[mkey]['ref_data'], test_run_id ) if not child_metrics_data: println( u"\t\tNo metrics data found for {0} {1}".format( mkey, str(test_run_id) ), debug ) continue #### #Add push log data to the ref_data structure in child_metric_data #for storage. ### extend_ref_data(child_metrics_data, mkey, push_node) if debug: println( u"\t\tValues passed to metric summary:{0}".format( str(child_metrics_data[mkey]['values']) ), debug ) summary_results = mtm.run_metric_summary( child_metrics_data[mkey]['ref_data'], child_metrics_data[mkey]['values'] ) if debug: println( u"\t\tSummary results:{0}".format(summary_results), debug ) parent_metrics_data = mtm.get_metrics_data_from_ref_data( child_metrics_data[mkey]['ref_data'], child_metrics_data[mkey]['ref_data']['threshold_test_run_id'] ) if debug: println( u"\t\tStoring child summary values:{0}".format( str(child_metrics_data[mkey]['values']) ), debug ) if mkey in parent_metrics_data: if debug: println( u"\t\tMetric datum key found in parent metrics data", debug ) println( u"\t\tparent data stored:{0}".format( str(parent_metrics_data[mkey]['values']) ), debug ) mtm.store_metric_summary_results( child_revision, child_metrics_data[mkey]['ref_data'], summary_results, child_metrics_data[mkey]['values'], child_metrics_data[mkey]['ref_data']['threshold_test_run_id'], parent_metrics_data[mkey]['values'] ) else: println( u"\t\tMetric datum key NOT found in parent metrics data", debug ) mtm.store_metric_summary_results( child_revision, child_metrics_data[mkey]['ref_data'], summary_results, child_metrics_data[mkey]['values'], child_metrics_data[mkey]['ref_data']['threshold_test_run_id'] )
def _run_metrics( test_run_id, mtm, plm, child_test_data, pushlog, pushlog_lookup, child_revision, push_node, branch, test_name, debug ): """ Run all metrics tests on the test_run_id provided. """ println(u"\tStarting _run_metrics()", debug) stored_metric_keys = [] for mkey in child_test_data: #### #Add push log data to the ref_data structure in child_metric_data #for storage. ### extend_ref_data(child_test_data, mkey, push_node) #Get the threshold data threshold_data = mtm.get_threshold_data( child_test_data[mkey]['ref_data'] ) if threshold_data: if debug: println(u"\tThreshold data found for metric datum", debug) println(u"\t\tCalling run_metric_method() with:", debug) println( u"\t\tchild values:{0}".format( str(child_test_data[mkey]['values']) ), debug ) println( u"\t\tthreshold values:{0}".format( str(threshold_data[mkey]['values']) ), debug ) println( u"\t\tthreshold metric values:{0}".format( str(threshold_data[mkey]['metric_values']) ), debug ) #Run the metric method try: test_result = mtm.run_metric_method( child_test_data[mkey]['ref_data'], child_test_data[mkey]['values'], threshold_data[mkey]['values'], threshold_data[mkey]['metric_values'], ) except MetricMethodError as e: ### #If we get an exception here, skip the test_run_id ### _handle_exception( mtm, e, test_name, child_revision, test_run_id, compute_test_run_metrics.__name__, debug ) continue if debug: #avoid formatting data if possible println( u"\t\tStoring results:{0}".format(str(test_result)), debug ) #Store the results mtm.store_metric_results( child_revision, child_test_data[mkey]['ref_data'], test_result, threshold_data[mkey]['ref_data']['test_run_id'] ) stored_metric_keys.append(mkey) else: #No threshold for the metrics datum #find a parent and initialize. This is #either the first or second time this metrics #datum has been received. # # If it's the first, no parent will be found and # no metric data will be calculated. # # If it's the second, a parent will be found, # a full set of metrics data will be computed, and # the threshold will be set. branch_id = push_node['branch_id'] if branch_id not in pushlog: #Get the last five days of the pushlog for #this branch. pushlog[ branch_id ] = plm.get_branch_pushlog( branch_id, 5, 0 ) _build_push_lookup( mtm, branch_id, pushlog, pushlog_lookup ) if child_revision in pushlog_lookup[branch_id]: index = pushlog_lookup[branch_id][child_revision] else: #revision is not found in the pushlog, we cannot #proceed without an index at this point println( u"\t\tRevision, {0}, not found in push log skipping".format(child_revision), debug ) break if debug: println( u"\t\tSearching pushlog for child: metric datum:{0}".format(mkey), debug ) msg = u"\t\tPush log: branch:{0},".format(branch) msg = u"{0} index:{1} push log length:{2}".format( msg, str(index), str(len(pushlog[ branch_id ])) ) println(msg, debug) println( u"\t\tChild values provided:{0}".format( str(child_test_data[mkey]['values']) ), debug ) #### #Walk the push log looking for a parent that passes the test #for the metric datum. #### parent_data, test_result = mtm.get_parent_test_data( pushlog[ branch_id ], index, mkey, child_test_data[mkey]['ref_data'], child_test_data[mkey]['values'] ) if parent_data and test_result: if debug: #avoid formatting data if possible println(u"\t\tParent found in push log", debug) println( u"\t\ttest result:{0}".format( str(test_result) ), debug ) ##Store the child data## mtm.store_metric_results( child_revision, child_test_data[mkey]['ref_data'], test_result, parent_data['ref_data']['test_run_id'] ) stored_metric_keys.append(mkey) #The parent metric data needs to also be explicitly #stored, adapt the parent data from the test result parent_test_result = test_result.copy() parent_test_result['stddev1'] = test_result['stddev2'] parent_test_result['mean1'] = test_result['mean2'] parent_test_result['h0_rejected'] = \ test_result['h0_rejected'] parent_test_result['p'] = test_result['p'] parent_test_result['trend_stddev'] = test_result['stddev2'] parent_test_result['trend_mean'] = test_result['mean2'] if debug: #avoid formatting data if possible println( u"\t\tStoring parent data: {0}".format( str(parent_test_result) ), debug ) mtm.store_metric_results( parent_data['ref_data']['revision'], parent_data['ref_data'], parent_test_result, parent_data['ref_data']['test_run_id'] ) else: println(u"\t\tNo parent found", debug) return stored_metric_keys
def compute_test_run_metrics( project, pushlog_project, debug, replicate_min, test_run_ids=[] ): """ Runs all metric tests and associated summaries on a list of test run ids """ ### #Insure that test_run_ids is iterable, if process objects generates #an error it's possible that test_run_ids will be explicitly set to #None ### test_run_ids = test_run_ids or [] ## #Get test data for test run ids ## plm = PushLogModel(pushlog_project) mtm = MetricsTestModel(project) ##### #We don't know if we need the pushlog, or for what branches #it will be required. Make sure to only retrieve once for each #branch encountered and only when we need it. #### pushlog = {} ##### #This data structure is used to lookup up the index position #of a revision in the push log to start walking from ##### pushlog_lookup = {} for test_run_id in test_run_ids: child_test_data = mtm.get_test_values_by_test_run_id(test_run_id) if not child_test_data: msg = u"No test data available for test run id {0}".format( test_run_id ) println(msg, debug) continue first_key = _get_first_mkey(child_test_data) rep_count = len(child_test_data[first_key]['values']) test_name = child_test_data[first_key]['ref_data']['test_name'] child_revision, push_node, branch = _get_revision_and_push_node( plm, child_test_data, first_key ) base_message = u"{0} {1}".format(child_revision, str(test_run_id)) if not check_run_conditions( test_name, rep_count, push_node, branch, replicate_min, debug ): println(u"Not able to run {0}\n".format(base_message), debug) continue #The test and its replicates pass the run conditions println(u"Running {0}".format(base_message), debug) stored_metric_keys = [] try: stored_metric_keys = _run_metrics( test_run_id, mtm, plm, child_test_data, pushlog, pushlog_lookup, child_revision, push_node, branch, test_name, debug ) except Exception as e: _handle_exception( mtm, e, test_name, child_revision, test_run_id, compute_test_run_metrics.__name__, debug ) try: _run_summary( test_run_id, mtm, plm, child_revision, child_test_data, stored_metric_keys, push_node, debug ) except Exception as e: _handle_exception( mtm, e, test_name, child_revision, test_run_id, compute_test_run_metrics.__name__, debug ) println( u"\tProcessing complete for {0}\n".format(base_message), debug ) plm.disconnect() mtm.disconnect()