def run(self, **kwargs): input_matrix = [] key_list = [] data_to_test = kwargs[DATA_FIELD] min_length = 99999999999 for key, val in data_to_test.iteritems(): points = list(val[0].get_points()) values = self._post_effect_output(points, val[1]) if len(values) == 0: log.error("No data points for measurement: " + str(key)) continue log.debug("Last value in series:" + str(values[-1])) log.info(str(key) + " has length " + str(len(values))) min_length = min(len(values), min_length) input_matrix.append(values) key_list.append(key) for values in input_matrix: if min_length < len(values): differ = len(values) - min_length if 0 < differ <= 5: values = values[:-differ] continue raise ValueError("Measurements have different lengths!") log.info("Counting Common factor... from " + str(len(key_list)) + " dimensions.") # Move to Common Factor! TODO: # corr_matrix = np.corrcoef(input_matrix) corrmat = np.corrcoef(input_matrix) corrmat = np.nan_to_num(corrmat) log.info(corrmat) eigenvalues, _ = np.linalg.eig(corrmat) eigenvalues = filter(lambda x: True if x > 1 else False, eigenvalues) eigenvalues = [x / len(key_list) for x in eigenvalues] input_matrix = np.array(input_matrix) input_matrix = np.transpose(input_matrix) factor = FactorAnalysis(n_components=len(eigenvalues)).fit(input_matrix) log.info(factor.components_) return { CfFinisher.KEY_FOR_DATA: self._prepare_data(key_list, eigenvalues, factor.components_), CfFinisher.KEY_NAME: "factors", }
def run(self): log.info("Starting Serenity Test Runner") for test_case in self.test_cases: if 'skip' in test_case.keys(): continue for i in xrange(test_case['iterations']): log.info("-------- Running " + test_case['name'] + " iter: " + \ str(i) + "/" + str(test_case['iterations'])) engine = PipelineEngine(test_case['workflow']) (result, exception) = engine.run(test_case['input'][i]) if result != 0: log.error("-------- TEST FAILED: " + str(exception)) else: log.info("-------- TEST SUCCESS") return 0
def run(self, **kwargs): input_matrix = [] key_list = [] data_to_test = kwargs[DATA_FIELD] min_length = 99999999999 for key, val in data_to_test.iteritems(): points = list(val[0].get_points()) values = self._post_effect_output(points, val[1]) if len(values) == 0: log.error("No data points for measurement: " + str(key)) continue log.debug("Last value in series:" + str(values[-1])) log.info(str(key) + " has length " + str(len(values))) min_length = min(len(values), min_length) input_matrix.append(values) key_list.append(key) for values in input_matrix: if min_length < len(values): differ = len(values) - min_length if 0 < differ <= 5: log.debug(len(values)) values = values[:-differ] log.debug(len(values)) continue raise ValueError("Measurements have different lengths!") log.info('Counting PCA... from ' + str(len(key_list)) + " dimensions.") corr_matrix = np.corrcoef(input_matrix) log.info(corr_matrix) return {PcaFinisher.KEY_FOR_DATA: self._prepare_data(key_list, corr_matrix.tolist()), PcaFinisher.KEY_NAME: 'correlations'}
def run(self, serializedUsage): """ Runs every task from workflow As parameter we expect ResourceUsage.proto from mesos.proto :return: Status of workflow execution """ usage = ResourceUsage() try: usage.ParseFromString(serializedUsage) except Exception as e: log.error("Exception occurred while parsing usage: " + str(e)) return 1, e queue = Queue.Queue() queue.put(self._first_task) run_res = (0, None) while not queue.empty(): task = queue.get() try: if task.input is None: result = task.run(usage=usage) else: result = task.run(**task.input) for t in task.next_success: t.input = result queue.put(t) except Exception as e: import traceback log.error(traceback.format_exc()) log.error("Exception occurred while execution tasks: " + str(e)) run_res = (1, e) for t in task.next_error: t.input = {"error": e} queue.put(t) return run_res