def _prepare_data(self, key_list, eigenvalues, factors): result = {} result["eigenvalues"] = eigenvalues result["factors"] = [] for factor in factors: val = max(factor) i = list(factor).index(val) result["factors"].append(key_list[i]) log.info(result) return result
def __init__(self, workflow_path=None): self._workflow_path = workflow_path self._options = self._get_options(self._get_config()) self._workflow_json = self._get_workflow() self._first_task, self._workflow = self._create_workflow( self._workflow_json ) self._initialize_objects() log.info( "Pypeline initialized with workflow: " + self._workflow_json['name'] )
def run(self, **kwargs): input_matrix = [] key_list = [] data_to_test = kwargs[DATA_FIELD] min_length = 99999999999 for key, val in data_to_test.iteritems(): points = list(val[0].get_points()) values = self._post_effect_output(points, val[1]) if len(values) == 0: log.error("No data points for measurement: " + str(key)) continue log.debug("Last value in series:" + str(values[-1])) log.info(str(key) + " has length " + str(len(values))) min_length = min(len(values), min_length) input_matrix.append(values) key_list.append(key) for values in input_matrix: if min_length < len(values): differ = len(values) - min_length if 0 < differ <= 5: values = values[:-differ] continue raise ValueError("Measurements have different lengths!") log.info("Counting Common factor... from " + str(len(key_list)) + " dimensions.") # Move to Common Factor! TODO: # corr_matrix = np.corrcoef(input_matrix) corrmat = np.corrcoef(input_matrix) corrmat = np.nan_to_num(corrmat) log.info(corrmat) eigenvalues, _ = np.linalg.eig(corrmat) eigenvalues = filter(lambda x: True if x > 1 else False, eigenvalues) eigenvalues = [x / len(key_list) for x in eigenvalues] input_matrix = np.array(input_matrix) input_matrix = np.transpose(input_matrix) factor = FactorAnalysis(n_components=len(eigenvalues)).fit(input_matrix) log.info(factor.components_) return { CfFinisher.KEY_FOR_DATA: self._prepare_data(key_list, eigenvalues, factor.components_), CfFinisher.KEY_NAME: "factors", }
def run(self): log.info("Starting Serenity Test Runner") for test_case in self.test_cases: if 'skip' in test_case.keys(): continue for i in xrange(test_case['iterations']): log.info("-------- Running " + test_case['name'] + " iter: " + \ str(i) + "/" + str(test_case['iterations'])) engine = PipelineEngine(test_case['workflow']) (result, exception) = engine.run(test_case['input'][i]) if result != 0: log.error("-------- TEST FAILED: " + str(exception)) else: log.info("-------- TEST SUCCESS") return 0
def run(self, **kwargs): input_matrix = [] key_list = [] data_to_test = kwargs[DATA_FIELD] min_length = 99999999999 for key, val in data_to_test.iteritems(): points = list(val[0].get_points()) values = self._post_effect_output(points, val[1]) if len(values) == 0: log.error("No data points for measurement: " + str(key)) continue log.debug("Last value in series:" + str(values[-1])) log.info(str(key) + " has length " + str(len(values))) min_length = min(len(values), min_length) input_matrix.append(values) key_list.append(key) for values in input_matrix: if min_length < len(values): differ = len(values) - min_length if 0 < differ <= 5: log.debug(len(values)) values = values[:-differ] log.debug(len(values)) continue raise ValueError("Measurements have different lengths!") log.info('Counting PCA... from ' + str(len(key_list)) + " dimensions.") corr_matrix = np.corrcoef(input_matrix) log.info(corr_matrix) return {PcaFinisher.KEY_FOR_DATA: self._prepare_data(key_list, corr_matrix.tolist()), PcaFinisher.KEY_NAME: 'correlations'}
import time from serenity_pypeline.logger import log from serenity_pypeline.pipeline_engine import PipelineEngine from tests.usage import getSerializedUsage ITERATIONS = 1 ITERVAL = 1 # seconds if __name__ == "__main__": iterations = ITERATIONS fake_input = getSerializedUsage() engine = PipelineEngine() log.info("-------- Initializied Pypeline Engine. ") for i in xrange(iterations): log.info("-------- Running Pypeline iteration: " + \ str(i) + "/" + str(iterations)) (result, exception) = engine.run(fake_input) time.sleep(ITERVAL)
def run(self, **kwargs): usage = kwargs['usage'] log.info(usage) return kwargs