def run(self, **kwargs):
input_matrix = []
key_list = []
data_to_test = kwargs[DATA_FIELD]
min_length = 99999999999
for key, val in data_to_test.iteritems():
points = list(val[0].get_points())
values = self._post_effect_output(points, val[1])
if len(values) == 0:
log.error("No data points for measurement: " + str(key))
continue
log.debug("Last value in series:" + str(values[-1]))
log.info(str(key) + " has length " + str(len(values)))
min_length = min(len(values), min_length)
input_matrix.append(values)
key_list.append(key)
for values in input_matrix:
if min_length < len(values):
differ = len(values) - min_length
if 0 < differ <= 5:
values = values[:-differ]
continue
raise ValueError("Measurements have different lengths!")
log.info("Counting Common factor... from " + str(len(key_list)) + " dimensions.")
# Move to Common Factor! TODO:
# corr_matrix = np.corrcoef(input_matrix)
corrmat = np.corrcoef(input_matrix)
corrmat = np.nan_to_num(corrmat)
log.info(corrmat)
eigenvalues, _ = np.linalg.eig(corrmat)
eigenvalues = filter(lambda x: True if x > 1 else False, eigenvalues)
eigenvalues = [x / len(key_list) for x in eigenvalues]
input_matrix = np.array(input_matrix)
input_matrix = np.transpose(input_matrix)
factor = FactorAnalysis(n_components=len(eigenvalues)).fit(input_matrix)
log.info(factor.components_)
return {
CfFinisher.KEY_FOR_DATA: self._prepare_data(key_list, eigenvalues, factor.components_),
CfFinisher.KEY_NAME: "factors",
}
def run(self):
log.info("Starting Serenity Test Runner")
for test_case in self.test_cases:
if 'skip' in test_case.keys():
continue
for i in xrange(test_case['iterations']):
log.info("-------- Running " + test_case['name'] + " iter: " + \
str(i) + "/" + str(test_case['iterations']))
engine = PipelineEngine(test_case['workflow'])
(result, exception) = engine.run(test_case['input'][i])
if result != 0:
log.error("-------- TEST FAILED: " + str(exception))
else:
log.info("-------- TEST SUCCESS")
return 0
def run(self, **kwargs):
input_matrix = []
key_list = []
data_to_test = kwargs[DATA_FIELD]
min_length = 99999999999
for key, val in data_to_test.iteritems():
points = list(val[0].get_points())
values = self._post_effect_output(points, val[1])
if len(values) == 0:
log.error("No data points for measurement: " + str(key))
continue
log.debug("Last value in series:" + str(values[-1]))
log.info(str(key) + " has length " + str(len(values)))
min_length = min(len(values), min_length)
input_matrix.append(values)
key_list.append(key)
for values in input_matrix:
if min_length < len(values):
differ = len(values) - min_length
if 0 < differ <= 5:
log.debug(len(values))
values = values[:-differ]
log.debug(len(values))
continue
raise ValueError("Measurements have different lengths!")
log.info('Counting PCA... from ' + str(len(key_list)) + " dimensions.")
corr_matrix = np.corrcoef(input_matrix)
log.info(corr_matrix)
return {PcaFinisher.KEY_FOR_DATA: self._prepare_data(key_list,
corr_matrix.tolist()),
PcaFinisher.KEY_NAME: 'correlations'}
def run(self, serializedUsage):
"""
Runs every task from workflow
As parameter we expect ResourceUsage.proto from mesos.proto
:return: Status of workflow execution
"""
usage = ResourceUsage()
try:
usage.ParseFromString(serializedUsage)
except Exception as e:
log.error("Exception occurred while parsing usage: " + str(e))
return 1, e
queue = Queue.Queue()
queue.put(self._first_task)
run_res = (0, None)
while not queue.empty():
task = queue.get()
try:
if task.input is None:
result = task.run(usage=usage)
else:
result = task.run(**task.input)
for t in task.next_success:
t.input = result
queue.put(t)
except Exception as e:
import traceback
log.error(traceback.format_exc())
log.error("Exception occurred while execution tasks: " + str(e))
run_res = (1, e)
for t in task.next_error:
t.input = {"error": e}
queue.put(t)
return run_res
