def check_hostpost(self, k): for node in self.model.get_hosts(types.NODE): # Check past readings readings = node.get_readings() # Calculate percentile on the data slc = readings[-k:] forecast = smoother.single_exponential_smoother(slc)[0] forecast = node.forecast() forecast = smoother.ar_forecast(slc) forecast = np.mean(slc) forecast = smoother.double_exponential_smoother(slc)[0] # percentile = np.percentile(slc, THR_PERCENTILE) # percentile_ = np.percentile(slc, 1 - THR_PERCENTILE) # overload = (percentile > THRESHOLD_OVERLOAD) # underload = (percentile_ < THRESHOLD_UNDERLOAD) k = K_VALUE overload = 0 underload = 0 for reading in readings[-k:]: if reading > THRESHOLD_OVERLOAD: overload += 1 if reading < THRESHOLD_UNDERLOAD: underload += 1 m = M_VALUE overload = (overload >= m) underload = (underload >= m) overload = (overload and forecast > THRESHOLD_OVERLOAD) underload = (underload and forecast < THRESHOLD_UNDERLOAD) if overload: print 'Overload in %s - %s' % (node.name, slc) # Update overload node.overloaded = overload node.underloaded = underload
def extract_profile(name, time, signal, sampling_frequency, cycle_time=hour(24), plot=False): tv = np.vectorize(to_weekday) time = tv(time) indices = np.where(time < 5) time = np.take(time, indices) signal = np.ravel(np.take(signal, indices)) elements_per_cycle = cycle_time / sampling_frequency cycle_count = len(signal) / elements_per_cycle # TODO: Filter extreme values > the 95th percentile # Remove the remainder elements signal = np.resize(signal, cycle_count * elements_per_cycle) # Reshape the signal (each cycle in its on row) signal = np.reshape(signal, (-1, elements_per_cycle)) # Get Buckets bucket_time = hour(1) bucket_count = cycle_time / bucket_time elements_per_bucket = elements_per_cycle / bucket_count # Split the signal buckets = np.hsplit(signal, bucket_count) # Reduce buckets # bucket_array = np.empty((signal.shape[0], 0), np.float32) bucket_array = np.empty(bucket_count, np.float32) i = 0 for bucket in buckets: # bucket = np.apply_along_axis(average_bucket, 1, bucket) # bucket = np.reshape(bucket, (signal.shape[0], 1)) # bucket_array = np.hstack((bucket_array, bucket)) np.reshape(bucket, (signal.shape[0] * elements_per_bucket, 1)) value = np.mean(bucket) # value = np.percentile(bucket, 90) bucket_array[i] = value i += 1 # Get averaged bucket # raw_profile = np.apply_along_axis(average_bucket, 0, bucket_array) raw_profile = bucket_array # Variance # TODO: Consider only values between the 5th and 95th percentile variance_array = np.empty(len(buckets), np.float32) for i in range(0, len(buckets)): bucket = buckets[i] bucket = np.ravel(bucket) var = np.std(bucket) variance_array[i] = var / 40 variance_array = np.ravel(variance_array) # # Variance calculation two # variance_array_2 = np.empty(len(buckets), np.float32) # per bucket averaged variance # for i in range(0, len(buckets)): # bucket = buckets[i] # variance = np.apply_along_axis(np.std, 1, bucket) # variance = np.median(np.ravel(variance)) # variance_array_2[i] = variance / 4 #2 # variance_array_2 = np.ravel(variance_array) # Increase signal resolution target_bucket_count = cycle_time / minu(5) resolution_factor = target_bucket_count / bucket_count noise_profile = np.ravel(np.array(zip(*[raw_profile for _ in xrange(resolution_factor)]))) # Smooth # smooth_profile = simple_moving_average(noise_profile, 7) # Smoothing contains the forecasted value - so remove the last value _, smooth_profile, _ = forecasting.single_exponential_smoother(noise_profile, 0.2) smooth_profile = smooth_profile[:-1] # Create noise noise_array = np.array(0, np.float32) for i in xrange(0, len(noise_profile), resolution_factor): j = i / resolution_factor variance = variance_array[j] if variance > 0: noise = np.random.normal(0, variance, resolution_factor) noise_array = np.hstack((noise_array, noise)) else: noise = np.zeros(resolution_factor, np.float32) noise_array = np.hstack((noise_array, noise)) # Apply noise smooth_profile = smooth_profile + noise_array[: len(smooth_profile)] # Normalize tv = np.vectorize(to_positive) x = 100 / np.percentile(signal, 99) # np.max(smooth_profile) smooth_profile *= x smooth_profile[smooth_profile > 100] = 100 # Frequency of result signal frequency = cycle_time / len(smooth_profile) # Plotting # if plot: _plot(name, smooth_profile, org_signal, variance_array_2) return smooth_profile, frequency