def check_hostpost(self, k):
for node in self.model.get_hosts(types.NODE):
# Check past readings
readings = node.get_readings()
# Calculate percentile on the data
slc = readings[-k:]
forecast = smoother.single_exponential_smoother(slc)[0]
forecast = node.forecast()
forecast = smoother.ar_forecast(slc)
forecast = np.mean(slc)
forecast = smoother.double_exponential_smoother(slc)[0]
# percentile = np.percentile(slc, THR_PERCENTILE)
# percentile_ = np.percentile(slc, 1 - THR_PERCENTILE)
# overload = (percentile > THRESHOLD_OVERLOAD)
# underload = (percentile_ < THRESHOLD_UNDERLOAD)
k = K_VALUE
overload = 0
underload = 0
for reading in readings[-k:]:
if reading > THRESHOLD_OVERLOAD: overload += 1
if reading < THRESHOLD_UNDERLOAD: underload += 1
m = M_VALUE
overload = (overload >= m)
underload = (underload >= m)
overload = (overload and forecast > THRESHOLD_OVERLOAD)
underload = (underload and forecast < THRESHOLD_UNDERLOAD)
if overload:
print 'Overload in %s - %s' % (node.name, slc)
# Update overload
node.overloaded = overload
node.underloaded = underload
def extract_profile(name, time, signal, sampling_frequency, cycle_time=hour(24), plot=False):
tv = np.vectorize(to_weekday)
time = tv(time)
indices = np.where(time < 5)
time = np.take(time, indices)
signal = np.ravel(np.take(signal, indices))
elements_per_cycle = cycle_time / sampling_frequency
cycle_count = len(signal) / elements_per_cycle
# TODO: Filter extreme values > the 95th percentile
# Remove the remainder elements
signal = np.resize(signal, cycle_count * elements_per_cycle)
# Reshape the signal (each cycle in its on row)
signal = np.reshape(signal, (-1, elements_per_cycle))
# Get Buckets
bucket_time = hour(1)
bucket_count = cycle_time / bucket_time
elements_per_bucket = elements_per_cycle / bucket_count
# Split the signal
buckets = np.hsplit(signal, bucket_count)
# Reduce buckets
# bucket_array = np.empty((signal.shape[0], 0), np.float32)
bucket_array = np.empty(bucket_count, np.float32)
i = 0
for bucket in buckets:
# bucket = np.apply_along_axis(average_bucket, 1, bucket)
# bucket = np.reshape(bucket, (signal.shape[0], 1))
# bucket_array = np.hstack((bucket_array, bucket))
np.reshape(bucket, (signal.shape[0] * elements_per_bucket, 1))
value = np.mean(bucket)
# value = np.percentile(bucket, 90)
bucket_array[i] = value
i += 1
# Get averaged bucket
# raw_profile = np.apply_along_axis(average_bucket, 0, bucket_array)
raw_profile = bucket_array
# Variance
# TODO: Consider only values between the 5th and 95th percentile
variance_array = np.empty(len(buckets), np.float32)
for i in range(0, len(buckets)):
bucket = buckets[i]
bucket = np.ravel(bucket)
var = np.std(bucket)
variance_array[i] = var / 40
variance_array = np.ravel(variance_array)
# # Variance calculation two
# variance_array_2 = np.empty(len(buckets), np.float32) # per bucket averaged variance
# for i in range(0, len(buckets)):
# bucket = buckets[i]
# variance = np.apply_along_axis(np.std, 1, bucket)
# variance = np.median(np.ravel(variance))
# variance_array_2[i] = variance / 4 #2
# variance_array_2 = np.ravel(variance_array)
# Increase signal resolution
target_bucket_count = cycle_time / minu(5)
resolution_factor = target_bucket_count / bucket_count
noise_profile = np.ravel(np.array(zip(*[raw_profile for _ in xrange(resolution_factor)])))
# Smooth
# smooth_profile = simple_moving_average(noise_profile, 7)
# Smoothing contains the forecasted value - so remove the last value
_, smooth_profile, _ = forecasting.single_exponential_smoother(noise_profile, 0.2)
smooth_profile = smooth_profile[:-1]
# Create noise
noise_array = np.array(0, np.float32)
for i in xrange(0, len(noise_profile), resolution_factor):
j = i / resolution_factor
variance = variance_array[j]
if variance > 0:
noise = np.random.normal(0, variance, resolution_factor)
noise_array = np.hstack((noise_array, noise))
else:
noise = np.zeros(resolution_factor, np.float32)
noise_array = np.hstack((noise_array, noise))
# Apply noise
smooth_profile = smooth_profile + noise_array[: len(smooth_profile)]
# Normalize
tv = np.vectorize(to_positive)
x = 100 / np.percentile(signal, 99) # np.max(smooth_profile)
smooth_profile *= x
smooth_profile[smooth_profile > 100] = 100
# Frequency of result signal
frequency = cycle_time / len(smooth_profile)
# Plotting
# if plot: _plot(name, smooth_profile, org_signal, variance_array_2)
return smooth_profile, frequency
