def beat(self):
self.logger.debug("Consolidate metrics:")
now = time.time()
beat_elapsed = 0
self.load_consolidation()
for record in self.records.values():
#self.logger.debug("Raw: %s" % record)
_id = record.get('_id')
name = record.get('crecord_name')
aggregation_interval = record.get('aggregation_interval')
self.logger.debug("'%s':" % name)
self.logger.debug(" + interval: %s" % aggregation_interval)
last_run = record.get('consolidation_ts', now)
elapsed = now - last_run
self.logger.debug(" + elapsed: %s" % elapsed)
if elapsed == 0 or elapsed >= aggregation_interval:
self.logger.debug("Step 1: Select metrics")
mfilter = json.loads(record.get('mfilter'))
self.logger.debug(' + mfilter: %s' % mfilter)
# Exclude internal metrics
mfilter = {'$and': [mfilter, {'me': {'$nin':internal_metrics}}]}
metric_list = self.manager.store.find(mfilter=mfilter)
self.logger.debug(" + %s metrics found" % metric_list.count())
if not metric_list.count():
self.storage.update(_id, { 'output_engine': "No metrics, check your filter" })
continue
aggregation_method = record.get('aggregation_method')
self.logger.debug(" + aggregation_method: %s" % aggregation_method)
consolidation_methods = record.get('consolidation_method')
if not isinstance(consolidation_methods, list):
consolidation_methods = [ consolidation_methods ]
self.logger.debug(" + consolidation_methods: %s" % consolidation_methods)
mType = mUnit = mMin = mMax = None
# Get metrics
metrics = []
for index, metric in enumerate(metric_list):
if index == 0 :
#mType = metric.get('t')
mMin = metric.get('mi')
mMax = metric.get('ma')
mUnit = metric.get('u')
if 'sum' in consolidation_methods:
maxSum = mMax
else:
if metric.get('mi') < mMin :
mMin = metric.get('mi')
if metric.get('ma') > mMax :
mMax = metric.get('ma')
if 'sum' in consolidation_methods and mMax:
maxSum += metric.get('ma')
if metric.get('u') != mUnit :
self.logger.warning("%s: too many units" % name)
output_message = "warning : too many units"
self.logger.debug(' + %s , %s , %s, %s' % (
metric.get('_id'),
metric.get('co'),
metric.get('re',''),
metric.get('me'))
)
metrics.append(metric.get('_id'))
self.logger.debug(' + mMin: %s' % mMin)
self.logger.debug(' + mMax: %s' % mMax)
self.logger.debug(' + mUnit: %s' % mUnit)
self.logger.debug("Step 2: Aggregate (%s)" % aggregation_method)
# Set time range
tstart = last_run
if elapsed == 0 or last_run < (now - 2 * aggregation_interval):
tstart = now - aggregation_interval
self.logger.debug(
" + From: %s To %s "%
(datetime.fromtimestamp(tstart).strftime('%Y-%m-%d %H:%M:%S'),
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'))
)
values = []
for mid in metrics:
points = self.manager.get_points(tstart=tstart, tstop=now, _id=mid)
fn = self.get_math_function(aggregation_method)
pValues = [point[1] for point in points]
if not len(pValues):
continue
values.append(fn(pValues))
self.logger.debug(" + %s values" % len(values))
if not len(values):
self.storage.update(_id, { 'output_engine': "No values, check your interval" })
continue
self.logger.debug("Step 3: Consolidate (%s)" % consolidation_methods)
perf_data_array = []
for consolidation_method in consolidation_methods:
fn = self.get_math_function(consolidation_method)
value = fn(values)
self.logger.debug(" + %s: %s %s" % (consolidation_method, value, mUnit))
perf_data_array.append({
'metric' : consolidation_method,
'value' : roundSignifiantDigit(value,3),
"unit": mUnit,
'max': maxSum if consolidation_method == 'sum' else mMax,
'min': mMin,
'type': 'GAUGE'
})
self.logger.debug("Step 4: Send event")
event = cevent.forger(
connector ="consolidation",
connector_name = "engine",
event_type = "consolidation",
source_type = "resource",
component = record['component'],
resource=record['resource'],
state=0,
timestamp=now,
state_type=1,
output="Consolidation: '%s' successfully computed" % name,
long_output="",
perf_data=None,
perf_data_array=perf_data_array,
display_name=name
)
rk = cevent.get_routingkey(event)
self.counter_event += 1
self.amqp.publish(event, rk, self.amqp.exchange_name_events)
self.timestamps[_id] = now
self.logger.debug("Step 5: Update configuration")
beat_elapsed = time.time() - now
self.storage.update(_id, {
'consolidation_ts': int(now),
'nb_items': len(metrics),
'output_engine': "Computation done in %.2fs (%s/%s)" % (beat_elapsed, len(values), len(metrics))
})
else:
self.logger.debug("Not the moment")
if not beat_elapsed:
beat_elapsed = time.time() - now
self.counter_worktime += beat_elapsed
def beat(self):
beat_start = time.time()
self.clean_consolidations()
non_loaded_records = self.storage.find(
{
'$and': [{
'crecord_type': 'consolidation'
}, {
'enable': True
}, {
'loaded': {
'$ne': True
}
}]
},
namespace="object")
if len(non_loaded_records) > 0:
for item in non_loaded_records:
self.logger.info("New consolidation found '%s', load" %
item.name)
self.load(item)
for _id in self.records.keys():
exists = self.storage.find_one({'_id': _id})
if not exists:
self.logger.info("%s deleted, remove from record list" %
self.records[_id]['crecord_name'])
del (self.records[_id])
for record in self.records.values():
consolidation_last_timestamp = self.timestamps[record.get('_id')]
aggregation_interval = record.get('aggregation_interval',
self.default_interval)
current_interval = int(time.time()) - consolidation_last_timestamp
self.logger.debug(
'current interval: %s , consolidation interval: %s' %
(current_interval, aggregation_interval))
if current_interval >= aggregation_interval:
self.logger.debug('Compute new consolidation for: %s' %
record.get('crecord_name', 'No name found'))
output_message = None
mfilter = json.loads(record.get('mfilter'))
mfilter = {
'$and': [mfilter, {
'me': {
'$nin': internal_metrics
}
}]
}
#self.logger.debug('the mongo filter is: %s' % mfilter)
metric_list = self.manager.store.find(mfilter=mfilter)
self.logger.debug('length of matching metric list is: %i' %
metric_list.count())
aggregation_method = record.get('aggregation_method', False)
consolidation_methods = record.get('consolidation_method',
False)
if not isinstance(consolidation_methods, list):
consolidation_methods = [consolidation_methods]
mType = mUnit = mMin = mMax = None
values = []
for index, metric in enumerate(metric_list):
if index == 0:
#mType = metric.get('t')
mMin = metric.get('mi')
mMax = metric.get('ma')
mUnit = metric.get('u')
if 'sum' in consolidation_methods:
maxSum = mMax
else:
if metric.get('mi') < mMin:
mMin = metric.get('mi')
if metric.get('ma') > mMax:
mMax = metric.get('ma')
if 'sum' in consolidation_methods:
maxSum += metric.get('ma')
if metric.get('u') != mUnit:
output_message = "warning : too many units"
self.logger.debug(' + Get points for: %s , %s , %s, %s' %
(metric.get('_id'), metric.get('co'),
metric.get('re', ''), metric.get('me')))
if int(
time.time()
) - aggregation_interval <= consolidation_last_timestamp + 60:
tstart = consolidation_last_timestamp
#self.logger.debug(' + Use original tstart: %i' % consolidation_last_timestamp)
else:
tstart = int(time.time()) - aggregation_interval
#self.logger.debug(' + new tstart: %i' % tstart)
self.logger.debug(
' + from: %s to %s' %
(datetime.fromtimestamp(tstart).strftime(
'%Y-%m-%d %H:%M:%S'),
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S')))
list_points = self.manager.get_points(
tstart=tstart,
tstop=time.time(),
_id=metric.get('_id'))
self.logger.debug(
' + Values on interval: %s' %
' '.join([str(value[1]) for value in list_points]))
if list_points:
fn = self.get_math_function(aggregation_method)
if fn:
point_value = fn(
[value[1] for value in list_points])
else:
point_value = list_points[len(list_points) - 1][1]
values.append(point_value)
self.logger.debug(
' + Summary of horizontal aggregation "%s":' %
aggregation_method)
self.logger.debug(values)
if not consolidation_methods:
self.storage.update(record.get('_id'), {
'output_engine':
"No second aggregation function given"
})
return
if len(values) == 0:
self.logger.debug(' + No values')
self.storage.update(
record.get('_id'), {
'output_engine': "No input values",
'consolidation_ts': int(time.time())
})
self.timestamps[record.get('_id')] = int(time.time())
return
list_perf_data = []
for function_name in consolidation_methods:
fn = self.get_math_function(function_name)
if not fn:
self.logger.debug(
'No function given for second aggregation')
self.storage.update(
record.get('_id'), {
'output_engine':
"No function given for second aggregation"
})
return
if len(values) == 0:
if not output_message:
self.storage.update(record.get('_id'),
{'output_engine': "No result"})
else:
self.storage.update(
record.get('_id'), {
'output_engine':
"there are issues : %s warning : No result"
% output_message
})
value = fn(values)
self.logger.debug(' + Result of aggregation for "%s": %f' %
(function_name, value))
list_perf_data.append({
'metric':
function_name,
'value':
roundSignifiantDigit(value, 3),
"unit":
mUnit,
'max':
maxSum if function_name == 'sum' else mMax,
'min':
mMin,
'type':
'GAUGE'
})
point_timestamp = int(time.time()) - current_interval / 2
event = cevent.forger(
connector="consolidation",
connector_name="engine",
event_type="consolidation",
source_type="resource",
component=record['component'],
resource=record['resource'],
state=0,
timestamp=point_timestamp,
state_type=1,
output="Consolidation: '%s' successfully computed" %
record.get('crecord_name', 'No name'),
long_output="",
perf_data=None,
perf_data_array=list_perf_data,
display_name=record['crecord_name'])
rk = cevent.get_routingkey(event)
self.counter_event += 1
self.amqp.publish(event, rk, self.amqp.exchange_name_events)
self.logger.debug('The following event was sent:')
self.logger.debug(event)
if not output_message:
engine_output = '%s : Computation done. Next Computation in %s s' % (
datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
str(aggregation_interval))
self.storage.update(record.get('_id'),
{'output_engine': engine_output})
else:
engine_output = '%s : Computation done but there are issues : "%s" . Next Computation in %s s' % (
datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
output_message, str(aggregation_interval))
self.storage.update(record.get('_id'),
{'output_engine': engine_output})
self.storage.update(record.get('_id'),
{'consolidation_ts': int(time.time())})
self.timestamps[record.get('_id')] = int(time.time())
self.counter_worktime += time.time() - beat_start
def beat(self):
self.logger.debug("Consolidate metrics:")
now = time.time()
beat_elapsed = 0
self.load_consolidation()
for record in self.records.values():
#self.logger.debug("Raw: %s" % record)
_id = record.get('_id')
name = record.get('crecord_name')
aggregation_interval = record.get('aggregation_interval')
self.logger.debug("'%s':" % name)
self.logger.debug(" + interval: %s" % aggregation_interval)
last_run = record.get('consolidation_ts', now)
elapsed = now - last_run
self.logger.debug(" + elapsed: %s" % elapsed)
if elapsed == 0 or elapsed >= aggregation_interval:
self.logger.debug("Step 1: Select metrics")
mfilter = json.loads(record.get('mfilter'))
self.logger.debug(' + mfilter: %s' % mfilter)
# Exclude internal metrics
mfilter = {'$and': [mfilter, {'me': {'$nin':internal_metrics}}]}
metric_list = self.manager.store.find(mfilter=mfilter)
self.logger.debug(" + %s metrics found" % metric_list.count())
if not metric_list.count():
self.storage.update(_id, { 'output_engine': "No metrics, check your filter" })
continue
aggregation_method = record.get('aggregation_method')
self.logger.debug(" + aggregation_method: %s" % aggregation_method)
consolidation_methods = record.get('consolidation_method')
if not isinstance(consolidation_methods, list):
consolidation_methods = [ consolidation_methods ]
self.logger.debug(" + consolidation_methods: %s" % consolidation_methods)
mType = mUnit = mMin = mMax = None
# Get metrics
metrics = []
for index, metric in enumerate(metric_list):
if index == 0 :
#mType = metric.get('t')
mMin = metric.get('mi')
mMax = metric.get('ma')
mUnit = metric.get('u')
if 'sum' in consolidation_methods:
maxSum = mMax
else:
if metric.get('mi') < mMin :
mMin = metric.get('mi')
if metric.get('ma') > mMax :
mMax = metric.get('ma')
if 'sum' in consolidation_methods and mMax:
maxSum += metric.get('ma')
if metric.get('u') != mUnit :
self.logger.warning("%s: too many units" % name)
output_message = "warning : too many units"
self.logger.debug(' + %s , %s , %s, %s' % (
metric.get('_id'),
metric.get('co'),
metric.get('re',''),
metric.get('me'))
)
metrics.append(metric.get('_id'))
self.logger.debug(' + mMin: %s' % mMin)
self.logger.debug(' + mMax: %s' % mMax)
self.logger.debug(' + mUnit: %s' % mUnit)
self.logger.debug("Step 2: Aggregate (%s)" % aggregation_method)
# Set time range
tstart = last_run
if elapsed == 0 or last_run < (now - 2 * aggregation_interval):
tstart = now - aggregation_interval
self.logger.debug(
" + From: %s To %s "%
(datetime.fromtimestamp(tstart).strftime('%Y-%m-%d %H:%M:%S'),
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'))
)
values = []
for mid in metrics:
points = self.manager.get_points(tstart=tstart, tstop=now, _id=mid)
fn = self.get_math_function(aggregation_method)
pValues = [point[1] for point in points]
if not len(pValues):
continue
values.append(fn(pValues))
self.logger.debug(" + %s values" % len(values))
if not len(values):
self.storage.update(_id, { 'output_engine': "No values, check your interval" })
continue
self.logger.debug("Step 3: Consolidate (%s)" % consolidation_methods)
perf_data_array = []
for consolidation_method in consolidation_methods:
fn = self.get_math_function(consolidation_method)
value = fn(values)
self.logger.debug(" + %s: %s %s" % (consolidation_method, value, mUnit))
perf_data_array.append({
'metric' : consolidation_method,
'value' : roundSignifiantDigit(value,3),
"unit": mUnit,
'max': maxSum if consolidation_method == 'sum' else mMax,
'min': mMin,
'type': 'GAUGE'
})
self.logger.debug("Step 4: Send event")
event = cevent.forger(
connector ="consolidation",
connector_name = "engine",
event_type = "consolidation",
source_type = "resource",
component = record['component'],
resource=record['resource'],
state=0,
timestamp=now,
state_type=1,
output="Consolidation: '%s' successfully computed" % name,
long_output="",
perf_data=None,
perf_data_array=perf_data_array,
display_name=name
)
rk = cevent.get_routingkey(event)
self.counter_event += 1
self.amqp.publish(event, rk, self.amqp.exchange_name_events)
self.timestamps[_id] = now
self.logger.debug("Step 5: Update configuration")
beat_elapsed = time.time() - now
self.storage.update(_id, {
'consolidation_ts': int(now),
'nb_items': len(metrics),
'output_engine': "Computation done in %.2fs (%s/%s)" % (beat_elapsed, len(values), len(metrics))
})
else:
self.logger.debug("Not the moment")
if not beat_elapsed:
beat_elapsed = time.time() - now
self.counter_worktime += beat_elapsed
def beat(self):
beat_start = time.time()
self.clean_consolidations()
non_loaded_records = self.storage.find({ '$and' : [{ 'crecord_type': 'consolidation' },{'enable': True}, {'loaded': { '$ne' : True} } ] }, namespace="object" )
if len(non_loaded_records) > 0 :
for item in non_loaded_records :
self.logger.info("New consolidation found '%s', load" % item.name)
self.load(item)
for _id in self.records.keys() :
exists = self.storage.find_one({ '_id': _id } )
if not exists:
self.logger.info("%s deleted, remove from record list" % self.records[_id]['crecord_name'])
del(self.records[_id])
for record in self.records.values():
consolidation_last_timestamp = self.timestamps[record.get('_id')]
aggregation_interval = record.get('aggregation_interval', self.default_interval)
current_interval = int(time.time()) - consolidation_last_timestamp
self.logger.debug('current interval: %s , consolidation interval: %s' % (current_interval,aggregation_interval))
if current_interval >= aggregation_interval:
self.logger.debug('Compute new consolidation for: %s' % record.get('crecord_name','No name found'))
output_message = None
mfilter = json.loads(record.get('mfilter'))
mfilter = {'$and': [mfilter, {'me': {'$nin':internal_metrics}}]}
#self.logger.debug('the mongo filter is: %s' % mfilter)
metric_list = self.manager.store.find(mfilter=mfilter)
self.logger.debug('length of matching metric list is: %i' % metric_list.count())
aggregation_method = record.get('aggregation_method', False)
consolidation_methods = record.get('consolidation_method', False)
if not isinstance(consolidation_methods, list):
consolidation_methods = [ consolidation_methods ]
mType = mUnit = mMin = mMax = None
values = []
for index,metric in enumerate(metric_list) :
if index == 0 :
#mType = metric.get('t')
mMin = metric.get('mi')
mMax = metric.get('ma')
mUnit = metric.get('u')
if 'sum' in consolidation_methods:
maxSum = mMax
else:
if metric.get('mi') < mMin :
mMin = metric.get('mi')
if metric.get('ma') > mMax :
mMax = metric.get('ma')
if 'sum' in consolidation_methods:
maxSum += metric.get('ma')
if metric.get('u') != mUnit :
output_message = "warning : too many units"
self.logger.debug(' + Get points for: %s , %s , %s, %s' % (metric.get('_id'),metric.get('co'),metric.get('re',''),metric.get('me')))
if int(time.time()) - aggregation_interval <= consolidation_last_timestamp + 60:
tstart = consolidation_last_timestamp
#self.logger.debug(' + Use original tstart: %i' % consolidation_last_timestamp)
else:
tstart = int(time.time()) - aggregation_interval
#self.logger.debug(' + new tstart: %i' % tstart)
self.logger.debug(
' + from: %s to %s' %
(datetime.fromtimestamp(tstart).strftime('%Y-%m-%d %H:%M:%S'),
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'))
)
list_points = self.manager.get_points(tstart=tstart,tstop=time.time(), _id=metric.get('_id'))
self.logger.debug(' + Values on interval: %s' % ' '.join([str(value[1]) for value in list_points]))
if list_points:
fn = self.get_math_function(aggregation_method)
if fn:
point_value = fn([value[1] for value in list_points])
else:
point_value = list_points[len(list_points)-1][1]
values.append(point_value)
self.logger.debug(' + Summary of horizontal aggregation "%s":' % aggregation_method)
self.logger.debug(values)
if not consolidation_methods:
self.storage.update(record.get('_id'), {'output_engine': "No second aggregation function given" } )
return
if len(values) == 0 :
self.logger.debug(' + No values')
self.storage.update(record.get('_id'), {
'output_engine': "No input values",
'consolidation_ts':int(time.time())
})
self.timestamps[record.get('_id')] = int(time.time())
return
list_perf_data = []
for function_name in consolidation_methods :
fn = self.get_math_function(function_name)
if not fn:
self.logger.debug('No function given for second aggregation')
self.storage.update(record.get('_id'), {'output_engine': "No function given for second aggregation"})
return
if len(values) == 0 :
if not output_message:
self.storage.update(record.get('_id'), {'output_engine': "No result" } )
else:
self.storage.update(record.get('_id'), {'output_engine': "there are issues : %s warning : No result" % output_message } )
value = fn(values)
self.logger.debug(' + Result of aggregation for "%s": %f' % (function_name,value))
list_perf_data.append({
'metric' : function_name,
'value' : roundSignifiantDigit(value,3),
"unit": mUnit,
'max': maxSum if function_name == 'sum' else mMax,
'min': mMin,
'type': 'GAUGE' } )
point_timestamp = int(time.time()) - current_interval/2
event = cevent.forger(
connector ="consolidation",
connector_name = "engine",
event_type = "consolidation",
source_type = "resource",
component = record['component'],
resource=record['resource'],
state=0,
timestamp=point_timestamp,
state_type=1,
output="Consolidation: '%s' successfully computed" % record.get('crecord_name','No name'),
long_output="",
perf_data=None,
perf_data_array=list_perf_data,
display_name=record['crecord_name']
)
rk = cevent.get_routingkey(event)
self.counter_event += 1
self.amqp.publish(event, rk, self.amqp.exchange_name_events)
self.logger.debug('The following event was sent:')
self.logger.debug(event)
if not output_message:
engine_output = '%s : Computation done. Next Computation in %s s' % (datetime.now().strftime('%Y-%m-%d %H:%M:%S'),str(aggregation_interval))
self.storage.update(record.get('_id'),{'output_engine':engine_output} )
else:
engine_output = '%s : Computation done but there are issues : "%s" . Next Computation in %s s' % (datetime.now().strftime('%Y-%m-%d %H:%M:%S'),output_message,str(aggregation_interval))
self.storage.update(record.get('_id'), {'output_engine': engine_output} )
self.storage.update(record.get('_id'), {'consolidation_ts':int(time.time())})
self.timestamps[record.get('_id')] = int(time.time())
self.counter_worktime += time.time() - beat_start
