def info(self, context, target=None):
# Figure out the target.
if target == None:
target = socket.getfqdn()
rrdpath = os.path.join(FLAGS.canary_rrdpath, target)
# Grab available metrics.
available = glob.glob(os.path.join(rrdpath, "*/*.rrd"))
metrics = {}
for filename in available:
# NOTE: Not sure quite why, but it seems like
# the rrdtool commands below barf unless they
# this happens -- maybe barfing on unicode?
filename = str(filename)
m = re.match("^%s/([^\/-]+)(-([^\/]+))?/([^\.]+)\.rrd$" % rrdpath, filename)
if m:
plugin = m.group(1)
unit = m.group(3)
key = m.group(4)
# NOTE: At this point we construct a metric name that is
# equivilant to how we deconstruct the name above. It's
# important that these two operations are symmetric for
# the sake of a user's sanity.
if unit:
metric = "%s[%s].%s" % (plugin, unit, key)
else:
metric = "%s.%s" % (plugin, key)
if not (metric in metrics):
metrics[metric] = {}
metrics[metric]["from_time"] = rrdtool.first(filename)
metrics[metric]["to_time"] = rrdtool.last(filename)
step = 1
pdps = []
cfs = []
for (k, v) in rrdtool.info(filename).items():
if re.match("^step$", k):
step = int(v)
continue
elif re.match("^rra\[\d+\]\.pdp_per_row$", k):
pdps.append(int(v))
continue
elif re.match("^rra\[\d+\]\.cf", k):
cfs.append(v)
continue
pdps = list(set(pdps))
cfs = list(set(cfs))
cfs.sort()
resolutions = map(lambda x: step * x, pdps)
resolutions.sort()
metrics[metric]["cfs"] = cfs
metrics[metric]["resolutions"] = resolutions
return metrics
def getStartTime(self):
"""Gets the timestamp of the first non-null entry in the RRD"""
first = rrdtool.first(self.filename)
end = rrdtool.last(self.filename)
cf = self.getPeriodCF(first)
try:
info , _ds_rrd , data = rrdtool.fetch(self.filename,
cf, "--start", str(first), "--end", str(end))
except rrdtool.error:
# Adjust for daylight saving times
first = first - 3600
end = end + 3600
info , _ds_rrd , data = rrdtool.fetch(self.filename,
cf, "--start", str(first), "--end", str(end))
#start_rrd = info[0]
#end_rrd = info[1]
step = info[2]
for line in data:
all_none = True
for value in line:
if value is not None:
all_none = False
if not all_none:
break
first = first + step
if first >= end:
raise RRDError("The RRD file looks empty !")
return first
def makeGraph(self,nameDB,nameGraph,title):
rrdtool.graph(
str(nameGraph+".png"),
"--start",str(rrdtool.first(nameDB)),
"--end",str(rrdtool.last(nameDB)),
str("DEF:"+nameGraph+"="+nameDB+":"+'barrame'+':AVERAGE'),
str("LINE1:"+nameGraph+"#ff0000")
)
def rrd_boundary(self, boundary):
if boundary == 'first':
obs_time = rrdtool.first(self.filename)
else:
obs_time = rrdtool.last(self.filename)
dt = datetime.datetime.fromtimestamp(obs_time)
return dt.replace(hour=0, minute=0, second=0, microsecond=0)
def getCurrentTimeStamp( self, dataPointName ):
"""
Return timestamp in rrd file for this moment
"""
fileName = zenPath( 'perf', 'Daemons', self.monitor,
'%s_%s' % ( self.daemonName, dataPointName + '.rrd' ) )
firstTime = rrdtool.first( fileName )
timestamp = int((time.time() - firstTime) / self.cycleTime) * self.cycleTime + firstTime
return timestamp
def getCurrentTimeStamp(self, dataPointName):
"""
Return timestamp in rrd file for this moment
"""
fileName = zenPath('perf', 'Daemons', self.monitor,
'%s_%s' % (self.daemonName, dataPointName + '.rrd'))
firstTime = rrdtool.first(fileName)
timestamp = int((time.time() - firstTime) /
self.cycleTime) * self.cycleTime + firstTime
return timestamp
def __init__(self, path, name, date_start=None, date_end=None):
self.rrd_name = name
if date_start is None or (date_start > rrdtool.last(str(os.path.join(path, name)))):
self.date_start = str(rrdtool.first(str(os.path.join(path, name))))
else:
self.date_start = str(date_start)
if date_end is None:
self.date_end = str(rrdtool.last(str(os.path.join(path, name))))
else:
self.date_end = str(date_end)
if float(self.date_start) > float(self.date_end):
raise Exception(
"Invalid date_start={0} and date_end={1}".format(str(datetime.fromtimestamp(float(self.date_start))),
str(datetime.fromtimestamp(float(self.date_end)))))
self.user_hash = os.path.split(path)[1]
self.user_path = os.path.join(
self.get_first_part_path(path, 3),
"users",
"user",
self.user_hash[:2],
self.user_hash
)
self.uuid = self.get_uuid_from_file(self.user_path)
self.age = 0
self.sn = self.get_machine_sn_from_file(self.user_path)
self.school = "unkown"
log.debug('*******************************************')
log.debug(' creating a RRD instance ')
log.debug('start: %s', str(datetime.fromtimestamp(float(self.date_start))))
log.debug('end: %s', str(datetime.fromtimestamp(float(self.date_end))))
log.debug('PATH: %s', path)
log.debug('RRD NAME: %s', name)
log.debug('\n')
try:
self.rrd = rrdtool.fetch(str(os.path.join(path, name)), 'AVERAGE', '-r 60', '-s ' + self.date_start,
'-e ' + self.date_end)
except Exception as e:
raise Exception("rrdtool.fetch: {0}".format(e))
log.debug(' DS ')
for item in self.DS.keys():
idx = self.get_ds_index(item)
if idx != -1:
self.DS[item] = idx
else:
log.warning('DS %s not found in header of %s rrd file', item, name)
log.debug('***********************************************')
def mincuad(archivo, inicio, final, umbral):
struct_fecha_inicio = time.strptime(inicio, '%Y-%m-%dT%H:%M:%S')
iniciosegundos = int(time.mktime(struct_fecha_inicio))
if (final == ""):
finalsegundos = int(time.time())
else:
struct_fecha_final = time.strptime(final, '%Y-%m-%dT%H:%M:%S')
finalsegundos = int(time.mktime(struct_fecha_final))
# Obtencion de los datos de fetch
[startStop, names, values] = rrdtool.fetch(archivo, 'AVERAGE', '-s',
str(iniciosegundos), 'e',
str(finalsegundos))
inicioMuestras = rrdtool.first(archivo)
steps = startStop[2] #Da los steps
div = obtenerDiv(iniciosegundos, inicioMuestras, steps)
values = [v[0] for v in values if v[0] is not None]
muestras = len(values)
#Asignacion tipo funcion
[x, y] = asignacion(values, div, inicioMuestras, steps)
sumax = sum(x)
sumay = sum(y)
mediax = sumax / muestras
mediay = sumay / muestras
xmediax = obtenerValorMedia(x, mediax)
ymediay = obtenerValorMedia(y, mediay)
xmediaxymediay = producto(xmediax, ymediay)
xmediaxcuadrado = producto(xmediax, xmediax)
m = sum(xmediaxymediay) / sum(xmediaxcuadrado)
b = mediay - (m * mediax)
resultado = int((umbral - b) / m)
fechaumbral = time.strftime('%Y-%m-%dT%H:%M:%S', time.localtime(resultado))
tiempoInicialPredicciones = x[0]
[predecidosx,
predecidosy] = obtenerPredecidos(m, b, tiempoInicialPredicciones, umbral,
steps)
archivorrdpredecido = crearGrafica(predecidosy, tiempoInicialPredicciones,
steps, names, archivo)
actualizarGrafica(archivorrdpredecido, predecidosx, predecidosy)
graficar(archivo, tiempoInicialPredicciones,
predecidosx[len(predecidosx) - 1], umbral)
return fechaumbral
def rrdfetch_wrapper(rrd_db, cf, resolution, start=-3600, end=-1):
# type check
assert type(rrd_db) is str
assert type(cf) is str
assert type(resolution) is int
assert type(start) is int
assert type(end) is int
assert start < end
#print 'start is %s, end is %s' % (start, end)
last = rrdtool.last(rrd_db)
first = rrdtool.first(rrd_db)
if start < 0:
start = last + start
if end < 0:
end = last + end
if start >= last or end <= first:
return (start, end, resolution), []
if start < first:
start = first
if end > last:
end = last
#print start, end
# adjust the start/end time to match the result with the args
#start = int(start) - int(resolution)
#end = int(end) - int(resolution) # comment this line if we are using the native
# python binding(rrdtoolmodule.so)
data = rrdtool.fetch(rrd_db, cf, "-r", str(resolution), "-e", str(end), \
"-s", str(start))
#logger.debug(r'rrdtool.fetch("%s", "%s", "-r", "%s", "-e", "%s", "-s", "%s")' %\
#(rrd_db, cf, str(resolution), str(end), str(start)))
#print data[0]
#return (data[0][0]+data[0][2], data[0][1], data[0][2]),\
#data[2][0:-1]
rstep = data[0][2]
rstart = data[0][0] + rstep
rend = data[0][1] - rstep # minus rstep if we are using the native python binding
rdata = []
timestamp = rstart
for d in data[2][0:-1]:
rdata.append([timestamp, d[0]])
timestamp += rstep
return (rstart, rend, rstep), rdata
def get_list(self):
"""
list vm stats with start and end times
"""
f = os.listdir(settings.PATH_TO_RRD)
rrds = {}
for rrd in f:
try:
t = []
t.append(rrdtool.first(settings.PATH_TO_RRD + rrd))
t.append(rrdtool.last(settings.PATH_TO_RRD + rrd))
rrds.update({os.path.splitext(rrd)[0]: t})
except Exception, e:
log.error(0, 'stat_error %s %s' % (rrd, e))
def get_duration_points(rrd_file, graph_period):
start = rrdtool.first(rrd_file, '--rraindex', graph_period)
end = rrdtool.last(rrd_file)
# Start gives us the start timestamp in the data but it might be null/empty
# (i.e no data entered for that time.)
# Find the timestamp in which data was first entered.
command = [
'rrdtool', 'fetch', rrd_file, 'AVERAGE',
'--start', str(start), '--end', str(end)]
output = subprocess.check_output(command)
actual_start = find_actual_start(output)
return actual_start, end
def graficarRRD(nombre, i):
tiempo_final = int(rrdtool.last("files/" + nombre + ".rrd"))
lateral = "";
if nombre == C.CPU_NAME:
lateral = "--vertical-label=%"
else:
lateral = "--vertical-label=Bytes"
tiempo_inicial = int(rrdtool.first("files/" + nombre + ".rrd"))
ret = rrdtool.graphv("images/" + nombre + ".png",
"--start", str(tiempo_inicial + 6800),
"--end", str(tiempo_final),
lateral,
"DEF:"+nombre+"=files/" + nombre + ".rrd:"+"cosa"+":AVERAGE",
"AREA:"+nombre+"#00FF00:" + nombre)
return
def get_list(self):
"""
list vm stats with start and end times
"""
f = os.listdir(settings.PATH_TO_RRD)
rrds = {}
for rrd in f:
try:
t = []
t.append(rrdtool.first(settings.PATH_TO_RRD + rrd))
t.append(rrdtool.last(settings.PATH_TO_RRD + rrd))
rrds.update({os.path.splitext(rrd)[0]: t})
except Exception, e:
log.error(0, 'stat_error %s %s' % (rrd, e))
def _calc_start_end(start, end, rrd):
"""Calculate start and end values for fetch command."""
# Use integers to calculate resolution
if end == 'now':
end = int(time.time())
if start == 'first':
with settings.rrd_lock:
start = rrdtool.first(rrd)
# For RRDtool to include start and end timestamps.
if isinstance(start, int):
start -= 1
if isinstance(end, int):
end -= 1
return start, end
def __init__(self, name_host, description, path_to_database, file_name,
start_point, end_point, type_command, height, width):
self.name_host = name_host
self.description = description
self.path_to_database = path_to_database
self.file_name = file_name
self.file = self.path_to_database + "/" + self.file_name
self.start_point = start_point
self.end_point = end_point
self.first = rrdtool.first(self.file)
self.last = rrdtool.last(self.file)
self.lastupdate = rrdtool.lastupdate(self.file)
self.type_command = type_command
self.height = height
self.width = width
self.list_ds = self.parse_ds
self.list_menu = []
def get_duration_points(rrd_file, graph_period):
start = rrdtool.first(rrd_file, '--rraindex', graph_period)
end = rrdtool.last(rrd_file)
# Start gives us the start timestamp in the data but it might be null/empty
# (i.e no data entered for that time.)
# Find the timestamp in which data was first entered.
command = [
'rrdtool', 'fetch', rrd_file, 'AVERAGE', '--start',
str(start), '--end',
str(end)
]
output = subprocess.check_output(command)
actual_start = find_actual_start(output)
return actual_start, end
def _calc_start_end(start, end, n_points, rrd):
"""Calculate start and end values for fetch command."""
# Use integers to calculate resolution
if end is None:
end = int(time.time())
if start is None: # Latest n_points
start = end - n_points * settings.STATS_INTERVAL
elif start == 'first': # Usually empty because 'first' is too old
with settings.RRD_LOCK:
start = rrdtool.first(rrd)
# For RRDtool to include start and end timestamps.
if isinstance(start, int):
start -= 1
if isinstance(end, int):
end -= 1
return start, end
def get_first_timestamp_rrd(rrdfile_name):
"""
get_first_timestamp_rrd:
get the first timestamp which have a value of numbers
Args:
rrdfile_name (string): it is the name of your file/rrd
Returns:
a dict with the value and a status_message
first_timestamp (dict): first real timestamp of rrd (is necessary for creating a png)
"""
first_timestamp = []
get_first_status_msg = ""
# is the given file a rrd
try:
db_first_timestamp = rrdtool.first(rrdfile_name)
get_first_status_msg = f"success: first timestamp of {rrdfile_name} was found"
except Exception as e:
get_first_status_msg = f"error: get_first_timestamp_rrd({rrdfile_name}) was not possible: {sys.exc_info()[1]} \n{e}"
get_first_msg = dict()
get_first_msg['first_timestamp'] = db_first_timestamp
get_first_msg['status'] = get_first_status_msg
return get_first_msg
log.exception(0, e)
return 0
stats = []
res = []
step = ds_info['step']
for key in ds_info.keys():
if 'index' in key:
if key[0:2] == "ds":
ds_name = key[3:]
ds_name = ds_name[0:ds_name.find(']')]
stats.append(ds_name)
if 'pdp_per_row' in key:
res.append(ds_info[key] * step)
first = rrdtool.first(filepath)
last = rrdtool.last(filepath)
return {
'stats': stats,
'resolutions': res,
'first': first,
'last': last
}
def get_vm_stats(self,
vm,
names,
start="-5min",
end="now",
resolution="10"):
def getFirstDate(datafile_name):
info = rrdtool.first(datafile_name)
return info
def rrd_fetch(file, cf):
first = str(rrdtool.first(file))
last = str(rrdtool.last(file))
return rrdtool.fetch(file, cf, "-s", first, "-e", last)
def ApplyHoltData(filename=None):
if not filename:
return
# Assign names for files
old_rrd = filename
old_rrd_renamed = old_rrd + ".old"
new_rrd = old_rrd
# Regexs to be used later
dssearch = re.compile('^ds\[(\S+)\]\.type')
rrasearch = re.compile('^rra\[(\d+)\]\.(\S+)')
datasearch = re.compile('(AVERAGE|MIN|MAX|LAST)')
holtsearch = re.compile(
'(HWPREDICT|SEASONAL|DEVSEASONAL|DEVPREDICT|FAILURES)')
ds_type_search = re.compile('(GAUGE|COUNTER|DERIVE|ABSOLUTE)')
datasources = []
data_rras = []
holt_rra = {}
rra = []
try:
data = rrdtool.info(old_rrd)
except:
return
start = rrdtool.first(old_rrd)
step = data['step']
HOLTWINTERS = holtsearch.search(str(data))
if HOLTWINTERS:
return
# Move the old file out of the way
shutil.move(old_rrd, old_rrd_renamed)
sorted_keys = data.keys()
sorted_keys.sort()
for key in sorted_keys:
value = data[key]
name_match = dssearch.search(key)
if name_match:
name = name_match.groups()[0]
datasources.append(name)
else:
rra_match = rrasearch.search(key)
if rra_match:
rranum, rraprop = rra_match.groups()
if rraprop == 'cf':
data_match = datasearch.search(value)
if data_match:
data_rras.append(rranum)
else:
holt_match = holtsearch.search(value)
if holt_match:
holt_rra[value] = rranum
for key in datasources:
type_str = "ds[%s].type" % key
min_str = "ds[%s].min" % key
max_str = "ds[%s].max" % key
minimal_heartbeat_str = "ds[%s].minimal_heartbeat" % key
type = data[type_str]
type_match = ds_type_search.search(type)
if type_match:
min_str = "ds[%s].min" % key
min = data[min_str]
if not min:
min = 'U'
max_str = "ds[%s].max" % key
max = data[max_str]
if not max:
max = 'U'
minimal_heartbeat_str = "ds[%s].minimal_heartbeat" % key
minimal_heartbeat = data[minimal_heartbeat_str]
args = "%s:%s:%s" % (minimal_heartbeat, min, max)
elif type == 'COMPUTE':
cdef_str = "ds[%s].cdef" % key
args = cdef_str
else:
continue
ds_string = 'DS:%s:%s:%s' % (key, type, args)
for key in data_rras:
cf_string = "rra[%s].cf" % key
xff_string = "rra[%s].xff" % key
pdp_per_row_string = "rra[%s].pdp_per_row" % key
rows_string = "rra[%s].rows" % key
rra.append('RRA:%s:%s:%s:%s' % (data[cf_string], data[xff_string], \
data[pdp_per_row_string], data[rows_string]))
# New Basic holt winters setup .. based on assumption we will tune
# parameters later
rra.append('RRA:HWPREDICT:1440:0.1:0.0035:288')
args=['foo.rrd','--start',str(start),'--step',str(step),' '.join(datasources),\
' '.join(rra)]
rra_string = ', '.join(rra)
rrdtool.create(new_rrd, "--start", str(start), "--step", str(step),
ds_string, *rra_string.split())
# Copy the data into the new file
startStop, names, values = rrdtool.fetch(old_rrd_renamed, 'AVERAGE',
'--start', str(start))
#print startStop[0]
time = start
#print step, time
for value in values:
time += step
if value[0]:
data_fmt = "%s:%s" % (time, value[0])
log.warn(data_fmt)
rrdtool.update(new_rrd, data_fmt)
def getFirstDate(datafile_name): info = rrdtool.first(datafile_name) return info
def realizar_prediccion(fecha_inicio, hora_inicio, fecha_termino, hora_termino,
agente, baseRRD, variable, limInf, limSup):
dirSinPuntos = agente.replace(
".", "_") # Se quitan los puntos de la IP y se ponen _
directorio = os.getcwd(
) + "/" + dirSinPuntos # Se obtiene el directorio en el que deben guardarse las graficas .png
baseRRD = directorio + "/" + baseRRD
rrdtool.dump(baseRRD, baseRRD +
"Test.xml") # NO borrar, sirve para depurar esta funcion
propiedades = []
posix_inicio = calculo_fecha_posix(fecha_inicio, hora_inicio)
posix_termino = calculo_fecha_posix(fecha_termino, hora_termino)
print 'Hr inicial: -> ' + str(posix_inicio)
print 'Hr termino: -> ' + str(posix_termino)
definicion = "DEF:" + variable + "=" + baseRRD + ":" + variable + ":AVERAGE" #de la tarea 'procesadores' de Cesar
color = "AREA:" + variable + "#CC99FF" + ":" + variable
# Se llena el arreglo con las propiedades
if (
posix_inicio < rrdtool.first(baseRRD)
): # Se valida que los tiempos dados esten entre los tiempos que tiene la RRD
posix_inicio = rrdtool.first(baseRRD)
#if(posix_termino>rrdtool.last(baseRRD)):
# print "el dato mas actual de "+baseRRD+ "es del momento"+str(rrdtool.last(baseRRD)), "usare ese"
#posix_termino=rrdtool.last(baseRRD)
# Verificacion de limites, para mostrarlos en la grafica
linea_lim_inf = ""
linea_lim_sup = ""
if (limInf != -1): # Se trazara la linea inferior
linea_lim_inf = "LINE1:" + str(limInf) + "#6600CC"
if (limSup != -1): # Se traza la linea superior
linea_lim_sup = "LINE1:" + str(limSup) + "#006666"
ret = rrdtool.graph(
"grafica" + variable + ".png",
"--start",
str(int(posix_inicio)),
"--end",
str(int(posix_termino) + TIEMPO_EXTRA),
"--vertical-label=" + variable,
"--vertical-label=" + "Uso de " + variable,
"--lower-limit=0",
"--upper-limit=100",
"--rigid",
definicion,
color,
linea_lim_inf,
linea_lim_sup,
"VDEF:a=" + variable + ",LSLSLOPE",
"VDEF:b=" + variable + ",LSLINT",
"CDEF:avg2=" + variable + ",POP,a,COUNT,*,b,+",
"CDEF:pred=avg2,0," + limSup + ",LIMIT",
"VDEF:maxpred=pred,MAXIMUM",
"GPRINT:maxpred:%c:strftime",
"GPRINT:maxpred:%5.4lf",
"PRINT:maxpred:%c:strftime", #se imprime la variable en ret para usarse a nivel de python
"PRINT:maxpred:%5.4lf",
"CDEF:predB=avg2,0," + limInf + ",LIMIT",
"VDEF:minpred=predB,MINIMUM",
"GPRINT:minpred:%c:strftime",
"GPRINT:minpred:%5.4lf",
"PRINT:minpred:%c:strftime",
"PRINT:minpred:%5.4lf",
"LINE2:avg2#99FF33")
print ret
momentoSUP = ret[2][0]
valorSUP = ret[2][1]
momentoINF = ret[2][2]
valorINF = ret[2][3]
# enviaAlerta("Prediccion lim.superior: "+momentoSUP+" Valor:"+valorSUP+
#"Prediccion lim.inferior: "+momentoINF+" Valor:"+valorINF, "grafica"+ variable + ".png")
print momentoSUP + valorSUP + momentoINF + valorINF
def sum_data(options):
""" summarize collected data """
# check if summary-file exists and is newer than database
sumfile = os.path.splitext(options.dbfile)[0] + ".summary"
if os.path.exists(sumfile) and (os.path.getmtime(options.dbfile) <=
os.path.getmtime(sumfile)):
# summary is current
f = open(sumfile, "r")
result = json.load(f)
f.close()
return result
else:
options.logger.msg("INFO", "creating summary-file: %s" % sumfile)
# create summary
try:
if options.ts_start > 0:
first = options.ts_start
else:
# either no data was collected or the summary file was deleted
options.logger.msg("WARN", "trying to recreate start timepoint")
first = rrdtool.first(options.dbfile)
options.logger.msg("INFO", "estimated start is %r" % first)
last = rrdtool.last(options.dbfile)
except Exception as e:
options.logger.msg("TRACE", traceback.format_exc())
options.logger.msg("ERROR", "no data in database: %s" % options.dbfile)
sys.exit(3)
# extract avg and max values
I_def = "DEF:I=%s:I:AVERAGE" % options.dbfile
I_avg = "VDEF:I_avg=I,AVERAGE"
I_max = "VDEF:I_max=I,MAXIMUM"
U_def = "DEF:U=%s:U:AVERAGE" % options.dbfile
U_avg = "VDEF:U_avg=U,AVERAGE"
U_max = "VDEF:U_max=U,MAXIMUM"
P_def = "DEF:P=%s:P:AVERAGE" % options.dbfile
P_avg = "VDEF:P_avg=P,AVERAGE"
P_max = "VDEF:P_max=P,MAXIMUM"
args = [
"rrdtool", "graphv", options.dbfile, "--start",
str(first), "--end",
str(last), I_def, I_avg, I_max, U_def, U_avg, U_max, P_def, P_avg,
P_max, "PRINT:I_avg:%8.4lf", "PRINT:I_max:%8.4lf",
"PRINT:U_avg:%8.4lf", "PRINT:U_max:%8.4lf", "PRINT:P_avg:%6.2lf",
"PRINT:P_max:%6.2lf"
]
info = rrdtool.graphv(options.dbfile, args[3:])
summary = {
"ts_start": first,
"ts_end": last,
"U_avg": float(info['print[2]']),
"U_max": float(info['print[3]']),
"P_avg": float(info['print[4]']),
"P_max": float(info['print[5]']),
"P_tot": round((last - first + 1) * float(info['print[4]']) / 3600, 2)
}
try:
if options.voltage:
summary["I_avg"] = float(info['print[0]'])
summary["I_max"] = float(info['print[1]'])
else:
summary["I_avg"] = int(float(info['print[0]']))
summary["I_max"] = int(float(info['print[1]']))
except:
pass
# write results to file
f = open(sumfile, "w")
json.dump(summary, f, indent=2, sort_keys=True)
f.close()
return summary
log.exception(0, e)
return 0
stats = []
res = []
step = ds_info['step']
for key in ds_info.keys():
if 'index' in key:
if key[0:2] == "ds":
ds_name = key[3:]
ds_name = ds_name[0: ds_name.find(']')]
stats.append(ds_name)
if 'pdp_per_row' in key:
res.append(ds_info[key] * step)
first = rrdtool.first(filepath)
last = rrdtool.last(filepath)
return {'stats': stats, 'resolutions': res, 'first': first, 'last': last}
def get_vm_stats(self, vm, names, start="-5min", end="now", resolution="10"):
if not check_stat_exists(vm):
raise CMException('stat_not_exists')
res = []
filename = get_path(vm)
info, ds_rrd, data = rrdtool.fetch(filename, "AVERAGE", "--start", str(start), "--end", str(end), "--resolution", str(resolution))
start_rrd = info[0]
end_rrd = info[1]
step = info[2]
ts = start_rrd
def ApplyHoltData(filename=None):
if not filename:
return
# Assign names for files
old_rrd = filename
old_rrd_renamed = old_rrd + ".old"
new_rrd = old_rrd
# Regexs to be used later
dssearch = re.compile("^ds\[(\S+)\]\.type")
rrasearch = re.compile("^rra\[(\d+)\]\.(\S+)")
datasearch = re.compile("(AVERAGE|MIN|MAX|LAST)")
holtsearch = re.compile("(HWPREDICT|SEASONAL|DEVSEASONAL|DEVPREDICT|FAILURES)")
ds_type_search = re.compile("(GAUGE|COUNTER|DERIVE|ABSOLUTE)")
datasources = []
data_rras = []
holt_rra = {}
rra = []
try:
data = rrdtool.info(old_rrd)
except:
return
start = rrdtool.first(old_rrd)
step = data["step"]
HOLTWINTERS = holtsearch.search(str(data))
if HOLTWINTERS:
return
# Move the old file out of the way
shutil.move(old_rrd, old_rrd_renamed)
sorted_keys = data.keys()
sorted_keys.sort()
for key in sorted_keys:
value = data[key]
name_match = dssearch.search(key)
if name_match:
name = name_match.groups()[0]
datasources.append(name)
else:
rra_match = rrasearch.search(key)
if rra_match:
rranum, rraprop = rra_match.groups()
if rraprop == "cf":
data_match = datasearch.search(value)
if data_match:
data_rras.append(rranum)
else:
holt_match = holtsearch.search(value)
if holt_match:
holt_rra[value] = rranum
for key in datasources:
type_str = "ds[%s].type" % key
min_str = "ds[%s].min" % key
max_str = "ds[%s].max" % key
minimal_heartbeat_str = "ds[%s].minimal_heartbeat" % key
type = data[type_str]
type_match = ds_type_search.search(type)
if type_match:
min_str = "ds[%s].min" % key
min = data[min_str]
if not min:
min = "U"
max_str = "ds[%s].max" % key
max = data[max_str]
if not max:
max = "U"
minimal_heartbeat_str = "ds[%s].minimal_heartbeat" % key
minimal_heartbeat = data[minimal_heartbeat_str]
args = "%s:%s:%s" % (minimal_heartbeat, min, max)
elif type == "COMPUTE":
cdef_str = "ds[%s].cdef" % key
args = cdef_str
else:
continue
ds_string = "DS:%s:%s:%s" % (key, type, args)
for key in data_rras:
cf_string = "rra[%s].cf" % key
xff_string = "rra[%s].xff" % key
pdp_per_row_string = "rra[%s].pdp_per_row" % key
rows_string = "rra[%s].rows" % key
rra.append("RRA:%s:%s:%s:%s" % (data[cf_string], data[xff_string], data[pdp_per_row_string], data[rows_string]))
# New Basic holt winters setup .. based on assumption we will tune
# parameters later
rra.append("RRA:HWPREDICT:1440:0.1:0.0035:288")
args = ["foo.rrd", "--start", str(start), "--step", str(step), " ".join(datasources), " ".join(rra)]
rra_string = ", ".join(rra)
rrdtool.create(new_rrd, "--start", str(start), "--step", str(step), ds_string, *rra_string.split())
# Copy the data into the new file
startStop, names, values = rrdtool.fetch(old_rrd_renamed, "AVERAGE", "--start", str(start))
# print startStop[0]
time = start
# print step, time
for value in values:
time += step
if value[0]:
data_fmt = "%s:%s" % (time, value[0])
log.warn(data_fmt)
rrdtool.update(new_rrd, data_fmt)
def EjecutarP(hilo, comunidad, ip, port, name):
agentPath = lbPathmc + name + "/"
fname = "netPred.rrd"
consulta = int(consultaSNMP(comunidad, ip, port, '1.3.6.1.2.1.2.2.1.10.1'))
valor = "N:" + str(consulta)
ret = rrdtool.update(str(agentPath + name + fname), valor)
rrdtool.dump(str(agentPath + name + fname),
str(agentPath) + name + 'netP.xml')
title = "Deteccion de comportamiento anomalo"
ultimo = rrdtool.last(str(agentPath + name + fname))
inicio = ultimo - 300
ayerInicio = (inicio - 300)
ayerFinal = ultimo - 300
primero = rrdtool.first(str(agentPath + name + fname))
#cambia el valor de alfa
rrdtool.tune(str(agentPath + name + fname), '--alpha', '0.8')
#rrdtool.tune(str(agentPath + name + fname),'--beta','0.1')
#rrdtool.tune(str(agentPath + name + fname),'--gamma','0.1')
ret = rrdtool.graphv(
str(agentPath + name) + "prediccion.png",
'--start',
str(inicio),
'--end',
str(ultimo + 5),
'--title=' + title,
"--vertical-label=Bytes/s",
'--slope-mode',
"DEF:obs=" + str(agentPath + name + fname) + ":inoctets:AVERAGE",
#"DEF:outoctets=" + str(agentPath + fname) + ":outoctets:AVERAGE",
"DEF:pred=" + str(agentPath + name + fname) + ":inoctets:HWPREDICT",
"DEF:dev=" + str(agentPath + name + fname) + ":inoctets:DEVPREDICT",
"DEF:fail=" + str(agentPath + name + fname) + ":inoctets:FAILURES",
"DEF:yvalue=" + str(agentPath + name + fname) +
":inoctets:AVERAGE:start=" + str(ayerInicio) + ":end=" +
str(ayerFinal),
'SHIFT:yvalue:300',
#"RRA:DEVSEASONAL:1d:0.1:2",
#"RRA:DEVPREDICT:5d:5",
#"RRA:FAILURES:1d:7:9:5""
"CDEF:scaledh=yvalue,8,*",
"CDEF:scaledobs=obs,8,*",
"CDEF:upper=pred,dev,2,*,+",
"CDEF:lower=pred,dev,2,*,-",
"CDEF:scaledupper=upper,8,*",
"CDEF:scaledlower=lower,8,*",
"CDEF:scaledpred=pred,8,*",
"AREA:scaledh#C9C9C9:Yesterday",
"TICK:fail#FDD017:1.0:FFallas",
"LINE3:scaledobs#00FF00:In traffic",
"LINE1:scaledpred#FF00FF:Prediccion\\n",
#"LINE1:outoctets#0000FF:Out traffic",
"LINE1:scaledupper#ff0000:Upper Bound Average bits in\\n",
"LINE1:scaledlower#0000FF:Lower Bound Average bits in",
"VDEF:lastfail=fail,LAST",
#"VDEF:max=fail,MAXIMUM",
#"VDEF:min=fail,MINIMUM",
"PRINT:lastfail: %c :strftime",
"PRINT:lastfail:%6.2lf %S ",
'PRINT:fail:MIN:%1.0lf',
'PRINT:fail:MAX:%1.0lf',
)
#bandera=0
time_falla = ret['print[0]']
ultima_falla = ret['print[1]']
fmin = ret['print[2]']
fmax = ret['print[3]']
#f = open(str(agentPath)+"log.txt","a")
#print("Fallas "+ str(ultima_falla) +"----"+ str(time_falla))
if float(ultima_falla) == 1:
if hilo.ban == 0: # inicio de falla bandera de 0 a 1
hilo.inicio = str(time_falla)
hilo.ban = 1
print("INICIO DE FALLA")
if hilo.unaVez == 0:
sendAlertEmail(
"Agente : " + name + " Inicio aberración : " +
str(hilo.inicio),
str(agentPath) + "prediccion.png",
str(agentPath + name + fname))
hilo.unaVez = 1
elif hilo.ban == 1: # aun no termina la falla y gardo el ultimo tiempo
hilo.fin = str(time_falla)
print("CONTINUA" + hilo.fin)
elif float(ultima_falla
) == 0 and hilo.ban == 1: #termina la falla bandera de 1 a 0
hilo.ban = 0
f = open(str(lbPathmc) + "log.txt", "a")
f.write(str("\tFalla " + name + "\n"))
f.write("Inicio : " + str(hilo.inicio) + "\n")
if hilo.fin == "":
hilo.fin = str(time_falla)
f.write("Fin: " + str(hilo.fin) + "\n")
else:
f.write("Fin: " + str(hilo.fin) + "\n")
print("FIN DE FALLA" + hilo.fin)
if hilo.unaVez == 1:
sendAlertEmail(
"Agente : " + name + " Fin aberración : " + str(hilo.fin),
str(agentPath) + "prediccion.png",
str(agentPath + name + fname))
hilo.unaVez = 2
hilo.fin = ""
hilo.inicio = ""
f.close()
def info(self, context, target=None):
# Figure out the target.
if target is None:
target = socket.getfqdn()
rrdpath = os.path.join(CONF.canary_rrdpath, target)
# Grab available metrics.
available = glob.glob(os.path.join(rrdpath, '*/*.rrd'))
metrics = {}
for filename in available:
# NOTE: Not sure quite why, but it seems like
# the rrdtool commands below barf unless they
# this happens -- maybe barfing on unicode?
filename = str(filename)
m = re.match("^%s/([^\/-]+)(-([^\/]+))?/([^\.]+)\.rrd$" % rrdpath,
filename)
if m:
plugin = m.group(1)
unit = m.group(3)
key = m.group(4)
# NOTE: At this point we construct a metric name that is
# equivilant to how we deconstruct the name above. It's
# important that these two operations are symmetric for
# the sake of a user's sanity.
if unit:
metric = "%s[%s].%s" % (plugin, unit, key)
else:
metric = "%s.%s" % (plugin, key)
if not (metric in metrics):
metrics[metric] = {}
metrics[metric]["from_time"] = rrdtool.first(filename)
metrics[metric]["to_time"] = rrdtool.last(filename)
step = 1
pdps = []
cfs = []
for (k, v) in rrdtool.info(filename).items():
if re.match("^step$", k):
step = int(v)
continue
elif re.match("^rra\[\d+\]\.pdp_per_row$", k):
pdps.append(int(v))
continue
elif re.match("^rra\[\d+\]\.cf", k):
cfs.append(v)
continue
pdps = list(set(pdps))
cfs = list(set(cfs))
cfs.sort()
resolutions = map(lambda x: step * x, pdps)
resolutions.sort()
metrics[metric]["cfs"] = cfs
metrics[metric]["resolutions"] = resolutions
return metrics
