def getMissingData(self, timestamp, metadata_key, event_type, disp=False):
filtersEsp = ApiFilters()
filtersEsp.verbose = disp
filtersEsp.metadata_key = metadata_key
filtersEsp.time_start = timestamp - 30000
filtersEsp.time_end = timestamp + 30000
conn = SocksSSLApiConnect("http://"+self.connect, filtersEsp)
if self.useSSL:
metadata = conn.get_metadata(cert=self.cert, key=self.certkey)
else:
metadata = conn.get_metadata()
datapoints = {}
datapoints[event_type] = {}
for md in metadata:
if not md.metadata_key == metadata_key:
continue
et = md.get_event_type(event_type)
if self.useSSL:
etSSL = EventTypeSSL(et, self.cert, self.certkey)
et = etSSL
dpay = et.get_data()
for dp in dpay.data:
if dp.ts_epoch == timestamp:
self.add2log("point found")
datapoints[event_type][dp.ts_epoch] = dp.val
return datapoints
def getMissingData(self, timestamp, metadata_key, event_type):
self.log.info("Trying to get missing data")
disp = self.debug
filtersEsp = ApiFilters()
filtersEsp.verbose = disp
filtersEsp.metadata_key = metadata_key
filtersEsp.time_start = timestamp - 30000
filtersEsp.time_end = timestamp + 30000
conn = SocksSSLApiConnect("http://" + self.connect, filtersEsp)
if self.useSSL:
metadata = conn.get_metadata(cert=self.cert, key=self.certkey)
else:
metadata = conn.get_metadata()
datapoints = {}
datapoints[event_type] = {}
for md in metadata:
if not md.metadata_key == metadata_key:
continue
et = md.get_event_type(event_type)
if self.useSSL:
etSSL = EventTypeSSL(et, self.cert, self.certkey)
et = etSSL
dpay = et.get_data()
for dp in dpay.data:
if dp.ts_epoch == timestamp:
self.log.debug("point found")
datapoints[event_type][dp.ts_epoch] = dp.val
return datapoints
def getDataHourChunks(self, time_start, time_end):
filters.time_start = time_start
filters.time_end = time_end
if self.useSSL == True:
self.conn = SocksSSLApiConnect("https://"+self.connect, filters)
else:
self.conn = SocksSSLApiConnect("http://"+self.connect, filters)
metadata = self.conn.get_metadata()
try:
#Test to see if https connection is succesfull
md = metadata.next()
self.readMetaData(md)
except StopIteration:
self.add2log("There is no metadat in this time range")
return
except ConnectionError as e:
#Test to see if https connection is sucesful
self.add2log("Unable to connect to %s, exception was %s, trying SSL" % ("http://"+self.connect, type(e)))
try:
metadata = self.conn.get_metadata(cert=self.cert, key=self.certkey)
md = metadata.next()
self.useSSL = True
self.readMetaData(md)
except StopIteration:
self.add2log("There is no metadat in this time range")
except ConnectionError as e:
raise Exception("Unable to connect to %s, exception was %s, " % ("https://"+self.connect, type(e)))
for md in metadata:
self.readMetaData(md)
def __init__(self,
start=1600,
connect='iut2-net3.iu.edu',
metricName='org.osg.general-perfsonar-simple.conf',
config=None,
log=None):
self.log = log
self.config = config
########################################
### New Section to read directly the configuration file
self.metricName = metricName
#conf_dir = os.path.join("/", "etc", "rsv", "metrics")
########################################
self.debug = self.str2bool(self.readConfigFile('debug', "false"))
verbose = self.debug
# Filter variables
filters.verbose = verbose
#filters.verbose = True
# this are the filters that later will be used for the data
self.time_end = int(time.time())
self.time_start = int(self.time_end - start)
# Filter for metadata
filters.time_start = int(self.time_end - start)
# Added time_end for bug that Andy found as sometime far in the future 24 hours
filters.time_end = self.time_end + 12 * 60 * 60
# For logging pourposes
filterDates = (strftime("%a, %d %b %Y %H:%M:%S ",
time.gmtime(self.time_start)),
strftime("%a, %d %b %Y %H:%M:%S",
time.gmtime(self.time_end)))
#filterDates = (strftime("%a, %d %b %Y %H:%M:%S ", time.gmtime(filters.time_start)))
self.log.info("Data interval is from %s to %s" % filterDates)
self.log.info("Metada interval is from %s to now" %
(filters.time_start))
# Connection info to the perfsonar remote host that the data is gathered from.
self.connect = connect
self.conn = SocksSSLApiConnect("http://" + self.connect, filters)
self.cert = self.readConfigFile('usercert')
self.certkey = self.readConfigFile('userkey')
# The tmp dir structure is: tmp/metricName/host/metadatafiles
self.tmpDir = os.path.join(self.readConfigFile('tmpdirectory'),
self.metricName, self.connect + '/')
# Convert the allowedEvents into a list
allowedEvents = self.readConfigFile('allowedEvents')
self.allowedEvents = allowedEvents.split(',')
self.useSSL = False
self.maxStart = int(self.readConfigFile('maxstart'))
self.summary = self.str2bool(self.readConfigFile('summary'))
def getDataHourChunks(self, time_start, time_end):
filters.time_start = time_start
filters.time_end = time_end
# Do we have to use Socks?
if self.useSSL == True:
self.conn = SocksSSLApiConnect("https://" + self.connect, filters)
else:
self.conn = SocksSSLApiConnect("http://" + self.connect, filters)
metadata = self.conn.get_metadata()
#self.conn = None
#metadata = None
#try:
# self.conn = ApiConnect("http://" + self.connect, filters)
# metadata = self.conn.get_metadata()
#except ApiConnectWarning as apiwarning:
# self.log.exception("Unable to connect to host")
# return
try:
#Test to see if https connection is succesfull
md = metadata.next()
self.readMetaData(md)
except StopIteration:
self.log.warning("There is no metadat in this time range")
return
except ConnectionError as e:
#Test to see if https connection is sucesful
self.log.exception(
"Unable to connect to %s, exception was %s, trying SSL" %
("http://" + self.connect, type(e)))
try:
metadata = self.conn.get_metadata(cert=self.cert,
key=self.certkey)
md = metadata.next()
self.useSSL = True
self.readMetaData(md)
except StopIteration:
self.log.warning("There is no metadata in this time range")
except ConnectionError as e:
raise Exception("Unable to connect to %s, exception was %s, " %
("https://" + self.connect, type(e)))
except Exception as e:
self.log.exception("Failed to read metadata")
return
for md in metadata:
self.readMetaData(md)
def __init__(self,verbose,start,end,connect,username=None,key=None, goc=None, allowedEvents='packet-loss-rate', cert=None, certkey=None, dq=None, tmp='/tmp/rsv-perfsonar/'):
# Filter variables
filters.verbose = verbose
#filters.verbose = True
# this are the filters that later will be used for the data
self.time_end = int(time.time())
self.time_start = int(self.time_end - start)
self.time_max_start = int(time.time()) - 24*60*60
# Filter for metadata
filters.time_start = int(self.time_end - 3*start)
# Added time_end for bug that Andy found as sometime far in the future 24 hours
filters.time_end = self.time_end + 24*60*60
# For logging pourposes
filterDates = (strftime("%a, %d %b %Y %H:%M:%S ", time.gmtime(self.time_start)), strftime("%a, %d %b %Y %H:%M:%S", time.gmtime(self.time_end)))
#filterDates = (strftime("%a, %d %b %Y %H:%M:%S ", time.gmtime(filters.time_start)))
self.add2log("Data interval is from %s to %s" %filterDates)
self.add2log("Metada interval is from %s to now" % (filters.time_start))
# gfiltesrs and in general g* means connecting to the cassandra db at the central place ie goc
gfilters.verbose = False
gfilters.time_start = int(self.time_end - 5*start)
gfilters.time_end = self.time_end
gfilters.input_source = connect
# Username/Key/Location/Delay
self.connect = connect
self.username = username
self.key = key
self.goc = goc
self.conn = SocksSSLApiConnect("http://"+self.connect, filters)
self.gconn = ApiConnect(self.goc, gfilters)
self.cert = cert
self.certkey = certkey
self.tmpDir = tmp + '/' + self.connect +'/'
# Convert the allowedEvents into a list
self.allowedEvents = allowedEvents.split(',')
# In general not use SSL for contacting the perfosnar hosts
self.useSSL = False
#Code to allow publishing data to the mq
self.mq = None
self.dq = dq
if self.dq != None and self.dq!='None':
try:
self.mq = DQS(path=self.dq, granularity=5)
except Exception as e:
self.add2log("Unable to create dirq %s, exception was %s, " % (self.dq, e))
def __init__(self, start = 1600, connect = 'iut2-net3.iu.edu', metricName='org.osg.general-perfsonar-simple.conf'):
########################################
### New Section to read directly the configuration file
self.metricName = metricName
conf_dir = os.path.join("/", "etc", "rsv", "metrics")
self.configFile = os.path.join(conf_dir, metricName + ".conf")
self.add2log("Configuration File: %s" % self.configFile)
self.config = ConfigParser.RawConfigParser()
########################################
self.debug = self.str2bool(self.readConfigFile('debug'))
verbose = self.debug
# Filter variables
filters.verbose = verbose
#filters.verbose = True
# this are the filters that later will be used for the data
self.time_end = int(time.time())
self.time_start = int(self.time_end - start)
# Filter for metadata
filters.time_start = int(self.time_end - start)
# Added time_end for bug that Andy found as sometime far in the future 24 hours
filters.time_end = self.time_end + 12*60*60
# For logging pourposes
filterDates = (strftime("%a, %d %b %Y %H:%M:%S ", time.gmtime(self.time_start)), strftime("%a, %d %b %Y %H:%M:%S", time.gmtime(self.time_end)))
#filterDates = (strftime("%a, %d %b %Y %H:%M:%S ", time.gmtime(filters.time_start)))
self.add2log("Data interval is from %s to %s" %filterDates)
self.add2log("Metada interval is from %s to now" % (filters.time_start))
# Connection info to the perfsonar remote host that the data is gathered from.
self.connect = connect
self.conn = SocksSSLApiConnect("http://"+self.connect, filters)
self.cert = self.readConfigFile('usercert')
self.certkey = self.readConfigFile('userkey')
# The tmp dir structure is: tmp/metricName/host/metadatafiles
self.tmpDir = os.path.join(self.readConfigFile('tmpdirectory'), self.metricName, self.connect + '/')
# Convert the allowedEvents into a list
allowedEvents = self.readConfigFile('allowedEvents')
self.allowedEvents = allowedEvents.split(',')
self.useSSL = False
self.maxStart = int(self.readConfigFile('maxstart'))
self.summary = self.str2bool(self.readConfigFile('summary'))
class EsmondUploader(object):
def add2log(self, log):
print strftime("%a, %d %b %Y %H:%M:%S", localtime()), str(log)
def __init__(self,verbose,start,end,connect,username=None,key=None, goc=None, allowedEvents='packet-loss-rate', cert=None, certkey=None, dq=None, tmp='/tmp/rsv-perfsonar/'):
# Filter variables
filters.verbose = verbose
#filters.verbose = True
# this are the filters that later will be used for the data
self.time_end = int(time.time())
self.time_start = int(self.time_end - start)
self.time_max_start = int(time.time()) - 24*60*60
# Filter for metadata
filters.time_start = int(self.time_end - 3*start)
# Added time_end for bug that Andy found as sometime far in the future 24 hours
filters.time_end = self.time_end + 24*60*60
# For logging pourposes
filterDates = (strftime("%a, %d %b %Y %H:%M:%S ", time.gmtime(self.time_start)), strftime("%a, %d %b %Y %H:%M:%S", time.gmtime(self.time_end)))
#filterDates = (strftime("%a, %d %b %Y %H:%M:%S ", time.gmtime(filters.time_start)))
self.add2log("Data interval is from %s to %s" %filterDates)
self.add2log("Metada interval is from %s to now" % (filters.time_start))
# gfiltesrs and in general g* means connecting to the cassandra db at the central place ie goc
gfilters.verbose = False
gfilters.time_start = int(self.time_end - 5*start)
gfilters.time_end = self.time_end
gfilters.input_source = connect
# Username/Key/Location/Delay
self.connect = connect
self.username = username
self.key = key
self.goc = goc
self.conn = SocksSSLApiConnect("http://"+self.connect, filters)
self.gconn = ApiConnect(self.goc, gfilters)
self.cert = cert
self.certkey = certkey
self.tmpDir = tmp + '/' + self.connect +'/'
# Convert the allowedEvents into a list
self.allowedEvents = allowedEvents.split(',')
# In general not use SSL for contacting the perfosnar hosts
self.useSSL = False
#Code to allow publishing data to the mq
self.mq = None
self.dq = dq
if self.dq != None and self.dq!='None':
try:
self.mq = DQS(path=self.dq, granularity=5)
except Exception as e:
self.add2log("Unable to create dirq %s, exception was %s, " % (self.dq, e))
# Publish message to Mq
def publishToMq(self, arguments, event_types, datapoints, summaries_data):
for event in datapoints.keys():
# filter events for mq (must be subset of the probe's filter)
if event not in ('path-mtu', 'histogram-owdelay','packet-loss-rate','histogram-ttl','throughput','packet-retransmits','packet-trace'):
continue
# skip events that have no datapoints
if not datapoints[event]:
continue
# compose msg
msg_head = { 'input-source' : arguments['input_source'],
'input-destination' : arguments['input_destination'],
'event-type' : event,
'rsv-timestamp' : "%s" % time.time(),
'summaries' : 0,
'destination' : '/topic/perfsonar.' + event}
msg_body = { 'meta': arguments }
if summaries_data[event]:
msg_body['summaries'] = summaries_data[event]
msg_head['summaries'] = 1
if datapoints[event]:
msg_body['datapoints'] = datapoints[event]
msg = Message(body=json.dumps(msg_body), header=msg_head)
# add to mq
try:
self.mq.add_message(msg)
except Exception as e:
self.add2log("Failed to add message to mq %s, exception was %s" % (self.dq, e))
# Get Data
def getData(self, disp=False, summary=True):
self.add2log("Only reading data for event types: %s" % (str(self.allowedEvents)))
if summary:
self.add2log("Reading Summaries")
else:
self.add2log("Omiting Sumaries")
metadata = self.conn.get_metadata()
try:
#Test to see if https connection is succesfull
md = metadata.next()
self.readMetaData(md, disp, summary)
except Exception as e:
#Test to see if https connection is sucesful
self.add2log("Unable to connect to %s, exception was %s, trying SSL" % ("http://"+self.connect, e))
try:
metadata = self.conn.get_metadata(cert=self.cert, key=self.certkey)
md = metadata.next()
self.useSSL = True
self.readMetaData(md, disp, summary)
except Exception as e:
raise Exception("Unable to connect to %s, exception was %s, " % ("https://"+self.connect, e))
for md in metadata:
self.readMetaData(md, disp, summary)
# Md is a metadata object of query
def readMetaData(self, md, disp=False, summary=True):
arguments = {}
# Building the arguments for the post
arguments = {
"subject_type": md.subject_type,
"source": md.source,
"destination": md.destination,
"tool_name": md.tool_name,
"measurement_agent": md.measurement_agent,
"input_source": md.input_source,
"input_destination": md.input_destination,
"tool_name": md.tool_name
}
if not md.time_duration is None:
arguments["time_duration"] = md.time_duration
if not md.ip_transport_protocol is None:
arguments["ip_transport_protocol"] = md.ip_transport_protocol
# Assigning each metadata object property to class variables
event_types = md.event_types
metadata_key = md.metadata_key
# print extra debugging only if requested
self.add2log("Reading New METADATA/DATA %s" % (md.metadata_key))
if disp:
self.add2log("Posting args: ")
self.add2log(arguments)
# Get Events and Data Payload
summaries = {}
summaries_data = {}
# datapoints is a dict of lists
# Each of its members are lists of datapoints of a given event_type
datapoints = {}
datapointSample = {}
#load next start times
self.time_starts = {}
try:
f = open(self.tmpDir+md.metadata_key, 'r')
self.time_starts = json.loads(f.read())
f.close()
except IOError:
self.add2log("first time for %s" % (md.metadata_key))
except ValueError:
# decoding failed
self.add2log("first time for %s" % (md.metadata_key))
for et in md.get_all_event_types():
if self.useSSL:
etSSL = EventTypeSSL(et, self.cert, self.certkey)
et = etSSL
# Adding the time.end filter for the data since it is not used for the metadata
#use previously recorded end time if available
et.filters.time_start = self.time_start
if et.event_type in self.time_starts.keys():
et.filters.time_start = self.time_starts[et.event_type]
self.add2log("loaded previous time_start %s" % et.filters.time_start)
# Not to go undefitly in the past but up to one day
if et.filters.time_start < self.time_max_start:
self.add2log("previous time_start %s too old. New time_start today - 24h: %s" % (et.filters.time_start, self.time_max_start) )
et.filters.time_start = self.time_max_start
et.filters.time_end = filters.time_end
eventype = et.event_type
datapoints[eventype] = {}
#et = md.get_event_type(eventype)
if summary:
summaries[eventype] = et.summaries
else:
summaries[eventype] = []
# Skip reading data points for certain event types to improv efficiency
if eventype not in self.allowedEvents:
continue
# Read summary data
summaries_data[eventype] = []
for summ in et.get_all_summaries():
if self.useSSL:
summSSL = SummarySSL(summ, self.cert, self.certkey)
summ = summSSL
summ_data = summ.get_data()
summ_dp = [ (dp.ts_epoch, dp.val) for dp in summ_data.data ]
if not summ_dp:
continue
summaries_data[eventype].append({'event_type': eventype,
'summary_type' : summ.summary_type,
'summary_window' : summ.summary_window,
'summary_data' : summ_dp })
# Read datapoints
dpay = et.get_data()
tup = ()
for dp in dpay.data:
tup = (dp.ts_epoch, dp.val)
datapoints[eventype][dp.ts_epoch] = dp.val
# print debugging data
self.add2log("For event type %s, %d new data points" %(eventype, len(datapoints[eventype])))
if len(datapoints[eventype]) > 0 and not isinstance(tup[1], (dict,list)):
# picking the first one as the sample
datapointSample[eventype] = tup[1]
self.add2log("Sample of the data being posted %s" % datapointSample)
try:
self.postData(arguments, event_types, summaries, summaries_data, metadata_key, datapoints, summary, disp)
except Exception as e:
raise Exception("Unable to post to %s, because exception %s. Check postgresql and cassandra services are up. Then check user and key are ok " %(self.goc, e))
def postDataSlow(self, json_payload, new_metadata_key, original_datapoints, disp=False):
data = json_payload["data"]
for data_point in data:
epoch = data_point['ts']
datapoints = data_point["val"]
for datavalue in datapoints:
new_event_type = datavalue['event-type']
value = datavalue['val']
et = EventTypeBulkPost(self.goc, username=self.username, api_key=self.key, metadata_key=new_metadata_key)
et.add_data_point(new_event_type, epoch, value)
try:
et.post_data()
if epoch >= self.time_starts[new_event_type]:
self.time_starts[event_type] = epoch + 1
f = open(self.tmpDir + metadata_key, 'w')
f.write(json.dumps(self.time_starts))
f.close()
except Exception as err:
self.add2log("Exception adding new point: %s" % err)
self.add2log(et.json_payload())
continue
# Experimental function to try to recover from missing packet-count-sent or packet-count-lost data
def getMissingData(self, timestamp, metadata_key, event_type, disp=False):
filtersEsp = ApiFilters()
filtersEsp.verbose = disp
filtersEsp.metadata_key = metadata_key
filtersEsp.time_start = timestamp - 30000
filtersEsp.time_end = timestamp + 30000
conn = SocksSSLApiConnect("http://"+self.connect, filtersEsp)
if self.useSSL:
metadata = conn.get_metadata(cert=self.cert, key=self.certkey)
else:
metadata = conn.get_metadata()
datapoints = {}
datapoints[event_type] = {}
for md in metadata:
if not md.metadata_key == metadata_key:
continue
et = md.get_event_type(event_type)
if self.useSSL:
etSSL = EventTypeSSL(et, self.cert, self.certkey)
et = etSSL
dpay = et.get_data()
for dp in dpay.data:
if dp.ts_epoch == timestamp:
self.add2log("point found")
datapoints[event_type][dp.ts_epoch] = dp.val
return datapoints
def postMetaData(self, arguments, event_types, summaries, summaries_data, metadata_key, datapoints, summary = True, disp=False):
mp = MetadataPost(self.goc, username=self.username, api_key=self.key, **arguments)
for event_type in summaries.keys():
mp.add_event_type(event_type)
if summary:
summary_window_map = {}
#organize summaries windows by type so that all windows of the same type are in an array
for summy in summaries[event_type]:
if summy[0] not in summary_window_map:
summary_window_map[summy[0]] = []
summary_window_map[summy[0]].append(summy[1])
#Add each summary type once and give the post object the array of windows
for summary_type in summary_window_map:
mp.add_summary_type(event_type, summary_type, summary_window_map[summary_type])
# Added the old metadata key
mp.add_freeform_key_value("org_metadata_key", metadata_key)
new_meta = mp.post_metadata()
return new_meta
# Post data points from a metadata
def postBulkData(self, new_meta, old_metadata_key, datapoints, disp=False):
et = EventTypeBulkPost(self.goc, username=self.username, api_key=self.key, metadata_key=new_meta.metadata_key)
for event_type in datapoints.keys():
for epoch in datapoints[event_type]:
# packet-loss-rate is read as a float but should be uploaded as a dict with denominator and numerator
if event_type in ['packet-loss-rate', 'packet-loss-rate-bidir']:
# Some extra protection incase the number of datapoints in packet-loss-setn and packet-loss-rate does not match
packetcountsent = 210
packetcountlost = 0
specialTypes = ['packet-count-sent', 'packet-count-lost']
if event_type == 'packet-loss-rate-bidir':
specialTypes = ['packet-count-sent', 'packet-count-lost-bidir']
for specialType in specialTypes:
if not epoch in datapoints[specialType].keys():
self.add2log("Something went wrong time epoch %s not found for %s fixing it" % (specialType, epoch))
time.sleep(5)
datapoints_added = self.getMissingData(epoch, old_metadata_key, specialType)
# Try to get the data once more because we know it is there
try:
value = datapoints_added[specialType][epoch]
except Exception as err:
datapoints_added[specialType][epoch] = 0
value = datapoints_added[specialType][epoch]
datapoints[specialType][epoch] = value
et.add_data_point(specialType, epoch, value)
packetcountsent = datapoints['packet-count-sent'][epoch]
if event_type == 'packet-loss-rate-bidir':
packetcountlost = datapoints['packet-count-lost-bidir'][epoch]
else:
packetcountlost = datapoints['packet-count-lost'][epoch]
et.add_data_point(event_type, epoch, {'denominator': packetcountsent, 'numerator': packetcountlost})
# For the rests the data points are uploaded as they are read
else:
# datapoint are tuples the first field is epoc the second the value
et.add_data_point(event_type, epoch, datapoints[event_type][epoch])
if disp:
self.add2log("Datapoints to upload:")
self.add2log(et.json_payload())
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('error', EventTypePostWarning)
try:
et.post_data()
# Some EventTypePostWarning went wrong:
except Exception as err:
self.add2log("Probably this data already existed")
#self.postDataSlow(json.loads(et.json_payload()), new_meta.metadata_key, datapoints, disp)
for event_type in datapoints.keys():
if len(datapoints[event_type].keys()) > 0:
if event_type not in self.time_starts:
self.time_starts[event_type] = 0
next_time_start = max(datapoints[event_type].keys())+1
if next_time_start > self.time_starts[event_type]:
self.time_starts[event_type] = int(next_time_start)
f = open(self.tmpDir + old_metadata_key, 'w')
f.write(json.dumps(self.time_starts))
f.close()
self.add2log("posting NEW METADATA/DATA %s" % new_meta.metadata_key)
def postData(self, arguments, event_types, summaries, summaries_data, metadata_key, datapoints, summary = True, disp=False):
lenght_post = -1
for event_type in datapoints.keys():
if len(datapoints[event_type])>lenght_post:
lenght_post = len(datapoints[event_type])
new_meta = self.postMetaData(arguments, event_types, summaries, summaries_data, metadata_key, datapoints, summary, disp)
# Catching bad posts
if new_meta is None:
raise Exception("Post metadata empty, possible problem with user and key")
if lenght_post == 0:
self.add2log("No new datapoints skipping posting for efficiency")
return
step_size = 100
for step in range(0, lenght_post, step_size):
chunk_datapoints = {}
for event_type in datapoints.keys():
chunk_datapoints[event_type] = {}
if len(datapoints[event_type].keys())>0:
pointsconsider = sorted(datapoints[event_type].keys())[step:step+step_size]
for point in pointsconsider:
chunk_datapoints[event_type][point] = datapoints[event_type][point]
self.postBulkData(new_meta, metadata_key, chunk_datapoints, disp=False)
# Publish to MQ
if self.mq and new_meta != None:
self.publishToMq(arguments, event_types, chunk_datapoints, summaries_data)
class Uploader(object):
def __init__(self,
start=1600,
connect='iut2-net3.iu.edu',
metricName='org.osg.general-perfsonar-simple.conf',
config=None,
log=None):
self.log = log
self.config = config
########################################
### New Section to read directly the configuration file
self.metricName = metricName
#conf_dir = os.path.join("/", "etc", "rsv", "metrics")
########################################
self.debug = self.str2bool(self.readConfigFile('debug', "false"))
verbose = self.debug
# Filter variables
filters.verbose = verbose
#filters.verbose = True
# this are the filters that later will be used for the data
self.time_end = int(time.time())
self.time_start = int(self.time_end - start)
# Filter for metadata
filters.time_start = int(self.time_end - start)
# Added time_end for bug that Andy found as sometime far in the future 24 hours
filters.time_end = self.time_end + 12 * 60 * 60
# For logging pourposes
filterDates = (strftime("%a, %d %b %Y %H:%M:%S ",
time.gmtime(self.time_start)),
strftime("%a, %d %b %Y %H:%M:%S",
time.gmtime(self.time_end)))
#filterDates = (strftime("%a, %d %b %Y %H:%M:%S ", time.gmtime(filters.time_start)))
self.log.info("Data interval is from %s to %s" % filterDates)
self.log.info("Metada interval is from %s to now" %
(filters.time_start))
# Connection info to the perfsonar remote host that the data is gathered from.
self.connect = connect
self.conn = SocksSSLApiConnect("http://" + self.connect, filters)
self.cert = self.readConfigFile('usercert')
self.certkey = self.readConfigFile('userkey')
# The tmp dir structure is: tmp/metricName/host/metadatafiles
self.tmpDir = os.path.join(self.readConfigFile('tmpdirectory'),
self.metricName, self.connect + '/')
# Convert the allowedEvents into a list
allowedEvents = self.readConfigFile('allowedEvents')
self.allowedEvents = allowedEvents.split(',')
self.useSSL = False
self.maxStart = int(self.readConfigFile('maxstart'))
self.summary = self.str2bool(self.readConfigFile('summary'))
# Get Data
def getData(self):
disp = self.debug
self.log.info("Only reading data for event types: %s" %
(str(self.allowedEvents)))
summary = self.summary
disp = self.debug
if summary:
self.log.info("Reading Summaries")
else:
self.log.info("Omitting Summaries")
period = 3600
for new_time_start in range(self.time_start, self.time_end, period):
self.getDataHourChunks(new_time_start, new_time_start + period)
def getDataHourChunks(self, time_start, time_end):
filters.time_start = time_start
filters.time_end = time_end
# Do we have to use Socks?
if self.useSSL == True:
self.conn = SocksSSLApiConnect("https://" + self.connect, filters)
else:
self.conn = SocksSSLApiConnect("http://" + self.connect, filters)
metadata = self.conn.get_metadata()
#self.conn = None
#metadata = None
#try:
# self.conn = ApiConnect("http://" + self.connect, filters)
# metadata = self.conn.get_metadata()
#except ApiConnectWarning as apiwarning:
# self.log.exception("Unable to connect to host")
# return
try:
#Test to see if https connection is succesfull
md = metadata.next()
self.readMetaData(md)
except StopIteration:
self.log.warning("There is no metadat in this time range")
return
except ConnectionError as e:
#Test to see if https connection is sucesful
self.log.exception(
"Unable to connect to %s, exception was %s, trying SSL" %
("http://" + self.connect, type(e)))
try:
metadata = self.conn.get_metadata(cert=self.cert,
key=self.certkey)
md = metadata.next()
self.useSSL = True
self.readMetaData(md)
except StopIteration:
self.log.warning("There is no metadata in this time range")
except ConnectionError as e:
raise Exception("Unable to connect to %s, exception was %s, " %
("https://" + self.connect, type(e)))
except Exception as e:
self.log.exception("Failed to read metadata")
return
for md in metadata:
self.readMetaData(md)
# Md is a metadata object of query
def readMetaData(self, md):
disp = self.debug
summary = self.summary
arguments = {}
# Building the arguments for the post
arguments = {
"subject_type": md.subject_type,
"source": md.source,
"destination": md.destination,
"tool_name": md.tool_name,
"measurement_agent": md.measurement_agent,
"input_source": md.input_source,
"input_destination": md.input_destination,
"tool_name": md.tool_name
}
if not md.time_duration is None:
arguments["time_duration"] = md.time_duration
if not md.ip_transport_protocol is None:
arguments["ip_transport_protocol"] = md.ip_transport_protocol
# Assigning each metadata object property to class variables
event_types = md.event_types
metadata_key = md.metadata_key
# print extra debugging only if requested
self.log.info("Reading New METADATA/DATA %s" % (md.metadata_key))
if disp:
self.log.debug("Posting args: ")
self.log.debug(arguments)
# Get Events and Data Payload
summaries = {}
summaries_data = {}
# datapoints is a dict of lists
# Each of its members are lists of datapoints of a given event_type
datapoints = {}
datapointSample = {}
#load next start times
self.time_starts = {}
try:
f = open(self.tmpDir + md.metadata_key, 'r')
self.time_starts = json.loads(f.read())
f.close()
except IOError:
self.log.exception("first time for %s" % (md.metadata_key))
except ValueError:
# decoding failed
self.log.exception("first time for %s" % (md.metadata_key))
for et in md.get_all_event_types():
if self.useSSL:
etSSL = EventTypeSSL(et, self.cert, self.certkey)
et = etSSL
# Adding the time.end filter for the data since it is not used for the metadata
#use previously recorded end time if available
et.filters.time_start = filters.time_start
if et.event_type in self.time_starts.keys():
et.filters.time_start = self.time_starts[et.event_type]
self.log.info("loaded previous time_start %s" %
et.filters.time_start)
et.filters.time_end = filters.time_end
if et.filters.time_end < et.filters.time_start:
continue
if (et.filters.time_end - et.filters.time_start) > self.maxStart:
et.filters.time_start = et.filters.time_end - self.maxStart
eventype = et.event_type
datapoints[eventype] = {}
if summary:
summaries[eventype] = et.summaries
else:
summaries[eventype] = []
# Skip reading data points for certain event types to improv efficiency
if eventype not in self.allowedEvents:
continue
# Read summary data
if summary:
summaries_data[eventype] = []
for summ in et.get_all_summaries():
if self.useSSL:
summSSL = SummarySSL(summ, self.cert, self.certkey)
summ = summSSL
summ_data = summ.get_data()
summ_dp = [(dp.ts_epoch, dp.val) for dp in summ_data.data]
if not summ_dp:
continue
summaries_data[eventype].append({
'event_type': eventype,
'summary_type': summ.summary_type,
'summary_window': summ.summary_window,
'summary_data': summ_dp
})
# Read datapoints
warnings.filterwarnings('error')
try:
dpay = et.get_data()
except Warning:
raise Exception("Unable to read data exception: %s")
tup = ()
for dp in dpay.data:
tup = (dp.ts_epoch, dp.val)
datapoints[eventype][dp.ts_epoch] = dp.val
# print debugging data
self.log.info("For event type %s, %d new data points" %
(eventype, len(datapoints[eventype])))
if len(datapoints[eventype]) > 0 and not isinstance(
tup[1], (dict, list)):
# picking the first one as the sample
datapointSample[eventype] = tup[1]
self.log.debug("Sample of the data being posted %s" % datapointSample)
try:
self.postData(arguments, event_types, summaries, summaries_data,
metadata_key, datapoints)
except Exception as e:
import traceback
traceback.print_exc()
raise e
#raise Exception("Unable to post because exception: %s" % repr(e))
# Experimental function to try to recover from missing packet-count-sent or packet-count-lost data
def getMissingData(self, timestamp, metadata_key, event_type):
self.log.info("Trying to get missing data")
disp = self.debug
filtersEsp = ApiFilters()
filtersEsp.verbose = disp
filtersEsp.metadata_key = metadata_key
filtersEsp.time_start = timestamp - 30000
filtersEsp.time_end = timestamp + 30000
conn = SocksSSLApiConnect("http://" + self.connect, filtersEsp)
if self.useSSL:
metadata = conn.get_metadata(cert=self.cert, key=self.certkey)
else:
metadata = conn.get_metadata()
datapoints = {}
datapoints[event_type] = {}
for md in metadata:
if not md.metadata_key == metadata_key:
continue
et = md.get_event_type(event_type)
if self.useSSL:
etSSL = EventTypeSSL(et, self.cert, self.certkey)
et = etSSL
dpay = et.get_data()
for dp in dpay.data:
if dp.ts_epoch == timestamp:
self.log.debug("point found")
datapoints[event_type][dp.ts_epoch] = dp.val
return datapoints
# Place holde for posting Data if it is to Esmond, ActiveMQ, RabbitMQ
def postData(self, arguments, event_types, summaries, summaries_data,
metadata_key, datapoints):
pass
def readConfigFile(self, key, default=""):
section = self.metricName + " args"
try:
return self.config.get(section, key)
except ConfigParser.NoOptionError:
self.log.warning("Error getting key: %s from config file", key)
return default
def str2bool(self, word):
return word.lower() in ("true")
def total_size(o, handlers={}, verbose=False):
""" Returns the approximate memory footprint an object and all of its contents.
Automatically finds the contents of the following builtin containers and
their subclasses: tuple, list, deque, dict, set and frozenset.
To search other containers, add handlers to iterate over their contents:
handlers = {SomeContainerClass: iter,
OtherContainerClass: OtherContainerClass.get_elements}
"""
dict_handler = lambda d: chain.from_iterable(d.items())
all_handlers = {
tuple: iter,
list: iter,
dict: dict_handler,
set: iter,
frozenset: iter,
}
#all_handlers.update(handlers) # user handlers take precedence
seen = set() # track which object id's have already been seen
default_size = sys.getsizeof(
0) # estimate sizeof object without __sizeof__
def sizeof(o):
if id(o) in seen: # do not double count the same object
return 0
seen.add(id(o))
s = sys.getsizeof(o, default_size)
for typ, handler in all_handlers.items():
if isinstance(o, typ):
s += sum(map(sizeof, handler(o)))
break
return s
class Uploader(object):
def add2log(self, log):
print strftime("%a, %d %b %Y %H:%M:%S", localtime()), str(log)
def __init__(self, start = 1600, connect = 'iut2-net3.iu.edu', metricName='org.osg.general-perfsonar-simple.conf'):
########################################
### New Section to read directly the configuration file
self.metricName = metricName
conf_dir = os.path.join("/", "etc", "rsv", "metrics")
self.configFile = os.path.join(conf_dir, metricName + ".conf")
self.add2log("Configuration File: %s" % self.configFile)
self.config = ConfigParser.RawConfigParser()
########################################
self.debug = self.str2bool(self.readConfigFile('debug'))
verbose = self.debug
# Filter variables
filters.verbose = verbose
#filters.verbose = True
# this are the filters that later will be used for the data
self.time_end = int(time.time())
self.time_start = int(self.time_end - start)
# Filter for metadata
filters.time_start = int(self.time_end - start)
# Added time_end for bug that Andy found as sometime far in the future 24 hours
filters.time_end = self.time_end + 12*60*60
# For logging pourposes
filterDates = (strftime("%a, %d %b %Y %H:%M:%S ", time.gmtime(self.time_start)), strftime("%a, %d %b %Y %H:%M:%S", time.gmtime(self.time_end)))
#filterDates = (strftime("%a, %d %b %Y %H:%M:%S ", time.gmtime(filters.time_start)))
self.add2log("Data interval is from %s to %s" %filterDates)
self.add2log("Metada interval is from %s to now" % (filters.time_start))
# Connection info to the perfsonar remote host that the data is gathered from.
self.connect = connect
self.conn = SocksSSLApiConnect("http://"+self.connect, filters)
self.cert = self.readConfigFile('usercert')
self.certkey = self.readConfigFile('userkey')
# The tmp dir structure is: tmp/metricName/host/metadatafiles
self.tmpDir = os.path.join(self.readConfigFile('tmpdirectory'), self.metricName, self.connect + '/')
# Convert the allowedEvents into a list
allowedEvents = self.readConfigFile('allowedEvents')
self.allowedEvents = allowedEvents.split(',')
self.useSSL = False
self.maxStart = int(self.readConfigFile('maxstart'))
self.summary = self.str2bool(self.readConfigFile('summary'))
# Get Data
def getData(self):
disp = self.debug
self.add2log("Only reading data for event types: %s" % (str(self.allowedEvents)))
summary = self.summary
disp = self.debug
if summary:
self.add2log("Reading Summaries")
else:
self.add2log("Omiting Sumaries")
period = 3600
for new_time_start in range(self.time_start, self.time_end, period):
self.getDataHourChunks(new_time_start, new_time_start + period)
def getDataHourChunks(self, time_start, time_end):
filters.time_start = time_start
filters.time_end = time_end
if self.useSSL == True:
self.conn = SocksSSLApiConnect("https://"+self.connect, filters)
else:
self.conn = SocksSSLApiConnect("http://"+self.connect, filters)
metadata = self.conn.get_metadata()
try:
#Test to see if https connection is succesfull
md = metadata.next()
self.readMetaData(md)
except StopIteration:
self.add2log("There is no metadat in this time range")
return
except ConnectionError as e:
#Test to see if https connection is sucesful
self.add2log("Unable to connect to %s, exception was %s, trying SSL" % ("http://"+self.connect, type(e)))
try:
metadata = self.conn.get_metadata(cert=self.cert, key=self.certkey)
md = metadata.next()
self.useSSL = True
self.readMetaData(md)
except StopIteration:
self.add2log("There is no metadat in this time range")
except ConnectionError as e:
raise Exception("Unable to connect to %s, exception was %s, " % ("https://"+self.connect, type(e)))
for md in metadata:
self.readMetaData(md)
# Md is a metadata object of query
def readMetaData(self, md):
disp = self.debug
summary = self.summary
arguments = {}
# Building the arguments for the post
arguments = {
"subject_type": md.subject_type,
"source": md.source,
"destination": md.destination,
"tool_name": md.tool_name,
"measurement_agent": md.measurement_agent,
"input_source": md.input_source,
"input_destination": md.input_destination,
"tool_name": md.tool_name
}
if not md.time_duration is None:
arguments["time_duration"] = md.time_duration
if not md.ip_transport_protocol is None:
arguments["ip_transport_protocol"] = md.ip_transport_protocol
# Assigning each metadata object property to class variables
event_types = md.event_types
metadata_key = md.metadata_key
# print extra debugging only if requested
self.add2log("Reading New METADATA/DATA %s" % (md.metadata_key))
if disp:
self.add2log("Posting args: ")
self.add2log(arguments)
# Get Events and Data Payload
summaries = {}
summaries_data = {}
# datapoints is a dict of lists
# Each of its members are lists of datapoints of a given event_type
datapoints = {}
datapointSample = {}
#load next start times
self.time_starts = {}
try:
f = open(self.tmpDir+md.metadata_key, 'r')
self.time_starts = json.loads(f.read())
f.close()
except IOError:
self.add2log("first time for %s" % (md.metadata_key))
except ValueError:
# decoding failed
self.add2log("first time for %s" % (md.metadata_key))
for et in md.get_all_event_types():
if self.useSSL:
etSSL = EventTypeSSL(et, self.cert, self.certkey)
et = etSSL
# Adding the time.end filter for the data since it is not used for the metadata
#use previously recorded end time if available
et.filters.time_start = filters.time_start
if et.event_type in self.time_starts.keys():
et.filters.time_start = self.time_starts[et.event_type]
self.add2log("loaded previous time_start %s" % et.filters.time_start)
et.filters.time_end = filters.time_end
if et.filters.time_end < et.filters.time_start:
continue
if (et.filters.time_end - et.filters.time_start) > self.maxStart:
et.filters.time_start = et.filters.time_end - self.maxStart
eventype = et.event_type
datapoints[eventype] = {}
if summary:
summaries[eventype] = et.summaries
else:
summaries[eventype] = []
# Skip reading data points for certain event types to improv efficiency
if eventype not in self.allowedEvents:
continue
# Read summary data
if summary:
summaries_data[eventype] = []
for summ in et.get_all_summaries():
if self.useSSL:
summSSL = SummarySSL(summ, self.cert, self.certkey)
summ = summSSL
summ_data = summ.get_data()
summ_dp = [ (dp.ts_epoch, dp.val) for dp in summ_data.data ]
if not summ_dp:
continue
summaries_data[eventype].append({'event_type': eventype,
'summary_type' : summ.summary_type,
'summary_window' : summ.summary_window,
'summary_data' : summ_dp })
# Read datapoints
warnings.filterwarnings('error')
try:
dpay = et.get_data()
except Warning:
raise Exception("Unable to read data exception: %s" )
tup = ()
for dp in dpay.data:
tup = (dp.ts_epoch, dp.val)
datapoints[eventype][dp.ts_epoch] = dp.val
# print debugging data
self.add2log("For event type %s, %d new data points" %(eventype, len(datapoints[eventype])))
if len(datapoints[eventype]) > 0 and not isinstance(tup[1], (dict,list)):
# picking the first one as the sample
datapointSample[eventype] = tup[1]
self.add2log("Sample of the data being posted %s" % datapointSample)
try:
self.postData(arguments, event_types, summaries, summaries_data, metadata_key, datapoints)
except Exception as e:
raise Exception("Unable to post because exception: %s" % repr(e))
# Experimental function to try to recover from missing packet-count-sent or packet-count-lost data
def getMissingData(self, timestamp, metadata_key, event_type):
disp = self.debug
filtersEsp = ApiFilters()
filtersEsp.verbose = disp
filtersEsp.metadata_key = metadata_key
filtersEsp.time_start = timestamp - 30000
filtersEsp.time_end = timestamp + 30000
conn = SocksSSLApiConnect("http://"+self.connect, filtersEsp)
if self.useSSL:
metadata = conn.get_metadata(cert=self.cert, key=self.certkey)
else:
metadata = conn.get_metadata()
datapoints = {}
datapoints[event_type] = {}
for md in metadata:
if not md.metadata_key == metadata_key:
continue
et = md.get_event_type(event_type)
if self.useSSL:
etSSL = EventTypeSSL(et, self.cert, self.certkey)
et = etSSL
dpay = et.get_data()
for dp in dpay.data:
if dp.ts_epoch == timestamp:
self.add2log("point found")
datapoints[event_type][dp.ts_epoch] = dp.val
return datapoints
# Place holde for posting Data if it is to Esmond, ActiveMQ, RabbitMQ
def postData(self, arguments, event_types, summaries, summaries_data, metadata_key, datapoints):
pass
def readConfigFile(self, key):
section = self.metricName + " args"
ret = self.config.read(self.configFile)
try:
return self.config.get(section, key)
except ConfigParser.NoOptionError:
self.add2log("ERROR: config knob %s not found in file: %s"% (self.key, self.configFile))
raise Exception("ERROR: config knob %s not found in file: %s"% (self.key, self.configFile))
def str2bool(self,word):
return word.lower() in ("true")
def total_size(o, handlers={}, verbose=False):
""" Returns the approximate memory footprint an object and all of its contents.
Automatically finds the contents of the following builtin containers and
their subclasses: tuple, list, deque, dict, set and frozenset.
To search other containers, add handlers to iterate over their contents:
handlers = {SomeContainerClass: iter,
OtherContainerClass: OtherContainerClass.get_elements}
"""
dict_handler = lambda d: chain.from_iterable(d.items())
all_handlers = {tuple: iter,
list: iter,
dict: dict_handler,
set: iter,
frozenset: iter,
}
#all_handlers.update(handlers) # user handlers take precedence
seen = set() # track which object id's have already been seen
default_size = sys.getsizeof(0) # estimate sizeof object without __sizeof__
def sizeof(o):
if id(o) in seen: # do not double count the same object
return 0
seen.add(id(o))
s = sys.getsizeof(o, default_size)
for typ, handler in all_handlers.items():
if isinstance(o, typ):
s += sum(map(sizeof, handler(o)))
break
return s