def export_removeRegisters(self, entriesList):
"""
Remove a record for a type
"""
setup = self.serviceInfoDict["clientSetup"]
expectedTypes = [basestring, datetime.datetime, datetime.datetime, list]
for entry in entriesList:
if len(entry) != 4:
return S_ERROR("Invalid records")
for i in range(len(entry)):
if not isinstance(entry[i], expectedTypes[i]):
return S_ERROR("%s field in the records should be %s" % (i, expectedTypes[i]))
ok = 0
for entry in entriesList:
startTime = int(Time.toEpoch(entry[1]))
endTime = int(Time.toEpoch(entry[2]))
record = entry[3]
result = self.__acDB.deleteRecord(
setup, entry[0], startTime, endTime, record
) # pylint: disable=too-many-function-args,no-member
if not result["OK"]:
return S_OK(ok)
ok += 1
return S_OK(ok)
def web_getHistoryValues(self):
try:
dbVars = [ str(f) for f in json.loads(self.request.arguments[ 'vars' ][0]) ]
except:
dbVars = [ 'Load', 'Jobs', 'TransferredFiles' ]
try:
timespan = int(self.request.arguments[ 'timespan' ][0])
except:
timespan = 86400
rpcClient = RPCClient("WorkloadManagement/VirtualMachineManager")
result = rpcClient.getHistoryValues(3600, {}, dbVars, timespan)
if not result[ 'OK' ]:
callback = {"success":"false", "error":result[ 'Message' ]}
return self.write(callback)
svcData = result[ 'Value' ]
data = []
olderThan = Time.toEpoch() - 400
for record in svcData[ 'Records' ]:
rL = []
for iP in range(len(svcData[ 'ParameterNames' ])):
param = svcData[ 'ParameterNames' ][iP]
if param == 'Update':
rL.append(Time.toEpoch(record[iP]))
else:
rL.append(record[iP])
if rL[0] < olderThan:
data.append(rL)
callback = {"success":"true", 'data': data, 'fields' : svcData[ 'ParameterNames' ]}
return self.write(callback)
def getPlotData( self ):
retVal = self.__parseFormParams()
if not retVal[ 'OK' ]:
c.error = retVal[ 'Message' ]
return render( "/error.mako" )
params = retVal[ 'Value' ]
repClient = ReportsClient( rpcClient = getRPCClient( "Accounting/ReportGenerator" ) )
retVal = repClient.getReport( *params )
if not retVal[ 'OK' ]:
c.error = retVal[ 'Message' ]
return render( "/error.mako" )
rawData = retVal[ 'Value' ]
groupKeys = rawData[ 'data' ].keys()
groupKeys.sort()
if 'granularity' in rawData:
granularity = rawData[ 'granularity' ]
data = rawData['data']
tS = int( Time.toEpoch( params[2] ) )
timeStart = tS - tS % granularity
strData = "epoch,%s\n" % ",".join( groupKeys )
for timeSlot in range( timeStart, int( Time.toEpoch( params[3] ) ), granularity ):
lineData = [ str( timeSlot ) ]
for key in groupKeys:
if timeSlot in data[ key ]:
lineData.append( str( data[ key ][ timeSlot ] ) )
else:
lineData.append( "" )
strData += "%s\n" % ",".join( lineData )
else:
strData = "%s\n" % ",".join( groupKeys )
strData += ",".join( [ str( rawData[ 'data' ][ k ] ) for k in groupKeys ] )
response.headers['Content-type'] = 'text/csv'
response.headers['Content-Disposition'] = 'attachment; filename="%s.csv"' % md5( str( params ) ).hexdigest()
response.headers['Content-Length'] = len( strData )
return strData
def export_removeRegisters(self, entriesList):
"""
Remove a record for a type
"""
setup = self.serviceInfoDict['clientSetup']
expectedTypes = [
six.string_types, datetime.datetime, datetime.datetime, list
]
for entry in entriesList:
if len(entry) != 4:
return S_ERROR("Invalid records")
for i in range(len(entry)):
if not isinstance(entry[i], expectedTypes[i]):
return S_ERROR("%s field in the records should be %s" %
(i, expectedTypes[i]))
ok = 0
for entry in entriesList:
startTime = int(Time.toEpoch(entry[1]))
endTime = int(Time.toEpoch(entry[2]))
record = entry[3]
result = self.__acDB.deleteRecord( # pylint: disable=no-member
setup, entry[0], startTime, endTime, record)
if not result['OK']:
return S_OK(ok)
ok += 1
return S_OK(ok)
def getHistoryValues(self):
try:
dbVars = [str(f) for f in simplejson.loads(request.params['vars'])]
except:
dbVars = ['Load', 'Jobs', 'TransferredFiles']
try:
timespan = int(request.params['timespan'])
except:
timespan = 86400
rpcClient = getRPCClient("WorkloadManagement/VirtualMachineManager")
result = rpcClient.getHistoryValues(3600, {}, dbVars, timespan)
if not result['OK']:
return S_ERROR(result['Message'])
svcData = result['Value']
data = []
olderThan = Time.toEpoch() - 400
for record in svcData['Records']:
rL = []
for iP in range(len(svcData['ParameterNames'])):
param = svcData['ParameterNames'][iP]
if param == 'Update':
rL.append(Time.toEpoch(record[iP]))
else:
rL.append(record[iP])
if rL[0] < olderThan:
data.append(rL)
return S_OK({'data': data, 'fields': svcData['ParameterNames']})
def web_getHistoryValues(self):
try:
dbVars = [
str(f) for f in json.loads(self.request.arguments['vars'][0])
]
except:
dbVars = ['Load', 'Jobs', 'TransferredFiles']
try:
timespan = int(self.request.arguments['timespan'][0])
except:
timespan = 86400
rpcClient = RPCClient("WorkloadManagement/VirtualMachineManager")
result = rpcClient.getHistoryValues(3600, {}, dbVars, timespan)
if not result['OK']:
callback = {"success": "false", "error": result['Message']}
return self.write(callback)
svcData = result['Value']
data = []
olderThan = Time.toEpoch() - 400
for record in svcData['Records']:
rL = []
for iP in range(len(svcData['ParameterNames'])):
param = svcData['ParameterNames'][iP]
if param == 'Update':
rL.append(Time.toEpoch(record[iP]))
else:
rL.append(record[iP])
if rL[0] < olderThan:
data.append(rL)
callback = {
"success": "true",
'data': data,
'fields': svcData['ParameterNames']
}
return self.write(callback)
def export_removeRegisters(self, entriesList):
"""
Remove a record for a type
"""
setup = self.serviceInfoDict['clientSetup']
expectedTypes = [
types.StringType, Time._dateTimeType, Time._dateTimeType,
types.ListType
]
for entry in entriesList:
if len(entry) != 4:
return S_ERROR("Invalid records")
for i in range(len(entry)):
if type(entry[i]) != expectedTypes[i]:
return S_ERROR("%s field in the records should be %s" %
(i, expectedTypes[i]))
ok = 0
for entry in entriesList:
typeName = "%s_%s" % (setup, entry[0])
startTime = int(Time.toEpoch(entry[1]))
endTime = int(Time.toEpoch(entry[2]))
record = entry[3]
result = gAccountingDB.deleteRecord(typeName, startTime, endTime,
record)
if not result['OK']:
return S_OK(ok)
ok += 1
return S_OK(ok)
def generateDocs(data, withTimeStamp=True):
"""Generator for fast bulk indexing, yields docs
:param list data: list of dictionaries
:param bool withTimeStamp: add the timestamps to the docs
:return: doc
"""
for doc in copy.deepcopy(data):
if "_type" not in doc:
doc["_type"] = "_doc"
if withTimeStamp:
if "timestamp" not in doc:
sLog.warn("timestamp is not given")
# if the timestamp is not provided, we use the current utc time.
timestamp = doc.get("timestamp", int(Time.toEpoch()))
try:
if isinstance(timestamp, datetime):
doc["timestamp"] = int(timestamp.strftime("%s")) * 1000
elif isinstance(timestamp, six.string_types):
timeobj = datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f")
doc["timestamp"] = int(timeobj.strftime("%s")) * 1000
else: # we assume the timestamp is an unix epoch time (integer).
doc["timestamp"] = timestamp * 1000
except (TypeError, ValueError) as e:
# in case we are not able to convert the timestamp to epoch time....
sLog.error("Wrong timestamp", e)
doc["timestamp"] = int(Time.toEpoch()) * 1000
sLog.debug("yielding %s" % doc)
yield doc
def getPlotData( self ):
retVal = self.__parseFormParams()
if not retVal[ 'OK' ]:
c.error = retVal[ 'Message' ]
return render( "/error.mako" )
params = retVal[ 'Value' ]
repClient = ReportsClient( rpcClient = getRPCClient( "Accounting/ReportGenerator" ) )
retVal = repClient.getReport( *params )
if not retVal[ 'OK' ]:
c.error = retVal[ 'Message' ]
return render( "/error.mako" )
rawData = retVal[ 'Value' ]
groupKeys = rawData[ 'data' ].keys()
groupKeys.sort()
if 'granularity' in rawData:
granularity = rawData[ 'granularity' ]
data = rawData['data']
tS = int( Time.toEpoch( params[2] ) )
timeStart = tS - tS % granularity
strData = "epoch,%s\n" % ",".join( groupKeys )
for timeSlot in range( timeStart, int( Time.toEpoch( params[3] ) ), granularity ):
lineData = [ str( timeSlot ) ]
for key in groupKeys:
if timeSlot in data[ key ]:
lineData.append( str( data[ key ][ timeSlot ] ) )
else:
lineData.append( "" )
strData += "%s\n" % ",".join( lineData )
else:
strData = "%s\n" % ",".join( groupKeys )
strData += ",".join( [ str( rawData[ 'data' ][ k ] ) for k in groupKeys ] )
response.headers['Content-type'] = 'text/csv'
response.headers['Content-Disposition'] = 'attachment; filename="%s.csv"' % md5( str( params ) ).hexdigest()
response.headers['Content-Length'] = len( strData )
return strData
def web_getRunningInstancesByImageHistory(self):
try:
bucketSize = int(self.request.arguments['bucketSize'][0])
except:
bucketSize = 900
try:
timespan = int(self.request.arguments['timespan'][0])
except:
timespan = 86400
rpcClient = RPCClient("WorkloadManagement/VirtualMachineManager")
result = rpcClient.getRunningInstancesByImageHistory(
timespan, bucketSize)
if not result['OK']:
callback = {"success": "false", "error": result['Message']}
return self.write(callback)
svcData = result['Value']
data = []
olderThan = Time.toEpoch() - 400
for record in svcData:
eTime = Time.toEpoch(record[0])
if eTime < olderThan:
rL = [eTime, record[1], int(record[2])]
data.append(rL)
callback = {"success": "true", 'data': data}
return self.write(callback)
def web_getRunningInstancesHistory(self):
try:
bucketSize = int(self.request.arguments[ 'bucketSize' ][0])
except:
bucketSize = 900
try:
timespan = int(self.request.arguments[ 'timespan' ][0])
except:
timespan = 86400
rpcClient = RPCClient("WorkloadManagement/VirtualMachineManager")
result = rpcClient.getRunningInstancesHistory(timespan, bucketSize)
if not result[ 'OK' ]:
callback = {"success":"false", "error":result[ 'Message' ]}
return self.write(callback)
svcData = result[ 'Value' ]
data = []
olderThan = Time.toEpoch() - 400
rL = []
for record in svcData:
eTime = Time.toEpoch(record[0])
if eTime < olderThan:
rL = [ eTime, int(record[1]) ]
data.append(rL)
callback = {"success":"true", 'data': data, "timespan": timespan}
return self.write(callback)
def getHistoryValues( self ):
try:
dbVars = [ str( f ) for f in simplejson.loads( request.params[ 'vars' ] ) ]
except:
dbVars = [ 'Load', 'Jobs', 'TransferredFiles' ]
try:
timespan = int( request.params[ 'timespan' ] )
except:
timespan = 86400
rpcClient = getRPCClient( "WorkloadManagement/VirtualMachineManager" )
result = rpcClient.getHistoryValues( 3600, {}, dbVars, timespan )
if not result[ 'OK' ]:
return S_ERROR( result[ 'Message' ] )
svcData = result[ 'Value' ]
data = []
olderThan = Time.toEpoch() - 400
for record in svcData[ 'Records' ]:
rL = []
for iP in range( len( svcData[ 'ParameterNames' ] ) ):
param = svcData[ 'ParameterNames' ][iP]
if param == 'Update':
rL.append( Time.toEpoch( record[iP] ) )
else:
rL.append( record[iP] )
if rL[0] < olderThan:
data.append( rL )
return S_OK( { 'data': data, 'fields' : svcData[ 'ParameterNames' ] } )
def export_commitRegisters(self, entriesList):
"""
Add a record for a type
"""
setup = self.serviceInfoDict['clientSetup']
expectedTypes = [
six.string_types, datetime.datetime, datetime.datetime, list
]
for entry in entriesList:
if len(entry) != 4:
return S_ERROR("Invalid records")
for i, _ in enumerate(entry):
if not isinstance(entry[i], expectedTypes[i]):
self.log.error(
"Unexpected type in report",
": field %d in the records should be %s (and it is %s)"
% (i, expectedTypes[i], type(entry[i])))
return S_ERROR("Unexpected type in report")
records = []
for entry in entriesList:
startTime = int(Time.toEpoch(entry[1]))
endTime = int(Time.toEpoch(entry[2]))
self.log.debug("inserting", entry)
records.append((setup, entry[0], startTime, endTime, entry[3]))
return self.__acDB.insertRecordBundleThroughQueue(records)
def export_commit( self, typeName, startTime, endTime, valuesList ):
"""
Add a record for a type
"""
setup = self.serviceInfoDict[ 'clientSetup' ]
startTime = int( Time.toEpoch( startTime ) )
endTime = int( Time.toEpoch( endTime ) )
return self.__acDB.insertRecordThroughQueue( setup, typeName, startTime, endTime, valuesList )
def export_commit( self, typeName, startTime, endTime, valuesList ):
"""
Add a record for a type
"""
setup = self.serviceInfoDict[ 'clientSetup' ]
startTime = int( Time.toEpoch( startTime ) )
endTime = int( Time.toEpoch( endTime ) )
return self.__acDB.insertRecordThroughQueue( setup, typeName, startTime, endTime, valuesList ) #pylint: disable=too-many-function-args,no-member
def export_remove( self, typeName, startTime, endTime, valuesList ):
"""
Remove a record for a type
"""
setup = self.serviceInfoDict[ 'clientSetup' ]
startTime = int( Time.toEpoch( startTime ) )
endTime = int( Time.toEpoch( endTime ) )
return self.__acDB.deleteRecord( setup, typeName, startTime, endTime, valuesList )
def export_remove( self, typeName, startTime, endTime, valuesList ):
"""
Remove a record for a type
"""
setup = self.serviceInfoDict[ 'clientSetup' ]
startTime = int( Time.toEpoch( startTime ) )
endTime = int( Time.toEpoch( endTime ) )
return self.__acDB.deleteRecord( setup, typeName, startTime, endTime, valuesList ) #pylint: disable=too-many-function-args,no-member
def bulk_index(self, indexprefix, doc_type='_doc', data=None, mapping=None, period=None):
"""
:param str indexPrefix: index name.
:param str doc_type: the type of the document
:param list data: contains a list of dictionary
:param dict mapping: the mapping used by elasticsearch
:param str period: We can specify which kind of indices will be created.
Currently only daily and monthly indexes are supported.
"""
sLog.verbose("Bulk indexing", "%d records will be inserted" % len(data))
if mapping is None:
mapping = {}
indexName = self.generateFullIndexName(indexprefix, period)
sLog.debug("Bulk indexing into %s/%s of %s" % (indexName, doc_type, data))
if not self.exists(indexName):
retVal = self.createIndex(indexprefix, mapping, period)
if not retVal['OK']:
return retVal
docs = []
for row in data:
body = {
'_index': indexName,
'_type': doc_type,
'_source': {}
}
body['_source'] = row
if 'timestamp' not in row:
sLog.warn("timestamp is not given! Note: the actual time is used!")
# if the timestamp is not provided, we use the current utc time.
timestamp = row.get('timestamp', int(Time.toEpoch()))
try:
if isinstance(timestamp, datetime):
body['_source']['timestamp'] = int(timestamp.strftime('%s')) * 1000
elif isinstance(timestamp, six.string_types):
timeobj = datetime.strptime(timestamp, '%Y-%m-%d %H:%M:%S.%f')
body['_source']['timestamp'] = int(timeobj.strftime('%s')) * 1000
else: # we assume the timestamp is an unix epoch time (integer).
body['_source']['timestamp'] = timestamp * 1000
except (TypeError, ValueError) as e:
# in case we are not able to convert the timestamp to epoch time....
sLog.error("Wrong timestamp", e)
body['_source']['timestamp'] = int(Time.toEpoch()) * 1000
docs += [body]
try:
res = bulk(self.__client, docs, chunk_size=self.__chunk_size)
except BulkIndexError as e:
return S_ERROR(e)
if res[0] == len(docs):
# we have inserted all documents...
return S_OK(len(docs))
else:
return S_ERROR(res)
return res
def export_commit(self, typeName, startTime, endTime, valuesList):
"""
Add a record for a type
"""
setup = self.serviceInfoDict['clientSetup']
startTime = int(Time.toEpoch(startTime))
endTime = int(Time.toEpoch(endTime))
return self.__acDB.insertRecordThroughQueue(setup, typeName, startTime,
endTime, valuesList) #pylint: disable=too-many-function-args,no-member
def export_remove(self, typeName, startTime, endTime, valuesList):
"""
Remove a record for a type
"""
setup = self.serviceInfoDict['clientSetup']
startTime = int(Time.toEpoch(startTime))
endTime = int(Time.toEpoch(endTime))
return self.__acDB.deleteRecord(setup, typeName, startTime, endTime,
valuesList) #pylint: disable=too-many-function-args,no-member
def bulk_index(self, indexprefix, doc_type, data, mapping=None, period=None):
"""
:param str indexPrefix: index name.
:param str doc_type: the type of the document
:param list data: contains a list of dictionary
:paran dict mapping: the mapping used by elasticsearch
:param str period: We can specify which kind of indices will be created.
Currently only daily and monthly indexes are supported.
"""
gLogger.info("%d records will be insert to %s" % (len(data), doc_type))
if mapping is None:
mapping = {}
indexName = generateFullIndexName(indexprefix, period)
gLogger.debug("inserting datat to %s index" % indexName)
if not self.exists(indexName):
retVal = self.createIndex(indexprefix, mapping, period)
if not retVal['OK']:
return retVal
docs = []
for row in data:
body = {
'_index': indexName,
'_type': doc_type,
'_source': {}
}
body['_source'] = row
if 'timestamp' not in row:
gLogger.warn("timestamp is not given! Note: the actual time is used!")
# if the timestamp is not provided, we use the current utc time.
timestamp = row.get('timestamp', int(Time.toEpoch()))
try:
if isinstance(timestamp, datetime):
body['_source']['timestamp'] = int(timestamp.strftime('%s')) * 1000
elif isinstance(timestamp, basestring):
timeobj = datetime.strptime(timestamp, '%Y-%m-%d %H:%M:%S.%f')
body['_source']['timestamp'] = int(timeobj.strftime('%s')) * 1000
else: # we assume the timestamp is an unix epoch time (integer).
body['_source']['timestamp'] = timestamp * 1000
except (TypeError, ValueError) as e:
# in case we are not able to convert the timestamp to epoch time....
gLogger.error("Wrong timestamp", e)
body['_source']['timestamp'] = int(Time.toEpoch()) * 1000
docs += [body]
try:
res = bulk(self.__client, docs, chunk_size=self.__chunk_size)
except BulkIndexError as e:
return S_ERROR(e)
if res[0] == len(docs):
# we have inserted all documents...
return S_OK(len(docs))
else:
return S_ERROR(res)
return res
def export_remove( self, typeName, startTime, endTime, valuesList ):
"""
Remove a record for a type
"""
setup = self.serviceInfoDict[ 'clientSetup' ]
typeName = "%s_%s" % ( setup, typeName )
startTime = int( Time.toEpoch( startTime ) )
endTime = int( Time.toEpoch( endTime ) )
return gAccountingDB.deleteRecord( typeName, startTime, endTime, valuesList )
def bulk_index(self, indexprefix, doc_type, data, mapping=None):
"""
:param str indexPrefix: it is the index name.
:param str doc_type: the type of the document
:param data: contains a list of dictionary
:type data: python:list
"""
gLogger.info("%d records will be insert to %s" % (len(data), doc_type))
if mapping is None:
mapping = {}
indexName = generateFullIndexName(indexprefix)
gLogger.debug("inserting datat to %s index" % indexName)
if not self.exists(indexName):
retVal = self.createIndex(indexprefix, mapping)
if not retVal['OK']:
return retVal
docs = []
for row in data:
body = {'_index': indexName, '_type': doc_type, '_source': {}}
body['_source'] = row
if 'timestamp' not in row:
gLogger.warn(
"timestamp is not given! Note: the actual time is used!")
timestamp = row.get(
'timestamp', int(Time.toEpoch())
) #if the timestamp is not provided, we use the current utc time.
try:
if isinstance(timestamp, datetime):
body['_source']['timestamp'] = int(
timestamp.strftime('%s')) * 1000
elif isinstance(timestamp, basestring):
timeobj = datetime.strptime(timestamp,
'%Y-%m-%d %H:%M:%S.%f')
body['_source']['timestamp'] = int(
timeobj.strftime('%s')) * 1000
else: #we assume the timestamp is an unix epoch time (integer).
body['_source']['timestamp'] = timestamp * 1000
except (TypeError, ValueError) as e:
# in case we are not able to convert the timestamp to epoch time....
gLogger.error("Wrong timestamp", e)
body['_source']['timestamp'] = int(Time.toEpoch()) * 1000
docs += [body]
try:
res = bulk(self.__client, docs, chunk_size=self.__chunk_size)
except BulkIndexError as e:
return S_ERROR(e)
if res[0] == len(docs):
# we have inserted all documents...
return S_OK(len(docs))
else:
return S_ERROR(res)
return res
def export_remove(self, typeName, startTime, endTime, valuesList):
"""
Remove a record for a type
"""
setup = self.serviceInfoDict['clientSetup']
typeName = "%s_%s" % (setup, typeName)
startTime = int(Time.toEpoch(startTime))
endTime = int(Time.toEpoch(endTime))
return gAccountingDB.deleteRecord(typeName, startTime, endTime,
valuesList)
def plotView(self):
"""
Plot a saved view
"""
plotRequest = {}
try:
webRequest = simplejson.loads(request.params['plotRequest'])
if 'id' not in webRequest:
return S_ERROR("Missing viewID in plot request")
plotRequest['id'] = webRequest['id']
if 'size' not in webRequest:
return S_ERROR("Missing plotsize in plot request")
plotRequest['size'] = webRequest['size']
if 'time' not in webRequest:
return S_ERROR("Missing time span in plot request")
timeReq = webRequest['time']
if timeReq['timespan'] < 0:
toSecs = self.__dateToSecs(str(timeReq['toDate']))
fromSecs = self.__dateToSecs(str(timeReq['fromDate']))
else:
toSecs = int(Time.toEpoch())
fromSecs = toSecs - timeReq['timespan']
plotRequest['fromSecs'] = fromSecs
plotRequest['toSecs'] = toSecs
if 'varData' in webRequest:
plotRequest['varData'] = webRequest['varData']
except Exception, e:
return self.__translateToExpectedExtResult(
S_ERROR("Error while processing plot parameters: %s" % str(e)))
def tryView(self):
"""
Try plotting graphs for a view
"""
try:
plotRequest = simplejson.loads(request.params['plotRequest'])
if 'timeLength' in request.params:
timeLength = str(request.params['timeLength'])
toSecs = int(Time.toEpoch())
if timeLength == "hour":
fromSecs = toSecs - 3600
elif timeLength == "day":
fromSecs = toSecs - 86400
elif timeLength == "month":
fromSecs = toSecs - 2592000
elif fromSecs == "year":
fromDate = toSecs - 31104000
else:
return S_ERROR("Time length value not valid")
else:
fromDate = str(request.params['fromDate'])
toDate = str(request.params['toDate'])
fromSecs = self.__dateToSecs(fromDate)
toSecs = self.__dateToSecs(toDate)
except Exception, e:
return S_ERROR("Error while processing plot parameters: %s" %
str(e))
def getActivitiesList(self):
try:
start = int(request.params['start'])
except:
start = 0
try:
limit = int(request.params['limit'])
except:
limit = 0
try:
sortField = str(request.params['sortField']).replace("_", ".")
sortDir = str(request.params['sortDirection'])
sort = [(sortField, sortDir)]
except:
sort = []
rpcClient = getRPCClient("Framework/Monitoring")
retVal = rpcClient.getActivitiesContents({}, sort, start, limit)
if not retVal['OK']:
return retVal
svcData = retVal['Value']
data = {'numActivities': svcData['TotalRecords'], 'activities': []}
now = Time.toEpoch()
for record in svcData['Records']:
formatted = {}
for i in range(len(svcData['Fields'])):
formatted[svcData['Fields'][i].replace(".", "_")] = record[i]
if 'activities_lastUpdate' in formatted:
formatted['activities_lastUpdate'] = now - int(
formatted['activities_lastUpdate'])
data['activities'].append(formatted)
return data
def web_plotView( self ):
plotRequest = {}
try:
if 'id' not in self.request.arguments:
self.finish( { 'success' : "false", 'error' : "Missing viewID in plot request" } )
return
plotRequest[ 'id' ] = self.request.arguments[ 'id' ][0]
if 'size' not in self.request.arguments:
self.finish( { 'success' : "false", 'error' : "Missing plotsize in plot request" } )
return
plotRequest[ 'size' ] = int( self.request.arguments[ 'size' ][0] )
timespan = int( self.request.arguments[ 'timespan' ][0] )
if timespan < 0:
toSecs = self.__dateToSecs( str( self.request.arguments[ 'toDate' ][0] ) )
fromSecs = self.__dateToSecs( str( self.request.arguments[ 'fromDate' ][0] ) )
else:
toSecs = int( Time.toEpoch() )
fromSecs = toSecs - timespan
plotRequest[ 'fromSecs' ] = fromSecs
plotRequest[ 'toSecs' ] = toSecs
if 'varData' in self.request.arguments:
plotRequest[ 'varData' ] = dict( json.loads( self.request.arguments[ 'varData' ][0] ) )
except Exception, e:
self.finish( { 'success' : "false", 'error' : "Error while processing plot parameters: %s" % str( e ) } )
return
def __consolidateMarks(self, allData):
"""
Copies all marks except last step ones and consolidates them.
:type allData: bool
:param allData: This is used to indicate whether all the data is present or not.
:return: dictionary of consolidatedMarks.
"""
consolidatedMarks = {}
remainderMarks = {}
for key in self.activitiesMarks:
if allData:
lastStepToSend = int(Time.toEpoch())
else:
lastStepToSend = self.__UTCStepTime(key)
consolidatedMarks[key] = {}
remainderMarks[key] = {}
for markTime in self.activitiesMarks[key]:
markValue = self.activitiesMarks[key][markTime]
if markTime >= lastStepToSend:
remainderMarks[key][markTime] = markValue
else:
consolidatedMarks[key][markTime] = markValue
# Consolidate the copied ones
totalValue = 0
for mark in consolidatedMarks[key][markTime]:
totalValue += mark
if self.activitiesDefinitions[key]["type"] == self.OP_MEAN:
totalValue /= len(consolidatedMarks[key][markTime])
consolidatedMarks[key][markTime] = totalValue
if len(consolidatedMarks[key]) == 0:
del consolidatedMarks[key]
self.activitiesMarks = remainderMarks
return consolidatedMarks
def _mbReceivedMsg(self, trid, msgObj):
result = self._authorizeProposal(
("Message", msgObj.getName()), trid,
self._transportPool.get(trid).getConnectingCredentials())
if not result["OK"]:
return result
result = self._instantiateHandler(trid)
if not result["OK"]:
return result
handlerObj = result["Value"]
response = handlerObj._rh_executeMessageCallback(msgObj)
if self.activityMonitoring and response["OK"]:
self.activityMonitoringReporter.addRecord({
"timestamp":
int(Time.toEpoch()),
"host":
Network.getFQDN(),
"componentType":
"service",
"component":
"_".join(self._name.split("/")),
"componentLocation":
self._cfg.getURL(),
"ServiceResponseTime":
response["Value"][1],
})
if response["OK"]:
return response["Value"][0]
else:
return response
def registerActivity(self, sourceId, acName, acDict):
"""
Register an activity.
:type sourceId: string
:param sourceId: The source id.
:type acName: string
:param acName: name of the activity.
:type acDict: dictionary
:param acDict: The activity dictionary containing information about 'category', 'description', 'bucketLength',
'type', 'unit'.
:return: a list of values.
"""
m = hashlib.md5()
acDict['name'] = acName
acDict['sourceId'] = sourceId
m.update(str(acDict).encode())
retList = self.__select("filename", "activities", acDict)
if len(retList) > 0:
return retList[0][0]
else:
acDict['lastUpdate'] = int(Time.toEpoch() - 86000)
filePath = m.hexdigest()
filePath = "%s/%s.rrd" % (filePath[:2], filePath)
self.log.info("Registering activity", str(acDict))
# This is basically called by the ServiceInterface inside registerActivities method and then all the activity
# information is stored in the sqlite3 db using the __insert method.
if self.__insert("activities", {
'id': 'NULL',
'filename': "'%s'" % filePath,
}, acDict) == 0:
return -1
return self.__select("filename", "activities", acDict)[0][0]
def web_plotView(self):
plotRequest = {}
try:
if 'id' not in self.request.arguments:
self.finish({ 'success' : "false", 'error' : "Missing viewID in plot request" })
return
plotRequest[ 'id' ] = self.request.arguments[ 'id' ][0]
if 'size' not in self.request.arguments:
self.finish({ 'success' : "false", 'error' : "Missing plotsize in plot request" })
return
plotRequest[ 'size' ] = int(self.request.arguments[ 'size' ][0])
timespan = int(self.request.arguments[ 'timespan' ][0])
if timespan < 0:
toSecs = self.__dateToSecs(str(self.request.arguments[ 'toDate' ][0]))
fromSecs = self.__dateToSecs(str(self.request.arguments[ 'fromDate' ][0]))
else:
toSecs = int(Time.toEpoch())
fromSecs = toSecs - timespan
plotRequest[ 'fromSecs' ] = fromSecs
plotRequest[ 'toSecs' ] = toSecs
if 'varData' in self.request.arguments:
plotRequest[ 'varData' ] = self.request.arguments[ 'varData' ][0]
except Exception, e:
self.finish({ 'success' : "false", 'error' : "Error while processing plot parameters: %s" % str(e) })
return
def web_tryView(self):
"""
Try plotting graphs for a view
"""
try:
plotRequest = json.loads(self.request.arguments[ 'plotRequest' ][0])
if 'timeLength' in self.request.arguments:
timeLength = str(self.request.arguments[ 'timeLength' ][0])
toSecs = int(Time.toEpoch())
if timeLength == "hour":
fromSecs = toSecs - 3600
elif timeLength == "day":
fromSecs = toSecs - 86400
elif timeLength == "month":
fromSecs = toSecs - 2592000
elif fromSecs == "year":
fromDate = toSecs - 31104000
else:
self.finish({"success":"false", "error":"Time length value not valid"})
return
else:
fromDate = str(self.request.arguments[ 'fromDate' ][0])
toDate = str(self.request.arguments[ 'toDate' ][0])
fromSecs = self.__dateToSecs(fromDate)
toSecs = self.__dateToSecs(toDate)
except Exception, e:
self.finish({"success":"false", "error":"Error while processing plot parameters: %s" % str(e)})
return
def tryView( self ):
"""
Try plotting graphs for a view
"""
try:
plotRequest = simplejson.loads( request.params[ 'plotRequest' ] )
if 'timeLength' in request.params:
timeLength = str( request.params[ 'timeLength' ] )
toSecs = int( Time.toEpoch() )
if timeLength == "hour":
fromSecs = toSecs - 3600
elif timeLength == "day":
fromSecs = toSecs - 86400
elif timeLength == "month":
fromSecs = toSecs - 2592000
elif fromSecs == "year":
fromDate = toSecs - 31104000
else:
return S_ERROR( "Time length value not valid" )
else:
fromDate = str( request.params[ 'fromDate' ] )
toDate = str( request.params[ 'toDate' ] )
fromSecs = self.__dateToSecs( fromDate )
toSecs = self.__dateToSecs( toDate )
except Exception, e:
return S_ERROR( "Error while processing plot parameters: %s" % str( e ) )
def __consolidateMarks(self, allData):
"""
Copies all marks except last step ones
and consolidates them
"""
consolidatedMarks = {}
remainderMarks = {}
for key in self.activitiesMarks:
if allData:
lastStepToSend = int(Time.toEpoch())
else:
lastStepToSend = self.__UTCStepTime(key)
consolidatedMarks[key] = {}
remainderMarks[key] = {}
for markTime in self.activitiesMarks[key]:
markValue = self.activitiesMarks[key][markTime]
if markTime >= lastStepToSend:
remainderMarks[key][markTime] = markValue
else:
consolidatedMarks[key][markTime] = markValue
# Consolidate the copied ones
totalValue = 0
for mark in consolidatedMarks[key][markTime]:
totalValue += mark
if self.activitiesDefinitions[key]['type'] == self.OP_MEAN:
totalValue /= len(consolidatedMarks[key][markTime])
consolidatedMarks[key][markTime] = totalValue
if len(consolidatedMarks[key]) == 0:
del (consolidatedMarks[key])
self.activitiesMarks = remainderMarks
return consolidatedMarks
def web_tryView( self ):
"""
Try plotting graphs for a view
"""
try:
plotRequest = json.loads( self.request.arguments[ 'plotRequest' ][0] )
if 'timeLength' in self.request.arguments:
timeLength = str( self.request.arguments[ 'timeLength' ][0] )
toSecs = int( Time.toEpoch() )
if timeLength == "hour":
fromSecs = toSecs - 3600
elif timeLength == "day":
fromSecs = toSecs - 86400
elif timeLength == "month":
fromSecs = toSecs - 2592000
elif fromSecs == "year":
fromDate = toSecs - 31104000
else:
self.finish( {"success":"false", "error":"Time length value not valid"} )
return
else:
fromDate = str( self.request.arguments[ 'fromDate' ][0] )
toDate = str( self.request.arguments[ 'toDate' ][0] )
fromSecs = self.__dateToSecs( fromDate )
toSecs = self.__dateToSecs( toDate )
except Exception, e:
self.finish( {"success":"false", "error":"Error while processing plot parameters: %s" % str( e )} )
return
def filterOngoing(selectOutput):
"""
Selects all the ongoing downtimes
"""
downtimes = selectOutput
downtimesFiltered = []
currentDate = Time.toEpoch(Time.dateTime())
for dt in downtimes:
dtStart = Time.toEpoch(dt['startDate'])
dtEnd = Time.toEpoch(dt['endDate'])
if (dtStart <= currentDate) and (dtEnd >= currentDate):
downtimesFiltered.append(dt)
return downtimesFiltered
def __consolidateMarks( self, allData ):
"""
Copies all marks except last step ones
and consolidates them
"""
consolidatedMarks = {}
remainderMarks = {}
for key in self.activitiesMarks:
if allData:
lastStepToSend = int( Time.toEpoch() )
else:
lastStepToSend = self.__UTCStepTime( key )
consolidatedMarks[ key ] = {}
remainderMarks [ key ] = {}
for markTime in self.activitiesMarks[ key ]:
markValue = self.activitiesMarks[ key ][ markTime ]
if markTime >= lastStepToSend:
remainderMarks[ key ][ markTime ] = markValue
else:
consolidatedMarks[ key ][ markTime ] = markValue
# Consolidate the copied ones
totalValue = 0
for mark in consolidatedMarks[ key ][ markTime ]:
totalValue += mark
if self.activitiesDefinitions[ key ][ 'type' ] == self.OP_MEAN:
totalValue /= len( consolidatedMarks[ key ][ markTime ] )
consolidatedMarks[ key ][ markTime ] = totalValue
if len( consolidatedMarks[ key ] ) == 0:
del( consolidatedMarks[ key ] )
self.activitiesMarks = remainderMarks
return consolidatedMarks
def plotView( self ):
"""
Plot a saved view
"""
plotRequest = {}
try:
webRequest = simplejson.loads( request.params[ 'plotRequest' ] )
if 'id' not in webRequest:
return S_ERROR( "Missing viewID in plot request" )
plotRequest[ 'id' ] = webRequest[ 'id' ]
if 'size' not in webRequest:
return S_ERROR( "Missing plotsize in plot request" )
plotRequest[ 'size' ] = webRequest[ 'size' ]
if 'time' not in webRequest:
return S_ERROR( "Missing time span in plot request" )
timeReq = webRequest[ 'time' ]
if timeReq[ 'timespan' ] < 0:
toSecs = self.__dateToSecs( str( timeReq[ 'toDate' ] ) )
fromSecs = self.__dateToSecs( str( timeReq[ 'fromDate' ] ) )
else:
toSecs = int( Time.toEpoch() )
fromSecs = toSecs - timeReq[ 'timespan' ]
plotRequest[ 'fromSecs' ] = fromSecs
plotRequest[ 'toSecs' ] = toSecs
if 'varData' in webRequest:
plotRequest[ 'varData' ] = webRequest[ 'varData' ]
except Exception, e:
return self.__translateToExpectedExtResult( S_ERROR( "Error while processing plot parameters: %s" % str( e ) ) )
def getActivitiesList( self ):
try:
start = int( request.params[ 'start' ] )
except:
start = 0
try:
limit = int( request.params[ 'limit' ] )
except:
limit = 0
try:
sortField = str( request.params[ 'sortField' ] ).replace( "_", "." )
sortDir = str( request.params[ 'sortDirection' ] )
sort = [ ( sortField, sortDir ) ]
except:
sort = []
rpcClient = getRPCClient( "Framework/Monitoring" )
retVal = rpcClient.getActivitiesContents( {}, sort, start, limit )
if not retVal[ 'OK' ]:
return retVal
svcData = retVal[ 'Value' ]
data = { 'numActivities' : svcData[ 'TotalRecords' ], 'activities' : [] }
now = Time.toEpoch()
for record in svcData[ 'Records' ]:
formatted = {}
for i in range( len( svcData[ 'Fields' ] ) ):
formatted[ svcData[ 'Fields' ][i].replace( ".", "_" ) ] = record[i]
if 'activities_lastUpdate' in formatted:
formatted[ 'activities_lastUpdate' ] = now - int( formatted[ 'activities_lastUpdate' ] )
data[ 'activities' ].append( formatted )
return data
def filterOngoing( selectOutput ):
'''
Selects all the ongoing downtimes
'''
downtimes = selectOutput
downtimesFiltered = []
currentDate = Time.toEpoch( Time.dateTime() )
for dt in downtimes:
dtStart = Time.toEpoch( dt[ 'startDate' ] )
dtEnd = Time.toEpoch( dt[ 'endDate' ] )
if ( dtStart <= currentDate ) and ( dtEnd >= currentDate ):
downtimesFiltered.append( dt )
return downtimesFiltered
def getUniqueValue(self, indexName, key, orderBy=False):
"""
:param str indexName: the name of the index which will be used for the query
:param dict orderBy: it is a dictionary in case we want to order the result {key:'desc'} or {key:'asc'}
:returns: a list of unique value for a certain key from the dictionary.
"""
query = self._Search(indexName)
endDate = datetime.utcnow()
startDate = endDate - timedelta(days=30)
timeFilter = self._Q('range',
timestamp={
'lte': int(Time.toEpoch(endDate)) * 1000,
'gte': int(Time.toEpoch(startDate)) * 1000,
})
query = query.filter('bool', must=timeFilter)
if orderBy:
query.aggs.bucket(key,
'terms',
field=key,
size=self.RESULT_SIZE,
order=orderBy).metric(key,
'cardinality',
field=key)
else:
query.aggs.bucket(key, 'terms', field=key,
size=self.RESULT_SIZE).metric(key,
'cardinality',
field=key)
try:
query = query.extra(
size=self.RESULT_SIZE) # do not need the raw data.
sLog.debug("Query", query.to_dict())
result = query.execute()
except TransportError as e:
return S_ERROR(e)
values = []
for bucket in result.aggregations[key].buckets:
values += [bucket['key']]
del query
sLog.debug("Nb of unique rows retrieved", len(values))
return S_OK(values)
def getUniqueValue(self, indexName, key, orderBy=False):
"""
:param str indexName the name of the index which will be used for the query
:param dict orderBy it is a dictionary in case we want to order the result {key:'desc'} or {key:'asc'}
It returns a list of unique value for a certain key from the dictionary.
"""
query = self._Search(indexName)
endDate = datetime.utcnow()
startDate = endDate - timedelta(days=30)
timeFilter = self._Q('range',
timestamp={'lte': int(Time.toEpoch(endDate)) * 1000,
'gte': int(Time.toEpoch(startDate)) * 1000, })
query = query.filter('bool', must=timeFilter)
if orderBy:
query.aggs.bucket(key,
'terms',
field=key,
size=self.RESULT_SIZE,
order=orderBy).metric(key,
'cardinality',
field=key)
else:
query.aggs.bucket(key,
'terms',
field=key,
size=self.RESULT_SIZE).metric(key,
'cardinality',
field=key)
try:
query = query.extra(size=self.RESULT_SIZE) # do not need the raw data.
gLogger.debug("Query", query.to_dict())
result = query.execute()
except TransportError as e:
return S_ERROR(e)
values = []
for bucket in result.aggregations[key].buckets:
values += [bucket['key']]
del query
gLogger.debug("Nb of unique rows retrieved", len(values))
return S_OK(values)
def export_commitRegisters( self, entriesList ):
"""
Add a record for a type
"""
setup = self.serviceInfoDict[ 'clientSetup' ]
expectedTypes = [ types.StringType, Time._dateTimeType, Time._dateTimeType, types.ListType ]
for entry in entriesList:
if len( entry ) != 4:
return S_ERROR( "Invalid records" )
for i in range( len( entry ) ):
if type( entry[i] ) != expectedTypes[i]:
return S_ERROR( "%s field in the records should be %s" % ( i, expectedTypes[i] ) )
records = []
for entry in entriesList:
startTime = int( Time.toEpoch( entry[1] ) )
endTime = int( Time.toEpoch( entry[2] ) )
records.append( ( setup, entry[0], startTime, endTime, entry[3] ) )
return self.__acDB.insertRecordBundleThroughQueue( records )
def execute(self):
""" Main execution method
"""
# Get the WMS Snapshot!
result = JobDB().getSummarySnapshot(self.__jobDBFields)
now = Time.dateTime()
if not result['OK']:
self.log.error(
"Can't get the JobDB summary",
"%s: won't commit at this cycle" % result['Message'])
return S_ERROR()
# Now we try to commit
values = result['Value'][1]
self.log.info("Start sending records")
for record in values:
record = record[1:]
rD = {}
for fV in self.__summaryDefinedFields:
rD[fV[0]] = fV[1]
for iP in range(len(self.__summaryKeyFieldsMapping)):
fieldName = self.__summaryKeyFieldsMapping[iP]
rD[self.__renameFieldsMapping.get(fieldName,
fieldName)] = record[iP]
record = record[len(self.__summaryKeyFieldsMapping):]
for iP in range(len(self.__summaryValueFieldsMapping)):
rD[self.__summaryValueFieldsMapping[iP]] = int(record[iP])
for backend in self.datastores:
if backend.lower() == 'monitoring':
rD['timestamp'] = int(Time.toEpoch(now))
self.datastores['Monitoring'].addRecord(rD)
elif backend.lower() == 'accounting':
acWMS = WMSHistory()
acWMS.setStartTime(now)
acWMS.setEndTime(now)
acWMS.setValuesFromDict(rD)
retVal = acWMS.checkValues()
if not retVal['OK']:
self.log.error("Invalid accounting record ",
"%s -> %s" % (retVal['Message'], rD))
else:
self.datastores['Accounting'].addRegister(acWMS)
for backend, datastore in self.datastores.items():
self.log.info("Committing to %s backend" % backend)
result = datastore.commit()
if not result['OK']:
self.log.error("Couldn't commit WMS history to %s" % backend,
result['Message'])
return S_ERROR()
self.log.verbose("Done committing to %s backend" % backend)
return S_OK()
def export_commitRegisters( self, entriesList ):
"""
Add a record for a type
"""
setup = self.serviceInfoDict[ 'clientSetup' ]
expectedTypes = [ basestring, datetime.datetime, datetime.datetime, list ]
for entry in entriesList:
if len( entry ) != 4:
return S_ERROR( "Invalid records" )
for i in range( len( entry ) ):
if not isinstance(entry[i], expectedTypes[i]):
gLogger.error( "Unexpected type in report",
": field %d in the records should be %s (and it is %s)" % ( i, expectedTypes[i], type(entry[i])) )
return S_ERROR( "Unexpected type in report" )
records = []
for entry in entriesList:
startTime = int( Time.toEpoch( entry[1] ) )
endTime = int( Time.toEpoch( entry[2] ) )
records.append( ( setup, entry[0], startTime, endTime, entry[3] ) )
return self.__acDB.insertRecordBundleThroughQueue( records )
def web_getCsvPlotData(self):
callback = {}
retVal = self.__parseFormParams()
if not retVal['OK']:
callback = {"success": "false", "error": retVal['Message']}
self.finish(callback)
params = retVal['Value']
repClient = ReportsClient(
rpcClient=RPCClient("Accounting/ReportGenerator"))
retVal = yield self.threadTask(repClient.getReport, *params)
if not retVal['OK']:
callback = {"success": "false", "error": retVal['Message']}
self.finish(callback)
rawData = retVal['Value']
groupKeys = rawData['data'].keys()
groupKeys.sort()
# print rawData['data']
if 'granularity' in rawData:
granularity = rawData['granularity']
data = rawData['data']
tS = int(Time.toEpoch(params[2]))
timeStart = tS - tS % granularity
strData = "epoch,%s\n" % ",".join(groupKeys)
for timeSlot in range(timeStart, int(Time.toEpoch(params[3])),
granularity):
lineData = [str(timeSlot)]
for key in groupKeys:
if timeSlot in data[key]:
lineData.append(str(data[key][timeSlot]))
else:
lineData.append("")
strData += "%s\n" % ",".join(lineData)
else:
strData = "%s\n" % ",".join(groupKeys)
strData += ",".join([str(rawData['data'][k]) for k in groupKeys])
self.set_header('Content-type', 'text/csv')
self.set_header(
'Content-Disposition',
'attachment; filename="%s.csv"' % md5(str(params)).hexdigest())
self.set_header('Content-Length', len(strData))
self.finish(strData)
def getRunningInstancesHistory(self):
try:
bucketSize = int(request.params['bucketSize'])
except:
bucketSize = 900
try:
timespan = int(request.params['timespan'])
except:
timespan = 86400
rpcClient = getRPCClient("WorkloadManagement/VirtualMachineManager")
result = rpcClient.getRunningInstancesHistory(timespan, bucketSize)
if not result['OK']:
return S_ERROR(result['Message'])
svcData = result['Value']
data = []
olderThan = Time.toEpoch() - 400
for record in svcData:
eTime = Time.toEpoch(record[0])
if eTime < olderThan:
rL = [eTime, int(record[1])]
data.append(rL)
return S_OK(data)
def getRunningInstancesHistory( self ):
try:
bucketSize = int( request.params[ 'bucketSize' ] )
except:
bucketSize = 900
try:
timespan = int( request.params[ 'timespan' ] )
except:
timespan = 86400
rpcClient = getRPCClient( "WorkloadManagement/VirtualMachineManager" )
result = rpcClient.getRunningInstancesHistory( timespan, bucketSize )
if not result[ 'OK' ]:
return S_ERROR( result[ 'Message' ] )
svcData = result[ 'Value' ]
data = []
olderThan = Time.toEpoch() - 400
for record in svcData:
eTime = Time.toEpoch( record[0] )
if eTime < olderThan:
rL = [ eTime, int( record[1] ) ]
data.append( rL )
return S_OK( data )
def export_removeRegisters( self, entriesList ):
"""
Remove a record for a type
"""
setup = self.serviceInfoDict[ 'clientSetup' ]
expectedTypes = [ types.StringType, Time._dateTimeType, Time._dateTimeType, types.ListType ]
for entry in entriesList:
if len( entry ) != 4:
return S_ERROR( "Invalid records" )
for i in range( len( entry ) ):
if type( entry[i] ) != expectedTypes[i]:
return S_ERROR( "%s field in the records should be %s" % ( i, expectedTypes[i] ) )
ok = 0
for entry in entriesList:
startTime = int( Time.toEpoch( entry[1] ) )
endTime = int( Time.toEpoch( entry[2] ) )
record = entry[3]
result = self.__acDB.deleteRecord( setup, entry[0], startTime, endTime, record )
if not result[ 'OK' ]:
return S_OK( ok )
ok += 1
return S_OK( ok )
def web_getCsvPlotData(self):
callback = {}
retVal = self.__parseFormParams()
if not retVal['OK']:
callback = {"success": "false", "error": retVal['Message']}
self.finish(callback)
params = retVal['Value']
repClient = ReportsClient(rpcClient=RPCClient("Accounting/ReportGenerator"))
retVal = yield self.threadTask(repClient.getReport, *params)
if not retVal['OK']:
callback = {"success": "false", "error": retVal['Message']}
self.finish(callback)
rawData = retVal['Value']
groupKeys = rawData['data'].keys()
groupKeys.sort()
# print rawData['data']
if 'granularity' in rawData:
granularity = rawData['granularity']
data = rawData['data']
tS = int(Time.toEpoch(params[2]))
timeStart = tS - tS % granularity
strData = "epoch,%s\n" % ",".join(groupKeys)
for timeSlot in range(timeStart, int(Time.toEpoch(params[3])), granularity):
lineData = [str(timeSlot)]
for key in groupKeys:
if timeSlot in data[key]:
lineData.append(str(data[key][timeSlot]))
else:
lineData.append("")
strData += "%s\n" % ",".join(lineData)
else:
strData = "%s\n" % ",".join(groupKeys)
strData += ",".join([str(rawData['data'][k]) for k in groupKeys])
self.set_header('Content-type', 'text/csv')
self.set_header('Content-Disposition', 'attachment; filename="%s.csv"' % md5(str(params)).hexdigest())
self.set_header('Content-Length', len(strData))
self.finish(strData)
def export_commitMarks( self, sourceId, activitiesDict, componentExtraInfo = {} ):
"""
Adds marks for activities
"""
nowEpoch = Time.toEpoch()
maxEpoch = nowEpoch + 7200
minEpoch = nowEpoch - 86400
invalidActivities = []
for acName in activitiesDict:
for time in activitiesDict[ acName ]:
if time > maxEpoch or time < minEpoch:
gLogger.info( "Time %s ( [%s,%s] ) is invalid for activity %s" % ( time, minEpoch, maxEpoch, acName ) )
invalidActivities.append( acName )
break
for acName in invalidActivities:
gLogger.info( "Not commiting activity %s" % acName )
del( activitiesDict[ acName ] )
return gServiceInterface.commitMarks( sourceId, activitiesDict, componentExtraInfo )
def __checkPlotRequest( self, reportRequest ):
"""
It check the plot attributes. We have to make sure that all attributes which are needed are provided.
:param dict reportRequest contains the plot attributes.
"""
# If extraArgs is not there add it
if 'extraArgs' not in reportRequest:
reportRequest[ 'extraArgs' ] = {}
if not isinstance( reportRequest[ 'extraArgs' ], self.__reportRequestDict[ 'extraArgs' ] ):
return S_ERROR( "Extra args has to be of type %s" % self.__reportRequestDict[ 'extraArgs' ] )
reportRequestExtra = reportRequest[ 'extraArgs' ]
# Check sliding plots
if 'lastSeconds' in reportRequestExtra:
try:
lastSeconds = long( reportRequestExtra[ 'lastSeconds' ] )
except ValueError:
gLogger.error( "lastSeconds key must be a number" )
return S_ERROR( "Value Error" )
if lastSeconds < 3600:
return S_ERROR( "lastSeconds must be more than 3600" )
now = Time.dateTime() #this is an UTC time
reportRequest[ 'endTime' ] = now
reportRequest[ 'startTime' ] = now - datetime.timedelta( seconds = lastSeconds )
else:
# if end date is not there, just set it to now
if not reportRequest.get( 'endTime' ):
# check the existence of the endTime it can be present and empty
reportRequest[ 'endTime' ] = Time.dateTime()
# Check keys
for key in self.__reportRequestDict:
if not key in reportRequest:
return S_ERROR( 'Missing mandatory field %s in plot request' % key )
if not isinstance( reportRequest[ key ], self.__reportRequestDict[ key ] ):
return S_ERROR( "Type mismatch for field %s (%s), required one of %s" % ( key,
str( type( reportRequest[ key ] ) ),
str( self.__reportRequestDict[ key ] ) ) )
if key in ( 'startTime', 'endTime' ):
reportRequest[ key ] = int( Time.toEpoch( reportRequest[ key ] ) )
return S_OK( reportRequest )
def __checkPlotRequest(self, reportRequest):
# If extraArgs is not there add it
if "extraArgs" not in reportRequest:
reportRequest["extraArgs"] = {}
if type(reportRequest["extraArgs"]) != self.__reportRequestDict["extraArgs"]:
return S_ERROR("Extra args has to be of type %s" % self.__reportRequestDict["extraArgs"])
reportRequestExtra = reportRequest["extraArgs"]
# Check sliding plots
if "lastSeconds" in reportRequestExtra:
try:
lastSeconds = long(reportRequestExtra["lastSeconds"])
except:
return S_ERROR("lastSeconds key must be a number")
if lastSeconds < 3600:
return S_ERROR("lastSeconds must be more than 3600")
now = Time.dateTime()
reportRequest["endTime"] = now
reportRequest["startTime"] = now - datetime.timedelta(seconds=lastSeconds)
else:
# if enddate is not there, just set it to now
if not reportRequest.get("endTime", False):
reportRequest["endTime"] = Time.dateTime()
# Check keys
for key in self.__reportRequestDict:
if not key in reportRequest:
return S_ERROR("Missing mandatory field %s in plot reques" % key)
requestKeyType = type(reportRequest[key])
if key in ("startTime", "endTime"):
if requestKeyType not in self.__reportRequestDict[key]:
return S_ERROR(
"Type mismatch for field %s (%s), required one of %s"
% (key, str(requestKeyType), str(self.__reportRequestDict[key]))
)
reportRequest[key] = int(Time.toEpoch(reportRequest[key]))
else:
if requestKeyType != self.__reportRequestDict[key]:
return S_ERROR(
"Type mismatch for field %s (%s), required %s"
% (key, str(requestKeyType), str(self.__reportRequestDict[key]))
)
return S_OK(reportRequest)
def execute( self ):
""" Main execution method
"""
result = gConfig.getSections( "/DIRAC/Setups" )
if not result[ 'OK' ]:
return result
validSetups = result[ 'Value' ]
self.log.info( "Valid setups for this cycle are %s" % ", ".join( validSetups ) )
# Get the WMS Snapshot!
result = self.jobDB.getSummarySnapshot( self.__jobDBFields )
now = Time.dateTime()
if not result[ 'OK' ]:
self.log.error( "Can't get the jobdb summary", result[ 'Message' ] )
else:
values = result[ 'Value' ][1]
self.log.info( "Start sending records!" )
for record in values:
recordSetup = record[0]
if recordSetup not in validSetups:
self.log.error( "Setup %s is not valid" % recordSetup )
continue
record = record[1:]
rD = {}
for fV in self.__summaryDefinedFields:
rD[ fV[0] ] = fV[1]
for iP in range( len( self.__summaryKeyFieldsMapping ) ):
fieldName = self.__summaryKeyFieldsMapping[iP]
rD[ self.__renameFieldsMapping.get( fieldName, fieldName ) ] = record[iP]
record = record[ len( self.__summaryKeyFieldsMapping ): ]
for iP in range( len( self.__summaryValueFieldsMapping ) ):
rD[ self.__summaryValueFieldsMapping[iP] ] = int( record[iP] )
rD['timestamp'] = int( Time.toEpoch( now ) )
self.monitoringReporter.addRecord( rD )
retVal = self.monitoringReporter.commit()
if retVal['OK']:
self.log.info( "The records are successfully sent to the Store!" )
else:
self.log.warn( "Faild to insert the records! It will be retried in the next iteration", retVal['Message'] )
return S_OK()
def registerActivity( self, sourceId, acName, acDict ):
"""
Register an activity
"""
m = hashlib.md5()
acDict[ 'name' ] = acName
acDict[ 'sourceId' ] = sourceId
m.update( str( acDict ) )
retList = self.__select( "filename", "activities", acDict )
if len( retList ) > 0:
return retList[0][0]
else:
acDict[ 'lastUpdate' ] = int( Time.toEpoch() - 86000 )
filePath = m.hexdigest()
filePath = "%s/%s.rrd" % ( filePath[:2], filePath )
self.log.info( "Registering activity", str( acDict ) )
if self.__insert( "activities", { 'id' : 'NULL',
'filename' : "'%s'" % filePath,
},
acDict ) == 0:
return -1
return self.__select( "filename", "activities", acDict )[0][0]
def execute( self ):
""" Main execution method
"""
result = gConfig.getSections( "/DIRAC/Setups" )
if not result[ 'OK' ]:
return result
validSetups = result[ 'Value' ]
gLogger.info( "Valid setups for this cycle are %s" % ", ".join( validSetups ) )
# Get the WMS Snapshot!
result = self.jobDB.getSummarySnapshot( self.__jobDBFields )
now = Time.dateTime()
if not result[ 'OK' ]:
gLogger.error( "Can't the the jobdb summary", result[ 'Message' ] )
else:
values = result[ 'Value' ][1]
gLogger.info( "Start sending records!" )
for record in values:
recordSetup = record[0]
if recordSetup not in validSetups:
gLogger.error( "Setup %s is not valid" % recordSetup )
continue
record = record[1:]
rD = {}
for fV in self.__summaryDefinedFields:
rD[ fV[0] ] = fV[1]
for iP in range( len( self.__summaryKeyFieldsMapping ) ):
fieldName = self.__summaryKeyFieldsMapping[iP]
rD[ self.__renameFieldsMapping.get( fieldName, fieldName ) ] = record[iP]
record = record[ len( self.__summaryKeyFieldsMapping ): ]
for iP in range( len( self.__summaryValueFieldsMapping ) ):
rD[ self.__summaryValueFieldsMapping[iP] ] = int( record[iP] )
rD['startTime'] = int( Time.toEpoch( now ) )
rD['metric'] = 'WMSHistory'
message = json.dumps( rD )
self.sendRecords( message )
gLogger.info( "The records are successfully sent!" )
return S_OK()
