def __generateDBs(self):
self.__log.notice("Creating default AccountingDB...")
self.__allDBs = {
self.__defaultDB: AccountingDB(readOnly=self.__readOnly)
}
types = self.__allDBs[self.__defaultDB].getRegisteredTypes()
result = gConfig.getOptionsDict(self.__csPath)
if not result['OK']:
gLogger.verbose("No extra databases defined in %s" % self.__csPath)
return
validTypes = TypeLoader().getTypes()
opts = result['Value']
for acType in opts:
if acType not in validTypes:
msg = "Oops... %s defined in %s is not a known accounting type" % (
acType, self.__csPath)
self.__log.fatal(msg)
raise RuntimeError(msg)
dbName = opts[acType]
gLogger.notice("%s type will be assigned to %s" % (acType, dbName))
if dbName not in self.__allDBs:
fields = dbName.split("/")
if len(fields) == 1:
dbName = "Accounting/%s" % dbName
gLogger.notice("Creating DB %s" % dbName)
self.__allDBs[dbName] = AccountingDB(dbName,
readOnly=self.__readOnly)
self.__dbByType[acType] = dbName
def initializeDataStoreHandler( serviceInfo ):
global gAccountingDB
gAccountingDB = AccountingDB()
gAccountingDB.autoCompactDB()
result = gAccountingDB.markAllPendingRecordsAsNotTaken()
if not result[ 'OK' ]:
return result
gThreadScheduler.addPeriodicTask( 60, gAccountingDB.loadPendingRecords )
return S_OK()
def initializeDataStoreHandler(serviceInfo):
global gAccountingDB
gAccountingDB = AccountingDB()
gAccountingDB.autoCompactDB()
result = gAccountingDB.markAllPendingRecordsAsNotTaken()
if not result['OK']:
return result
gThreadScheduler.addPeriodicTask(60, gAccountingDB.loadPendingRecords)
return S_OK()
def initializeReportGeneratorHandler(serviceInfo):
global gAccountingDB
gAccountingDB = AccountingDB(readOnly=True)
#Get data location
reportSection = PathFinder.getServiceSection("Accounting/ReportGenerator")
dataPath = gConfig.getValue("%s/DataLocation" % reportSection,
"data/accountingGraphs")
dataPath = dataPath.strip()
if "/" != dataPath[0]:
dataPath = os.path.realpath(
"%s/%s" %
(gConfig.getValue('/LocalSite/InstancePath', rootPath), dataPath))
gLogger.info("Data will be written into %s" % dataPath)
try:
os.makedirs(dataPath)
except:
pass
try:
testFile = "%s/acc.jarl.test" % dataPath
fd = file(testFile, "w")
fd.close()
os.unlink(testFile)
except IOError:
gLogger.fatal("Can't write to %s" % dataPath)
return S_ERROR("Data location is not writable")
gDataCache.setGraphsLocation(dataPath)
gMonitor.registerActivity("plotsDrawn", "Drawn plot images",
"Accounting reports", "plots", gMonitor.OP_SUM)
gMonitor.registerActivity("reportsRequested", "Generated reports",
"Accounting reports", "reports", gMonitor.OP_SUM)
return S_OK()
from __future__ import print_function
__RCSID__ = "$Id$"
import pytest
from DIRAC.Core.Base.Script import parseCommandLine
parseCommandLine()
from DIRAC import gLogger
from DIRAC.AccountingSystem.DB.AccountingDB import AccountingDB
gLogger.setLevel("DEBUG")
acDB = AccountingDB()
startTime = 1262200000
middleTime = 1262300000
endTime = 1262400000
keyValues_1 = [
"User_1",
"UserGroup_1",
"Site_1",
"GridCE_1",
"GridMiddleware_1",
"GridResourceBroker_1",
"GridStatus_1",
]
nonKeyValue_1 = [123]
class DataStoreHandler(RequestHandler):
__acDB = AccountingDB()
@classmethod
def initializeHandler(cls, svcInfoDict):
multiPath = PathFinder.getDatabaseSection("Accounting/MultiDB")
cls.__acDB = MultiAccountingDB(multiPath)
cls.__acDB.autoCompactDB()
result = cls.__acDB.markAllPendingRecordsAsNotTaken()
if not result['OK']:
return result
gThreadScheduler.addPeriodicTask(60, cls.__acDB.loadPendingRecords)
return S_OK()
types_registerType = [basestring, list, list, list]
def export_registerType(self, typeName, definitionKeyFields,
definitionAccountingFields, bucketsLength):
"""
Register a new type. (Only for all powerful admins)
(Bow before me for I am admin! :)
"""
retVal = gConfig.getSections("/DIRAC/Setups")
if not retVal['OK']:
return retVal
errorsList = []
for setup in retVal['Value']:
retVal = self.__acDB.registerType(setup, typeName,
definitionKeyFields,
definitionAccountingFields,
bucketsLength) #pylint: disable=too-many-function-args
if not retVal['OK']:
errorsList.append(retVal['Message'])
if errorsList:
return S_ERROR("Error while registering type:\n %s" %
"\n ".join(errorsList))
return S_OK()
types_setBucketsLength = [basestring, list]
def export_setBucketsLength(self, typeName, bucketsLength):
"""
Change the buckets Length. (Only for all powerful admins)
(Bow before me for I am admin! :)
"""
retVal = gConfig.getSections("/DIRAC/Setups")
if not retVal['OK']:
return retVal
errorsList = []
for setup in retVal['Value']:
retVal = self.__acDB.changeBucketsLength(setup, typeName,
bucketsLength) #pylint: disable=too-many-function-args
if not retVal['OK']:
errorsList.append(retVal['Message'])
if errorsList:
return S_ERROR("Error while changing bucketsLength type:\n %s" %
"\n ".join(errorsList))
return S_OK()
types_regenerateBuckets = [basestring]
def export_regenerateBuckets(self, typeName):
"""
Recalculate buckets. (Only for all powerful admins)
(Bow before me for I am admin! :)
"""
retVal = gConfig.getSections("/DIRAC/Setups")
if not retVal['OK']:
return retVal
errorsList = []
for setup in retVal['Value']:
retVal = self.__acDB.regenerateBuckets(setup, typeName) #pylint: disable=too-many-function-args
if not retVal['OK']:
errorsList.append(retVal['Message'])
if errorsList:
return S_ERROR("Error while recalculating buckets for type:\n %s" %
"\n ".join(errorsList))
return S_OK()
types_getRegisteredTypes = []
def export_getRegisteredTypes(self):
"""
Get a list of registered types (Only for all powerful admins)
(Bow before me for I am admin! :)
"""
return self.__acDB.getRegisteredTypes()
types_deleteType = [basestring]
def export_deleteType(self, typeName):
"""
Delete accounting type and ALL its contents. VERY DANGEROUS! (Only for all powerful admins)
(Bow before me for I am admin! :)
"""
retVal = gConfig.getSections("/DIRAC/Setups")
if not retVal['OK']:
return retVal
errorsList = []
for setup in retVal['Value']:
retVal = self.__acDB.deleteType(setup, typeName) #pylint: disable=too-many-function-args
if not retVal['OK']:
errorsList.append(retVal['Message'])
if errorsList:
return S_ERROR("Error while deleting type:\n %s" %
"\n ".join(errorsList))
return S_OK()
types_commit = [basestring, datetime.datetime, datetime.datetime, list]
def export_commit(self, typeName, startTime, endTime, valuesList):
"""
Add a record for a type
"""
setup = self.serviceInfoDict['clientSetup']
startTime = int(Time.toEpoch(startTime))
endTime = int(Time.toEpoch(endTime))
return self.__acDB.insertRecordThroughQueue(setup, typeName, startTime,
endTime, valuesList) #pylint: disable=too-many-function-args
types_commitRegisters = [list]
def export_commitRegisters(self, entriesList):
"""
Add a record for a type
"""
setup = self.serviceInfoDict['clientSetup']
expectedTypes = [
basestring, datetime.datetime, datetime.datetime, list
]
for entry in entriesList:
if len(entry) != 4:
return S_ERROR("Invalid records")
for i in range(len(entry)):
if not isinstance(entry[i], expectedTypes[i]):
gLogger.error(
"Unexpected type in report",
": field %d in the records should be %s (and it is %s)"
% (i, expectedTypes[i], type(entry[i])))
return S_ERROR("Unexpected type in report")
records = []
for entry in entriesList:
startTime = int(Time.toEpoch(entry[1]))
endTime = int(Time.toEpoch(entry[2]))
records.append((setup, entry[0], startTime, endTime, entry[3]))
return self.__acDB.insertRecordBundleThroughQueue(records)
types_compactDB = []
def export_compactDB(self):
"""
Compact the db by grouping buckets
"""
return self.__acDB.compactBuckets()
types_remove = [basestring, datetime.datetime, datetime.datetime, list]
def export_remove(self, typeName, startTime, endTime, valuesList):
"""
Remove a record for a type
"""
setup = self.serviceInfoDict['clientSetup']
startTime = int(Time.toEpoch(startTime))
endTime = int(Time.toEpoch(endTime))
return self.__acDB.deleteRecord(setup, typeName, startTime, endTime,
valuesList) #pylint: disable=too-many-function-args
types_removeRegisters = [list]
def export_removeRegisters(self, entriesList):
"""
Remove a record for a type
"""
setup = self.serviceInfoDict['clientSetup']
expectedTypes = [
basestring, datetime.datetime, datetime.datetime, list
]
for entry in entriesList:
if len(entry) != 4:
return S_ERROR("Invalid records")
for i in range(len(entry)):
if not isinstance(entry[i], expectedTypes[i]):
return S_ERROR("%s field in the records should be %s" %
(i, expectedTypes[i]))
ok = 0
for entry in entriesList:
startTime = int(Time.toEpoch(entry[1]))
endTime = int(Time.toEpoch(entry[2]))
record = entry[3]
result = self.__acDB.deleteRecord(setup, entry[0], startTime,
endTime, record) #pylint: disable=too-many-function-args
if not result['OK']:
return S_OK(ok)
ok += 1
return S_OK(ok)