def dataLoggingClient(cls):
""" DataLoggingClient getter
:param cls: class reference
"""
if not cls.__dataLoggingClient:
cls.__dataLoggingClient = DataLoggingClient()
return cls.__dataLoggingClient
def dataLoggingClient( cls ):
""" DataLoggingClient getter
:param cls: class reference
"""
if not cls.__dataLoggingClient:
from DIRAC.DataManagementSystem.Client.DataLoggingClient import DataLoggingClient
cls.__dataLoggingClient = DataLoggingClient()
return cls.__dataLoggingClient
def test( self ):
""" test
:param self: self reference
"""
dlc = DataLoggingClient()
self.assertEqual( isinstance( dlc, DataLoggingClient ), True )
self.assertEqual( dlc.getServer(), "DataManagement/DataLogging" )
ping = dlc.ping()
self.assertEqual( ping["OK"], True )
def initialize(self):
self.section = PathFinder.getAgentSection(AGENT_NAME)
self.RequestDB = RequestDBMySQL()
self.TransferDB = TransferDB()
self.DataLog = DataLoggingClient()
self.factory = StorageFactory()
self.rm = ReplicaManager()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'DataManager')
return S_OK()
def initialize(self):
self.RequestDBClient = RequestClient()
self.ReplicaManager = ReplicaManager()
self.DataLog = DataLoggingClient()
self.maxNumberOfThreads = self.am_getOption('NumberOfThreads', 1)
self.threadPoolDepth = self.am_getOption('ThreadPoolDepth', 1)
self.threadPool = ThreadPool(1, self.maxNumberOfThreads)
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'DataManager')
return S_OK()
def initialize(self):
self.RequestDBClient = RequestClient()
self.ReplicaManager = ReplicaManager()
self.DataLog = DataLoggingClient()
gMonitor.registerActivity("Iteration", "Agent Loops", "TransferAgent",
"Loops/min", gMonitor.OP_SUM)
gMonitor.registerActivity("Execute", "Request Processed",
"TransferAgent", "Requests/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Done", "Request Completed", "TransferAgent",
"Requests/min", gMonitor.OP_SUM)
gMonitor.registerActivity("Replicate and register",
"Replicate and register operations",
"TransferAgent", "Attempts/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Replicate", "Replicate operations",
"TransferAgent", "Attempts/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Put and register",
"Put and register operations",
"TransferAgent", "Attempts/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Put", "Put operations", "TransferAgent",
"Attempts/min", gMonitor.OP_SUM)
gMonitor.registerActivity("Replication successful",
"Successful replications", "TransferAgent",
"Successful/min", gMonitor.OP_SUM)
gMonitor.registerActivity("Put successful", "Successful puts",
"TransferAgent", "Successful/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Replication failed", "Failed replications",
"TransferAgent", "Failed/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Put failed", "Failed puts", "TransferAgent",
"Failed/min", gMonitor.OP_SUM)
gMonitor.registerActivity("Replica registration successful",
"Successful replica registrations",
"TransferAgent", "Successful/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("File registration successful",
"Successful file registrations",
"TransferAgent", "Successful/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Replica registration failed",
"Failed replica registrations",
"TransferAgent", "Failed/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("File registration failed",
"Failed file registrations", "TransferAgent",
"Failed/min", gMonitor.OP_SUM)
self.maxNumberOfThreads = self.am_getOption('NumberOfThreads', 1)
self.threadPoolDepth = self.am_getOption('ThreadPoolDepth', 1)
self.threadPool = ThreadPool(1, self.maxNumberOfThreads)
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'DataManager')
return S_OK()
def setUp(self):
self.dlc = DataLoggingClient()
def dataLoggingClient(cls):
""" DataLoggingClient getter """
if not cls.__dataLoggingClient:
from DIRAC.DataManagementSystem.Client.DataLoggingClient import DataLoggingClient
cls.__dataLoggingClient = DataLoggingClient()
return cls.__dataLoggingClient
targets = []
for x in range(4):
targets.append(targetSE + str(random.randint(0, randomMax)))
for call in calls:
for x in range(2):
call.addAction(
DLAction(DLFile(files[x * 2]), 'Successful',
DLStorageElement(sources[x * 2]),
DLStorageElement(targets[x * 2]), blob, None, None))
call.addAction(
DLAction(DLFile(files[x * 2 + 1]), 'Failed',
DLStorageElement(sources[x * 2 + 1]),
DLStorageElement(targets[x * 2 + 1]), blob,
'errorMessage', random.randint(1, 1999)))
return sequence
done = False
start = time.time()
client = DataLoggingClient(url=servAddress)
while not done:
seq = makeSequence()
res = client.insertSequence(seq)
if not res['OK']:
print 'error %s' % res['Message']
if (time.time() - start > maxDuration):
done = True
def initialize(self):
self.ReplicaManager = ReplicaManager()
self.DataLog = DataLoggingClient()
self.DataIntegrityClient = DataIntegrityClient()
if self.am_getOption('DirectDB', False):
from DIRAC.StorageManagementSystem.DB.MigrationMonitoringDB import MigrationMonitoringDB
self.MigrationMonitoringDB = MigrationMonitoringDB()
else:
from DIRAC.StorageManagementSystem.Client.MigrationMonitoringClient import MigrationMonitoringClient
self.MigrationMonitoringDB = MigrationMonitoringClient()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'DataManager')
self.userName = '******'
self.storageElements = self.am_getOption('StorageElements',
['CERN-RAW'])
self.lastMonitors = {}
gMonitor.registerActivity("Iteration", "Agent Loops/min",
"MigrationMonitoringAgent", "Loops",
gMonitor.OP_SUM)
if self.storageElements:
gLogger.info(
"Agent will be initialised to monitor the following SEs:")
for se in self.storageElements:
gLogger.info(se)
self.lastMonitors[se] = datetime.datetime.utcfromtimestamp(0.0)
gMonitor.registerActivity("Iteration%s" % se,
"Agent Loops/min",
"MigrationMonitoringAgent", "Loops",
gMonitor.OP_SUM)
gMonitor.registerActivity("MigratingFiles%s" % se,
"Files waiting for migration",
"MigrationMonitoringAgent", "Files",
gMonitor.OP_MEAN)
gMonitor.registerActivity("MigratedFiles%s" % se,
"Newly migrated files",
"MigrationMonitoringAgent", "Files",
gMonitor.OP_SUM)
gMonitor.registerActivity("TotalMigratedFiles%s" % se,
"Total migrated files",
"MigrationMonitoringAgent", "Files",
gMonitor.OP_ACUM)
gMonitor.registerActivity("TotalMigratedSize%s" % se,
"Total migrated file size",
"MigrationMonitoringAgent", "GB",
gMonitor.OP_ACUM)
gMonitor.registerActivity("ChecksumMatches%s" % se,
"Successfully migrated files",
"MigrationMonitoringAgent", "Files",
gMonitor.OP_SUM)
gMonitor.registerActivity("TotalChecksumMatches%s" % se,
"Total successfully migrated files",
"MigrationMonitoringAgent", "Files",
gMonitor.OP_ACUM)
gMonitor.registerActivity("ChecksumMismatches%s" % se,
"Erroneously migrated files",
"MigrationMonitoringAgent", "Files",
gMonitor.OP_SUM)
gMonitor.registerActivity("TotalChecksumMismatches%s" % se,
"Total erroneously migrated files",
"MigrationMonitoringAgent", "Files",
gMonitor.OP_ACUM)
gMonitor.registerActivity("MigrationTime%s" % se,
"Average migration time",
"MigrationMonitoringAgent",
"Seconds", gMonitor.OP_MEAN)
return S_OK()
