def __copyToExternalSE(self, localFilePath, sbPath):
"""
Copy uploaded file to external SE
"""
try:
rm = ReplicaManager()
result = rm.put(sbPath, localFilePath, self.__externalSEName)
if not result['OK']:
return result
if 'Successful' not in result['Value']:
gLogger.verbose("Oops, no successful transfers there",
str(result))
return S_ERROR(
"RM returned OK to the action but no successful transfers were there"
)
okTrans = result['Value']['Successful']
if sbPath not in okTrans:
gLogger.verbose(
"Ooops, SB transfer wasn't in the successful ones",
str(result))
return S_ERROR(
"RM returned OK to the action but SB transfer wasn't in the successful ones"
)
return S_OK((self.__externalSEName, okTrans[sbPath]))
except Exception, e:
return S_ERROR("Error while moving sandbox to SE: %s" % str(e))
def __init__( self, *args, **kwargs ):
''' c'tor
'''
AgentModule.__init__( self, *args, **kwargs )
# # replica manager
self.replicaManager = ReplicaManager()
# # transformation client
self.transClient = TransformationClient()
# # wms client
self.wmsClient = WMSClient()
# # request client
self.requestClient = RequestClient()
# # file catalog clinet
self.metadataClient = FileCatalogClient()
# # placeholders for CS options
# # transformations types
self.transformationTypes = None
# # directory locations
self.directoryLocations = None
# # transformation metadata
self.transfidmeta = None
# # archive periof in days
self.archiveAfter = None
# # active SEs
self.activeStorages = None
# # transformation log SEs
self.logSE = None
# # enable/disable execution
self.enableFlag = None
def setReplicaProblematic(self,lfn,se,pfn='',reason='Access failure'):
""" Set replica status to Problematic in the File Catalog
@param lfn: lfn of the problematic file
@param se: storage element
@param pfn: physical file name
@param reason: as name suggests...
@return: S_OK()
"""
rm = ReplicaManager()
source = "Job %d at %s" % (self.jobID,DIRAC.siteName())
result = rm.setReplicaProblematic((lfn,pfn,se,reason),source)
if not result['OK'] or result['Value']['Failed']:
# We have failed the report, let's attempt the Integrity DB faiover
integrityDB = RPCClient('DataManagement/DataIntegrity',timeout=120)
fileMetadata = {'Prognosis':reason,'LFN':lfn,'PFN':pfn,'StorageElement':se}
result = integrityDB.insertProblematic(source,fileMetadata)
if not result['OK']:
# Add it to the request
if self.workflow_commons.has_key('Request'):
request = self.workflow_commons['Request']
subrequest = DISETSubRequest(result['rpcStub']).getDictionary()
request.addSubRequest(subrequest,'integrity')
return S_OK()
def __init__(self, operation=None, csPath=None):
"""c'tor
:param self: self reference
:param Operation operation: Operation instance
:param str csPath: CS path for this handler
"""
# # base classes ctor
super(PutAndRegister, self).__init__(operation, csPath)
# # gMonitor stuff
gMonitor.registerActivity("PutAtt", "File put attempts",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("PutFail", "Failed file puts",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("PutOK", "Successful file puts",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RegisterOK",
"Successful file registrations",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RegisterFail", "Failed file registrations",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
self.rm = ReplicaManager()
def registerInputData(filepath, size, prefix='/cepc/lustre-ro'):
infoDict = {}
infoDict['PFN'] = ''
infoDict['Size'] = size
infoDict['SE'] = 'IHEP-STORM'
infoDict['GUID'] = commands.getoutput('uuidgen')
infoDict['Checksum'] = ''
fileDict = {}
lfn = prefix + filepath
fileDict[lfn] = infoDict
fcc = FileCatalogClient('DataManagement/FileCatalog')
rm = ReplicaManager()
result = {}
result['lfn'] = lfn
result['is_registered'] = False
#查询
for repeatTimes in range(10):
is_registered = fcc.isFile(lfn)
if (is_registered['OK']
and is_registered['Value']['Successful'].has_key(lfn)):
break
# else:
# continue
if not is_registered['OK']: #查询失败
result['is_registered'] = 'querry error. unkown'
print 'Failed to query %s in DFC. Error message is %s' % (
lfn, is_registered['Message'])
if is_registered['Value']['Successful'][lfn]: #已注册
result['is_registered'] = True
for repeatTimes in range(10):
is_removed = rm.removeCatalogFile(lfn) #删除
if (is_removed['OK']
and is_removed['Value']['Successful'][lfn]['FileCatalog']):
result['is_removed'] = True
break
# else:
# continue
if not is_removed['OK']: #删除失败
result['is_removed'] = 'remove error'
print 'Failed to remove %s from DFC.' % lfn
#add
for repeatTimes in range(10):
is_added = fcc.addFile(fileDict) #add/register
if (is_added['OK'] and is_added['Value']['Successful'][lfn]):
result['OK'] = True
return result
# else:
# continue
if not is_added['OK']: #add unsuccessfully
result['OK'] = False
result['Message'] = is_added['Message']
elif is_added['Value']['Failed']:
result['OK'] = False
result['Message'] = 'Failed to add file' + lfn
return result
def __init__(self, argumentsDict):
""" Standard constructor
"""
self.name = COMPONENT_NAME
self.log = gLogger.getSubLogger(self.name)
self.inputData = argumentsDict['InputData']
self.configuration = argumentsDict['Configuration']
self.fileCatalogResult = argumentsDict['FileCatalog']
self.jobID = None
self.replicaManager = ReplicaManager()
def initialize(self):
self.RequestDBClient = RequestClient()
self.ReplicaManager = ReplicaManager()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'DataManager')
return S_OK()
def setUp( self ):
super( IntegrationTest, self ).setUp()
rm = ReplicaManager()
res = rm.removeFile( ['/lhcb/testCfg/testVer/LOG/00012345/0006/00012345_00067890.tar',
'/lhcb/testCfg/testVer/SIM/00012345/0006/00012345_00067890_1.sim'],
force = True )
if not res['OK']:
print "Could not remove files", res['Message']
exit( 1 )
def initialize( self ):
self.replicaManager = ReplicaManager()
#self.stagerClient = StorageManagerClient()
self.dataIntegrityClient = DataIntegrityClient()
self.storageDB = StorageManagementDB()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'DataManager' )
return S_OK()
def registerInputData(filepath, size, prefix = '/cepc/lustre-ro'):
infoDict = {}
infoDict['PFN'] = ''
infoDict['Size'] = size
infoDict['SE'] = 'IHEP-STORM'
infoDict['GUID'] = commands.getoutput('uuidgen')
infoDict['Checksum'] = ''
fileDict = {}
lfn = prefix + filepath
fileDict[lfn] = infoDict
fcc = FileCatalogClient('DataManagement/FileCatalog')
rm = ReplicaManager()
result = {}
result['lfn'] = lfn
result['is_registered'] = False
#查询
for repeatTimes in range(10):
is_registered = fcc.isFile(lfn)
if (is_registered['OK'] and is_registered['Value']['Successful'].has_key(lfn)):
break
# else:
# continue
if not is_registered['OK']:#查询失败
result['is_registered'] = 'querry error. unkown'
print 'Failed to query %s in DFC. Error message is %s' %(lfn, is_registered['Message'])
if is_registered['Value']['Successful'][lfn]:#已注册
result['is_registered'] = True
for repeatTimes in range(10):
is_removed = rm.removeCatalogFile(lfn)#删除
if (is_removed['OK'] and is_removed['Value']['Successful'][lfn]['FileCatalog']):
result['is_removed'] = True
break
# else:
# continue
if not is_removed['OK']:#删除失败
result['is_removed'] = 'remove error'
print 'Failed to remove %s from DFC.' %lfn
#add
for repeatTimes in range(10):
is_added = fcc.addFile(fileDict)#add/register
if (is_added['OK'] and is_added['Value']['Successful'][lfn]):
result['OK'] = True
return result
# else:
# continue
if not is_added['OK']:#add unsuccessfully
result['OK'] = False
result['Message'] = is_added['Message']
elif is_added['Value']['Failed']:
result['OK'] = False
result['Message'] = 'Failed to add file' + lfn
return result
def initialize( self ):
self.replicaManager = ReplicaManager()
self.stagerClient = StorageManagerClient()
self.pinLifeTime = 60 * 60 * 24 * 7 # 7 days
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'DataManager' )
return S_OK()
def __init__(self, requestObject=False):
""" Constructor function, can specify request object to instantiate
FailoverTransfer or a new request object is created.
"""
self.log = gLogger.getSubLogger("FailoverTransfer")
self.rm = ReplicaManager()
self.request = requestObject
if not self.request:
self.request = RequestContainer()
self.request.setRequestName('default_request.xml')
self.request.setSourceComponent('FailoverTransfer')
def __init__( self, requestObject = None ):
""" Constructor function, can specify request object to instantiate
FailoverTransfer or a new request object is created.
"""
self.log = gLogger.getSubLogger( "FailoverTransfer" )
self.replicaMgr = ReplicaManager()
self.request = requestObject
if not self.request:
self.request = Request()
self.request.RequestName = 'default_request.xml'
self.request.SourceComponent = 'FailoverTransfer'
def __init__(self, argumentsDict):
""" Standard constructor
"""
self.name = COMPONENT_NAME
self.log = gLogger.getSubLogger(self.name)
self.inputData = argumentsDict['InputData']
self.configuration = argumentsDict['Configuration']
self.fileCatalogResult = argumentsDict['FileCatalog']
# By default put each input data file into a separate directory
self.inputDataDirectory = argumentsDict.get('InputDataDirectory',
'PerFile')
self.jobID = None
self.replicaManager = ReplicaManager()
self.counter = 1
def initialize(self):
self.section = PathFinder.getAgentSection(AGENT_NAME)
self.RequestDB = RequestDBMySQL()
self.TransferDB = TransferDB()
self.DataLog = DataLoggingClient()
self.factory = StorageFactory()
self.rm = ReplicaManager()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'DataManager')
return S_OK()
def initialize(self):
self.RequestDBClient = RequestClient()
self.ReplicaManager = ReplicaManager()
self.DataLog = DataLoggingClient()
self.maxNumberOfThreads = self.am_getOption('NumberOfThreads', 1)
self.threadPoolDepth = self.am_getOption('ThreadPoolDepth', 1)
self.threadPool = ThreadPool(1, self.maxNumberOfThreads)
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'DataManager')
return S_OK()
def initialize(self):
self.replicaManager = ReplicaManager()
#self.stagerClient = StorageManagerClient()
self.dataIntegrityClient = DataIntegrityClient()
self.storageDB = StorageManagementDB()
# pin lifetime = 1 day
self.pinLifetime = self.am_getOption('PinLifetime', THROTTLING_TIME)
# Resources helper
self.resources = Resources()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'DataManager')
return S_OK()
def __init__(self):
"""Module initialization.
"""
super(UploadLogFile, self).__init__()
self.version = __RCSID__
self.log = gLogger.getSubLogger("UploadLogFile")
self.PRODUCTION_ID = None
self.JOB_ID = None
self.workflow_commons = None
self.request = None
self.logFilePath = ""
self.logLFNPath = ""
self.logdir = ""
self.logSE = self.ops.getValue("/LogStorage/LogSE", "LogSE")
self.root = gConfig.getValue("/LocalSite/Root", os.getcwd())
self.logSizeLimit = self.ops.getValue("/LogFiles/SizeLimit", 20 * 1024 * 1024)
self.logExtensions = []
self.failoverSEs = gConfig.getValue("/Resources/StorageElementGroups/Tier1-Failover", [])
self.diracLogo = self.ops.getValue(
"/SAM/LogoURL", "https://lhcbweb.pic.es/DIRAC/images/logos/DIRAC-logo-transp.png"
)
self.rm = ReplicaManager()
self.experiment = "CLIC"
self.enable = True
self.failoverTest = False # flag to put log files to failover by default
self.jobID = ""
def replicaManager(cls):
""" ReplicaManager getter
:param cls: class reference
"""
if not cls.__replicaManager:
cls.__replicaManager = ReplicaManager()
return cls.__replicaManager
def __init__( self, *args, **kwargs ):
''' c'tor
'''
AgentModule.__init__( self, *args, **kwargs )
# # replica manager
self.replicaManager = ReplicaManager()
# # transformation client
self.transClient = TransformationClient()
# # wms client
self.wmsClient = WMSClient()
# # request client
self.requestClient = RequestClient()
# # file catalog clinet
self.metadataClient = FileCatalogClient()
# # placeholders for CS options
# # transformations types
self.transformationTypes = None
# # directory locations
self.directoryLocations = None
# # transformation metadata
self.transfidmeta = None
# # archive periof in days
self.archiveAfter = None
# # active SEs
self.activeStorages = None
# # transformation log SEs
self.logSE = None
# # enable/disable execution
self.enableFlag = None
def __init__(self):
super(OverlayInput, self).__init__()
self.enable = True
self.STEP_NUMBER = ''
self.log = gLogger.getSubLogger( "OverlayInput" )
self.applicationName = 'OverlayInput'
self.curdir = os.getcwd()
self.applicationLog = ''
self.printoutflag = ''
self.prodid = 0
self.detector = '' ##needed for backward compatibility
self.detectormodel = ""
self.energytouse = ''
self.energy = 0
self.nbofeventsperfile = 100
self.lfns = []
self.nbfilestoget = 0
self.BkgEvtType = 'gghad'
self.BXOverlay = 0
self.ggtohadint = 3.2
self.nbsigeventsperfile = 0
self.nbinputsigfile = 1
self.NbSigEvtsPerJob = 0
self.rm = ReplicaManager()
self.fc = FileCatalogClient()
self.site = DIRAC.siteName()
self.machine = 'clic_cdr'
def __init__(self, loggerIn=None):
""" Initialization of module base.
loggerIn is a logger object that can be passed so that the logging will be more clear.
"""
if not loggerIn:
self.log = gLogger.getSubLogger('ModuleBase')
else:
self.log = loggerIn
# These 2 are used in many places, so it's good to have them available here.
self.opsH = Operations()
self.rm = ReplicaManager()
# Some job parameters
self.production_id = 0
self.prod_job_id = 0
self.jobID = 0
self.step_number = 0
self.step_id = 0
self.jobType = ''
self.executable = ''
self.command = None
self.workflowStatus = None
self.stepStatus = None
self.workflow_commons = None
self.step_commons = None
# These are useful objects (see the getFileReporter(), getJobReporter() and getRequestContainer() functions)
self.fileReport = None
self.jobReport = None
self.request = None
def initialize( self ):
"""Sets defaults """
self.replicaManager = ReplicaManager()
self.transClient = TransformationClient()
self.wmsClient = WMSClient()
self.requestClient = RequestClient()
self.metadataClient = FileCatalogClient()
self.storageUsageClient = StorageUsageClient()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'DataManager' )
self.transformationTypes = sortList( self.am_getOption( 'TransformationTypes', ['MCSimulation', 'DataReconstruction', 'DataStripping', 'MCStripping', 'Merge', 'Replication'] ) )
gLogger.info( "Will consider the following transformation types: %s" % str( self.transformationTypes ) )
self.directoryLocations = sortList( self.am_getOption( 'DirectoryLocations', ['TransformationDB', 'StorageUsage', 'MetadataCatalog'] ) )
gLogger.info( "Will search for directories in the following locations: %s" % str( self.directoryLocations ) )
self.transfidmeta = self.am_getOption( 'TransfIDMeta', "TransformationID" )
gLogger.info( "Will use %s as metadata tag name for TransformationID" % self.transfidmeta )
self.archiveAfter = self.am_getOption( 'ArchiveAfter', 7 ) # days
gLogger.info( "Will archive Completed transformations after %d days" % self.archiveAfter )
self.activeStorages = sortList( self.am_getOption( 'ActiveSEs', [] ) )
gLogger.info( "Will check the following storage elements: %s" % str( self.activeStorages ) )
self.logSE = self.am_getOption( 'TransformationLogSE', 'LogSE' )
gLogger.info( "Will remove logs found on storage element: %s" % self.logSE )
return S_OK()
def __init__(self):
super(WhizardAnalysis, self).__init__()
self.enable = True
self.STEP_NUMBER = ''
self.debug = True
self.log = gLogger.getSubLogger( "WhizardAnalysis" )
self.SteeringFile = ''
self.OutputFile = ''
self.NumberOfEvents = 1
self.Lumi = 0
self.applicationName = 'whizard'
self.evttype = ""
self.RandomSeed = 0
self.energy = 3000.
self.getProcessInFile = False
self.rm = ReplicaManager()
self.processlist = None
self.jobindex = None
self.parameters = {}
self.susymodel = 0
self.Model = ''
self.genmodel = GeneratorModels()
self.eventstring = ['! ', 'Fatal error:', 'PYSTOP', 'No matrix element available',
'Floating point exception', 'Event generation finished.', " n_events","luminosity", " sum "]
self.excludeAllButEventString = False
self.steeringparameters = ''
self.options = None
self.optionsdict = {}
self.OptionsDictStr = ''
self.GenLevelCutDictStr = ''
self.genlevelcuts = {}
self.willCut = False
self.useGridFiles = False
def finish(self):
"""
after having set all the files, this one does all the job
@return:
"""
rc = 0
rm = ReplicaManager()
for item in self.listFileStaged:
#print("SE '"+self.SE+"' == : '"+str(self.SE == "False")+"'")
if not self.SE:
self.log.info("No SE available for '" + item[0] + "'")
rc += 1
continue
else:
self.log.info("Trying to store '" + item[0] + "' in SE : '" +
self.SE + "' ...")
result = rm.putAndRegister(item[1], item[0], self.SE)
if not result['OK']:
self.log.info('ERROR %s' % (result['Message']))
self.log.info("Wait 5sec before trying again...")
time.sleep(5)
result = rm.putAndRegister(item[1], item[0], self.SE)
if not result['OK']:
self.log.info('ERROR %s' % (result['Message']))
while not result['OK']:
self.listSEs.remove(
self.SE
) # make sure not to pick the same SE again.
self.__pickRandomSE()
if not self.SE:
break
self.log.info("Trying with another SE : '" +
self.SE + "' . In 5sec...")
time.sleep(5)
result = rm.putAndRegister(item[1], item[0],
self.SE)
if result['OK']:
self.log.info("file stored : '" + item[1] +
"' in '" + self.SE + "'")
else:
self.log.error(
"ERROR : failed to store the file '" +
item[1] + "' ...")
rc += 1
return rc
def replicaManager( cls ):
""" ReplicaManager getter
:param cls: class reference
"""
if not cls.__replicaManager:
from DIRAC.DataManagementSystem.Client.ReplicaManager import ReplicaManager
cls.__replicaManager = ReplicaManager()
return cls.__replicaManager
def removeOutputData(baseDir):
res = getProxyInfo(False, False)
if not res["OK"]:
gLogger.error("Failed to get client proxy information.", res["Message"])
return S_ERROR("Failed to get client proxy information.", res["Message"])
# ######################################################################################################## #
rm = ReplicaManager()
result = rm.cleanLogicalDirectory(baseDir)
print "Ignore the message about the file '" + baseDir + "dirac_directory'"
if not result["OK"]:
print "ERROR: %s" % (result["Message"])
return S_ERROR("Failed to Suppress the directory : '" + baseDir + "'")
return S_OK(baseDir + " has been supressed")
def _getClients( self ):
""" returns the clients used in the threads
"""
threadTransformationClient = TransformationClient()
threadReplicaManager = ReplicaManager()
return {'TransformationClient': threadTransformationClient,
'ReplicaManager': threadReplicaManager}
def __prepareFileForHTTP( self, lfn, key ):
""" proxied preapre file for HTTP """
global HTTP_PATH
res = self.__prepareSecurityDetails()
if not res['OK']:
return res
# Clear the local cache
getFileDir = "%s/%s" % ( HTTP_PATH, key )
os.makedirs(getFileDir)
# Get the file to the cache
from DIRAC.DataManagementSystem.Client.ReplicaManager import ReplicaManager
replicaMgr = ReplicaManager()
result = replicaMgr.getFile( lfn, destinationDir=getFileDir )
result['CachePath'] = getFileDir
return result
def __prepareFileForHTTP(self, lfn, key):
global httpPath
res = self.__prepareSecurityDetails()
if not res['OK']:
return res
# Clear the local cache
getFileDir = "%s/%s" % (httpPath, key)
os.makedirs(getFileDir)
# Get the file to the cache
from DIRAC.DataManagementSystem.Client.ReplicaManager import ReplicaManager
rm = ReplicaManager()
result = rm.getFile(lfn, destinationDir=getFileDir)
result['CachePath'] = getFileDir
return result
def __copyToExternalSE( self, localFilePath, sbPath ):
"""
Copy uploaded file to external SE
"""
try:
rm = ReplicaManager()
result = rm.put( sbPath, localFilePath, self.__externalSEName )
if not result[ 'OK' ]:
return result
if 'Successful' not in result[ 'Value' ]:
gLogger.verbose( "Oops, no successful transfers there", str( result ) )
return S_ERROR( "RM returned OK to the action but no successful transfers were there" )
okTrans = result[ 'Value' ][ 'Successful' ]
if sbPath not in okTrans:
gLogger.verbose( "Ooops, SB transfer wasn't in the successful ones", str( result ) )
return S_ERROR( "RM returned OK to the action but SB transfer wasn't in the successful ones" )
return S_OK( ( self.__externalSEName, okTrans[ sbPath ] ) )
except Exception, e:
return S_ERROR( "Error while moving sandbox to SE: %s" % str( e ) )
def initialize( self ):
self.RequestDBClient = RequestClient()
self.ReplicaManager = ReplicaManager()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'DataManager' )
return S_OK()
def removeOutputData(baseDir):
res = getProxyInfo(False, False)
if not res['OK']:
gLogger.error("Failed to get client proxy information.",
res['Message'])
return S_ERROR("Failed to get client proxy information.",
res['Message'])
# ######################################################################################################## #
rm = ReplicaManager()
result = rm.cleanLogicalDirectory(baseDir)
print "Ignore the message about the file '" + baseDir + "dirac_directory'"
if not result['OK']:
print 'ERROR: %s' % (result['Message'])
return S_ERROR("Failed to Suppress the directory : '" + baseDir + "'")
return S_OK(baseDir + " has been supressed")
def __init__(self):
"""Module initialization.
"""
super(GetSRMFile, self).__init__()
self.version = __RCSID__
self.log = gLogger.getSubLogger('GetSRMFile')
self.repman = ReplicaManager()
self.srmfiles = []
self.files = []
self.counter = 1
def __init__( self, argumentsDict ): """ Standard constructor """ self.name = COMPONENT_NAME self.log = gLogger.getSubLogger( self.name ) self.inputData = argumentsDict['InputData'] self.configuration = argumentsDict['Configuration'] self.fileCatalogResult = argumentsDict['FileCatalog'] self.jobID = None self.rm = ReplicaManager()
def initialize(self):
"""Sets defaults """
self.replicaManager = ReplicaManager()
self.transClient = TransformationClient()
self.wmsClient = WMSClient()
self.requestClient = RequestClient()
self.metadataClient = FileCatalogClient()
self.storageUsageClient = StorageUsageClient()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'DataManager')
self.transformationTypes = sortList(
self.am_getOption('TransformationTypes', [
'MCSimulation', 'DataReconstruction', 'DataStripping',
'MCStripping', 'Merge', 'Replication'
]))
gLogger.info("Will consider the following transformation types: %s" %
str(self.transformationTypes))
self.directoryLocations = sortList(
self.am_getOption(
'DirectoryLocations',
['TransformationDB', 'StorageUsage', 'MetadataCatalog']))
gLogger.info(
"Will search for directories in the following locations: %s" %
str(self.directoryLocations))
self.transfidmeta = self.am_getOption('TransfIDMeta',
"TransformationID")
gLogger.info("Will use %s as metadata tag name for TransformationID" %
self.transfidmeta)
self.archiveAfter = self.am_getOption('ArchiveAfter', 7) # days
gLogger.info("Will archive Completed transformations after %d days" %
self.archiveAfter)
self.activeStorages = sortList(self.am_getOption('ActiveSEs', []))
gLogger.info("Will check the following storage elements: %s" %
str(self.activeStorages))
self.logSE = self.am_getOption('TransformationLogSE', 'LogSE')
gLogger.info("Will remove logs found on storage element: %s" %
self.logSE)
return S_OK()
def getProcessList(self):
""" Get the L{ProcessList} needed by L{Whizard}.
@return: process list object
"""
processlistpath = gConfig.getValue("/LocalSite/ProcessListPath", "")
if not processlistpath:
gLogger.info('Will download the process list locally. To gain time, please put it somewhere and add to \
your dirac.cfg the entry /LocalSite/ProcessListPath pointing to the file')
pathtofile = self.ops.getValue("/ProcessList/Location", "")
if not pathtofile:
gLogger.error("Could not get path to process list")
processlist = ""
else:
rm = ReplicaManager()
rm.getFile(pathtofile)
processlist = os.path.basename(pathtofile)
else:
processlist = processlistpath
self.pl = ProcessList(processlist)
return self.pl
def __init__(self, useCertificates=False):
"""c'tor
:param self: self reference
:param bool useCertificates: flag to enable/disable certificates
"""
Client.__init__(self)
self.log = gLogger.getSubLogger("DataManagement/FTSClient")
self.setServer("DataManagement/FTSManager")
# getting other clients
self.ftsValidator = FTSValidator()
self.replicaManager = ReplicaManager()
self.storageFactory = StorageFactory()
url = PathFinder.getServiceURL("DataManagement/FTSManager")
if not url:
raise RuntimeError(
"CS option DataManagement/FTSManager URL is not set!")
self.ftsManager = RPCClient(url)
def __init__(self, operation=None, csPath=None):
"""c'tor
:param self: self reference
:param Operation operation: Operation instance
:param str csPath: CS path for this handler
"""
super(ReplicateAndRegister, self).__init__(operation, csPath)
# # own gMonitor stuff for files
gMonitor.registerActivity("ReplicateAndRegisterAtt",
"Replicate and register attempted",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("ReplicateOK", "Replications successful",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("ReplicateFail", "Replications failed",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RegisterOK", "Registrations successful",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RegisterFail", "Registrations failed",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
# # for FTS
gMonitor.registerActivity("FTSScheduleAtt", "Files schedule attempted",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("FTSScheduleOK", "File schedule successful",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("FTSScheduleFail", "File schedule failed",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
# # SE cache
self.seCache = {}
# Clients
self.rm = ReplicaManager()
self.ftsClient = FTSClient()
def __init__(self):
'''
Constructor
'''
super(MoveInFC, self).__init__()
self.enable = False
self.STEP_NUMBER = ''
self.log = gLogger.getSubLogger( "MoveInFC" )
self.applicationName = 'MoveInFC'
self.rm = ReplicaManager()
self.listoutput = {}
self.outputpath = ''
def __init__(self,requestObject=False):
""" Constructor function, can specify request object to instantiate
FailoverTransfer or a new request object is created.
"""
self.log = gLogger.getSubLogger( "FailoverTransfer" )
self.rm = ReplicaManager()
self.request = requestObject
if not self.request:
self.request = RequestContainer()
self.request.setRequestName('default_request.xml')
self.request.setSourceComponent('FailoverTransfer')
def __init__( self, *args, **kwargs ):
""" c'tor
"""
AgentModule.__init__( self, *args, **kwargs )
self.integrityClient = DataIntegrityClient()
self.replicaManager = ReplicaManager()
self.transClient = TransformationClient()
self.fileCatalogClient = FileCatalogClient()
agentTSTypes = self.am_getOption( 'TransformationTypes', [] )
if agentTSTypes:
self.transformationTypes = agentTSTypes
else:
self.transformationTypes = Operations().getValue( 'Transformations/DataProcessing', ['MCSimulation', 'Merge'] )
self.directoryLocations = sortList( self.am_getOption( 'DirectoryLocations', ['TransformationDB',
'MetadataCatalog'] ) )
self.activeStorages = sortList( self.am_getOption( 'ActiveSEs', [] ) )
self.transfidmeta = self.am_getOption( 'TransfIDMeta', "TransformationID" )
self.enableFlag = True
def initializeOptimizer(cls):
"""Initialize specific parameters
"""
cls.ex_setProperty('shifterProxy', 'DataManager')
cls.__SEStatus = DictCache.DictCache()
try:
cls.__replicaMan = ReplicaManager()
except Exception, e:
msg = 'Failed to create ReplicaManager'
cls.log.exception(msg)
return S_ERROR(msg + str(e))
def __init__( self, argumentsDict ): """ Standard constructor """ self.name = COMPONENT_NAME self.log = gLogger.getSubLogger( self.name ) self.inputData = argumentsDict['InputData'] self.configuration = argumentsDict['Configuration'] self.fileCatalogResult = argumentsDict['FileCatalog'] # By default put each input data file into a separate directory self.inputDataDirectory = argumentsDict.get( 'InputDataDirectory', 'PerFile' ) self.jobID = None self.replicaManager = ReplicaManager() self.counter = 1
def finish(self):
"""
after having set all the files, this one does all the job
@return:
"""
rc = 0
rm = ReplicaManager()
for item in self.listFileStaged:
#self.log.info("SE '"+self.SE+"' == : '"+str(self.SE == "False")+"'")
if not self.SE:
self.log.info("No SE available for '"+item[0]+"'")
rc+=1
continue
else:
self.log.info("Trying to store '"+item[0]+"' in SE : '"+self.SE+"' ...")
result = rm.putAndRegister( item[1], item[0], self.SE)
if not result['OK']:
self.log.warning('ERROR %s' % ( result['Message'] ))
self.listSEs.remove(self.SE) # make sure not to pick the same SE again.
self.log.info("Wait 5sec before trying again...")
time.sleep(5)
result = rm.putAndRegister( item[1], item[0], self.SE)
if not result['OK']:
self.log.warning('ERROR %s' % ( result['Message'] ))
while not result['OK'] :
self.__pickRandomSE()
if not self.SE:
break
self.log.info("Trying with another SE : '"+self.SE+"' . In 5sec...")
time.sleep(5)
result = rm.putAndRegister( item[1], item[0], self.SE)
if result['OK']:
self.log.info("file stored : '"+item[1]+"' in '"+self.SE+"'")
else:
self.log.info("ERROR : failed to store the file '"+item[1]+"' ...")
self.listSEs.remove(self.SE) # make sure not to pick the same SE again.
rc += 1
return rc
def initialize( self ):
"""
Called by the framework upon startup, before any cycle (execute method bellow)
"""
self.requestDBClient = RequestClient()
# the RequestAgentMixIn needs the capitalized version, until is is fixed keep this.
self.RequestDBClient = self.requestDBClient
self.replicaManager = ReplicaManager()
gMonitor.registerActivity( "Iteration", "Agent Loops", "RemovalAgent", "Loops/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "Execute", "Request Processed", "RemovalAgent", "Requests/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "Done", "Request Completed", "RemovalAgent", "Requests/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "PhysicalRemovalAtt", "Physical removals attempted",
"RemovalAgent", "Removal/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "PhysicalRemovalDone", "Successful physical removals",
"RemovalAgent", "Removal/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "PhysicalRemovalFail", "Failed physical removals",
"RemovalAgent", "Removal/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "PhysicalRemovalSize", "Physically removed size",
"RemovalAgent", "Bytes", gMonitor.OP_ACUM )
gMonitor.registerActivity( "ReplicaRemovalAtt", "Replica removal attempted",
"RemovalAgent", "Removal/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "ReplicaRemovalDone", "Successful replica removals",
"RemovalAgent", "Removal/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "ReplicaRemovalFail", "Failed replica removals",
"RemovalAgent", "Removal/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RemoveFileAtt", "File removal attempted",
"RemovalAgent", "Removal/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RemoveFileDone", "File removal done",
"RemovalAgent", "Removal/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RemoveFileFail", "File removal failed",
"RemovalAgent", "Removal/min", gMonitor.OP_SUM )
self.maxNumberOfThreads = self.am_getOption( 'NumberOfThreads', self.maxNumberOfThreads )
self.maxRequestsInQueue = self.am_getOption( 'RequestsInQueue', self.maxRequestsInQueue )
self.threadPool = ThreadPool( 1, self.maxNumberOfThreads, self.maxRequestsInQueue )
# Set the ThreadPool in daemon mode to process new ThreadedJobs as they are inserted
self.threadPool.daemonize()
self.maxRequests = self.am_getOption( 'MaxRequestsPerCycle', 1200. )
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'DataManager' )
return S_OK()
def __init__(self, agentName, baseAgentName=False, properties=dict()):
""" c'tor
:param self: self reference
:param str agentName: name of agent
:param bool baseAgentName: whatever
:param dict properties: whatever else
"""
AgentModule.__init__(self, agentName, baseAgentName, properties)
## replica manager
self.replicaManager = ReplicaManager()
## transformation client
self.transClient = TransformationClient()
## wms client
self.wmsClient = WMSClient()
## request client
self.requestClient = RequestClient()
## file catalog clinet
self.metadataClient = FileCatalogClient()
## storage usage agent
self.storageUsageClient = StorageUsageClient()
## placeholders for CS options
## transformations types
self.transformationTypes = None
## directory locations
self.directoryLocations = None
## transformation metadata
self.transfidmeta = None
## archive periof in days
self.archiveAfter = None
## active SEs
self.activeStorages = None
## transformation log SEs
self.logSE = None
## enable/disable execution
self.enableFlag = None
def initialize( self ):
"""Sets defaults
"""
self.integrityClient = DataIntegrityClient()
self.replicaManager = ReplicaManager()
self.transClient = TransformationClient()
self.storageUsageClient = StorageUsageClient()
self.fileCatalogClient = FileCatalogClient()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'DataManager' )
self.transformationTypes = sortList( self.am_getOption( 'TransformationTypes', ['MCSimulation', 'DataReconstruction', 'DataStripping', 'MCStripping', 'Merge'] ) )
gLogger.info( "Will treat the following transformation types: %s" % str( self.transformationTypes ) )
self.directoryLocations = sortList( self.am_getOption( 'DirectoryLocations', ['TransformationDB', 'StorageUsage', 'MetadataCatalog'] ) )
gLogger.info( "Will search for directories in the following locations: %s" % str( self.directoryLocations ) )
self.activeStorages = sortList( self.am_getOption( 'ActiveSEs', [] ) )
gLogger.info( "Will check the following storage elements: %s" % str( self.activeStorages ) )
self.transfidmeta = self.am_getOption( 'TransfIDMeta', "TransformationID" )
gLogger.info( "Will use %s as metadata tag name for TransformationID" % self.transfidmeta )
return S_OK()
def initialize(self):
self.pluginLocation = self.am_getOption(
'PluginLocation',
'DIRAC.TransformationSystem.Agent.TransformationPlugin')
self.checkCatalog = self.am_getOption('CheckCatalog', 'yes')
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/ProductionManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'ProductionManager')
self.transDB = TransformationClient('TransformationDB')
self.rm = ReplicaManager()
return S_OK()
def initialize(self):
"""Sets defaults
"""
self.enableFlag = '' #defined below
self.replicaManager = ReplicaManager()
self.prodDB = TransformationClient()
self.requestClient = RequestClient()
self.taskIDName = 'TaskID'
self.externalStatus = 'ExternalStatus'
self.externalID = 'ExternalID'
self.am_setOption('PollingTime',2*60*60) #no stalled jobs are considered so can be frequent
self.enableFlag = self.am_getOption('EnableFlag', False)
self.am_setModuleParam("shifterProxy", "ProductionManager")
self.ops = Operations()
return S_OK()
def initialize( self ):
self.section = PathFinder.getAgentSection( AGENT_NAME )
self.RequestDB = RequestDBMySQL()
self.TransferDB = TransferDB()
self.DataLog = DataLoggingClient()
self.factory = StorageFactory()
self.rm = ReplicaManager()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'DataManager' )
return S_OK()
def downloadFile(TarBallURL, app_tar, folder_name):
""" Get the file locally.
"""
#need to make sure the url ends with /, other wise concatenation below returns bad url
if TarBallURL[-1] != "/":
TarBallURL += "/"
app_tar_base = os.path.basename(app_tar)
if TarBallURL.find("http://")>-1:
try :
gLogger.debug("Downloading software", '%s' % (folder_name))
#Copy the file locally, don't try to read from remote, soooo slow
#Use string conversion %s%s to set the address, makes the system more stable
urllib.urlretrieve("%s%s" % (TarBallURL, app_tar), app_tar_base)
except:
gLogger.exception()
return S_ERROR('Exception during url retrieve')
else:
rm = ReplicaManager()
resget = rm.getFile("%s%s" % (TarBallURL, app_tar))
if not resget['OK']:
gLogger.error("File could not be downloaded from the grid")
return resget
return S_OK()
def initialize( self ):
self.replicaManager = ReplicaManager()
#self.stagerClient = StorageManagerClient()
self.dataIntegrityClient = DataIntegrityClient()
self.storageDB = StorageManagementDB()
# pin lifetime = 1 day
self.pinLifetime = self.am_getOption( 'PinLifetime', THROTTLING_TIME )
# Resources helper
self.resources = Resources()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'DataManager' )
return S_OK()
def initialize( self ):
self.RequestDBClient = RequestClient()
self.ReplicaManager = ReplicaManager()
self.DataLog = DataLoggingClient()
self.maxNumberOfThreads = self.am_getOption( 'NumberOfThreads', 1 )
self.threadPoolDepth = self.am_getOption( 'ThreadPoolDepth', 1 )
self.threadPool = ThreadPool( 1, self.maxNumberOfThreads )
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'DataManager' )
return S_OK()
def __init__( self, *args, **kwargs ):
""" c'tor
"""
AgentModule.__init__( self, *args, **kwargs )
self.integrityClient = DataIntegrityClient()
self.replicaManager = ReplicaManager()
self.transClient = TransformationClient()
self.fileCatalogClient = FileCatalogClient()
agentTSTypes = self.am_getOption( 'TransformationTypes', [] )
if agentTSTypes:
self.transformationTypes = agentTSTypes
else:
self.transformationTypes = Operations().getValue( 'Transformations/DataProcessing', ['MCSimulation', 'Merge'] )
self.directoryLocations = sortList( self.am_getOption( 'DirectoryLocations', ['TransformationDB', 'MetadataCatalog'] ) )
self.activeStorages = sortList( self.am_getOption( 'ActiveSEs', [] ) )
self.transfidmeta = self.am_getOption( 'TransfIDMeta', "TransformationID" )
def __init__( self, useCertificates = False ):
"""c'tor
:param self: self reference
:param bool useCertificates: flag to enable/disable certificates
"""
Client.__init__( self )
self.log = gLogger.getSubLogger( "DataManagement/FTSClient" )
self.setServer( "DataManagement/FTSManager" )
# getting other clients
self.ftsValidator = FTSValidator()
self.replicaManager = ReplicaManager()
self.storageFactory = StorageFactory()
url = PathFinder.getServiceURL( "DataManagement/FTSManager" )
if not url:
raise RuntimeError( "CS option DataManagement/FTSManager URL is not set!" )
self.ftsManager = RPCClient( url )