def getFile(lfn, se=""):
dm = DataManager()
download_ok = 0
get_active_replicas_ok = False
lfn_on_se = False
error_msg = ""
if se:
for i in range(0, 5):
result = dm.getActiveReplicas(lfn)
if result["OK"] and result["Value"]["Successful"]:
get_active_replicas_ok = True
lfnReplicas = result["Value"]["Successful"]
if se in lfnReplicas[lfn]:
lfn_on_se = True
break
time.sleep(3)
print "- Get replicas for %s failed, try again" % lfn
if not get_active_replicas_ok:
return S_ERROR("Get replicas error: %s" % lfn)
if lfn_on_se:
se = StorageElement(se)
# try 5 times
for j in range(0, 5):
result = se.getFile(lfn)
if result["OK"] and result["Value"]["Successful"] and result["Value"]["Successful"].has_key(lfn):
break
time.sleep(random.randint(180, 600))
print "- %s getStorageFile(%s) failed, try again" % (lfn, se)
if result["OK"]:
if result["Value"]["Successful"] and result["Value"]["Successful"].has_key(lfn):
download_ok = 1
else:
error_msg = "Downloading %s from SE %s error!" % (lfn, se)
else:
error_msg = result["Message"]
else:
if se:
print 'File %s not found on SE "%s" after %s tries, trying other SE' % (lfn, se, i + 1)
# try 5 times
for j in range(0, 5):
result = dm.getFile(lfn)
if result["OK"] and result["Value"]["Successful"] and result["Value"]["Successful"].has_key(lfn):
break
time.sleep(random.randint(180, 600))
print "- getFile(%s) failed, try again" % lfn
if result["OK"]:
if result["Value"]["Successful"] and result["Value"]["Successful"].has_key(lfn):
download_ok = 2
else:
error_msg = "Downloading %s from random SE error!" % lfn
else:
error_msg = result["Message"]
if download_ok:
return S_OK({lfn: {"DownloadOK": download_ok, "Retry": j + 1}})
return S_ERROR(error_msg)
def __init__(self):
super(WhizardAnalysis, self).__init__()
self.enable = True
self.STEP_NUMBER = ''
self.debug = True
self.log = gLogger.getSubLogger("WhizardAnalysis")
self.SteeringFile = ''
self.OutputFile = ''
self.NumberOfEvents = 1
self.Lumi = 0
self.applicationName = 'whizard'
self.evttype = ""
self.RandomSeed = 0
self.getProcessInFile = False
self.datMan = DataManager()
self.processlist = None
self.parameters = {}
self.susymodel = 0
self.Model = ''
self.genmodel = GeneratorModels()
self.eventstring = [
'! ', 'Fatal error:', 'PYSTOP', 'No matrix element available',
'Floating point exception', 'Event generation finished.',
" n_events", "luminosity", " sum "
]
self.excludeAllButEventString = False
self.steeringparameters = ''
self.options = None
self.optionsdict = {}
self.OptionsDictStr = ''
self.GenLevelCutDictStr = ''
self.genlevelcuts = {}
self.willCut = False
self.useGridFiles = False
def __init__(self, soft_category):
""" Constructor
"""
self.CVMFS_DIR = '/cvmfs/sw.cta-observatory.org/software'
self.LFN_ROOT = '/vo.cta.in2p3.fr/software'
self.SOFT_CATEGORY_DICT = soft_category
self.dm = DataManager()
def __init__(self, **kwargs):
GridBackend.__init__(self, catalogue_prefix='', **kwargs)
from DIRAC.Core.Base import Script
Script.initialize()
from DIRAC.FrameworkSystem.Client.ProxyManagerClient import ProxyManagerClient
self.pm = ProxyManagerClient()
proxy = self.pm.getUserProxiesInfo()
if not proxy['OK']:
raise BackendException("Proxy error.")
from DIRAC.Interfaces.API.Dirac import Dirac
self.dirac = Dirac()
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
self.fc = FileCatalog()
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
self.dm = DataManager()
self._xattr_cmd = sh.Command('gfal-xattr').bake(_tty_out=False)
self._replica_checksum_cmd = sh.Command('gfal-sum').bake(_tty_out=False)
self._bringonline_cmd = sh.Command('gfal-legacy-bringonline').bake(_tty_out=False)
self._cp_cmd = sh.Command('gfal-copy').bake(_tty_out=False)
self._ls_se_cmd = sh.Command('gfal-ls').bake(color='never', _tty_out=False)
self._move_cmd = sh.Command('gfal-rename').bake(_tty_out=False)
self._mkdir_cmd = sh.Command('gfal-mkdir').bake(_tty_out=False)
self._replicate_cmd = sh.Command('dirac-dms-replicate-lfn').bake(_tty_out=False)
self._add_cmd = sh.Command('dirac-dms-add-file').bake(_tty_out=False)
def __copyToExternalSE(self, localFilePath, sbPath):
"""
Copy uploaded file to external SE
"""
try:
dm = DataManager()
result = dm.put(sbPath, localFilePath, self.__externalSEName)
if not result['OK']:
return result
if 'Successful' not in result['Value']:
gLogger.verbose("Oops, no successful transfers there",
str(result))
return S_ERROR(
"RM returned OK to the action but no successful transfers were there"
)
okTrans = result['Value']['Successful']
if sbPath not in okTrans:
gLogger.verbose(
"Ooops, SB transfer wasn't in the successful ones",
str(result))
return S_ERROR(
"RM returned OK to the action but SB transfer wasn't in the successful ones"
)
return S_OK((self.__externalSEName, okTrans[sbPath]))
except Exception as e:
gLogger.error("Error while moving sandbox to SE",
"%s" % repr(e).replace(',)', ')'))
return S_ERROR("Error while moving sandbox to SE")
def __init__(self): super(OverlayInput, self).__init__() self.enable = True self.STEP_NUMBER = '' self.log = gLogger.getSubLogger( "OverlayInput" ) self.applicationName = 'OverlayInput' self.curdir = os.getcwd() self.applicationLog = '' self.printoutflag = '' self.prodid = 0 self.detector = '' ##needed for backward compatibility self.detectormodel = "" self.energytouse = '' self.energy = 0 self.nbofeventsperfile = 100 self.lfns = [] self.nbfilestoget = 0 self.BkgEvtType = 'gghad' self.metaEventType = self.BkgEvtType self.BXOverlay = 0 self.ggtohadint = 3.2 self.nbsigeventsperfile = 0 self.nbinputsigfile = 1 self.NbSigEvtsPerJob = 0 self.datMan = DataManager() self.fcc = FileCatalogClient() self.site = DIRAC.siteName() self.useEnergyForFileLookup = True self.machine = 'clic_cdr' self.pathToOverlayFiles = '' self.processorName = ''
def main():
Script.parseCommandLine()
args = Script.getPositionalArgs()
if len(args) != 1:
Script.showHelp(exitCode=1)
inputFileName = args[0]
if os.path.exists(inputFileName):
lfns = [lfn.strip().split()[0] for lfn in sorted(open(inputFileName, 'r').read().splitlines())]
else:
lfns = [inputFileName]
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
dm = DataManager()
retVal = 0
for lfn in [lfn for lfn in lfns if lfn]:
gLogger.notice("Cleaning directory %r ... " % lfn)
result = dm.cleanLogicalDirectory(lfn)
if not result['OK']:
gLogger.error('Failed to clean directory', result['Message'])
retVal = -1
else:
if not result['Value']['Failed']:
gLogger.notice('OK')
else:
for folder, message in result['Value']['Failed'].items():
gLogger.error('Failed to clean folder', "%r: %s" % (folder, message))
retVal = -1
DIRACExit(retVal)
def uploadProcessListToFileCatalog(self, path_to_process_list, appVersion):
"""Upload the new processList to the FileCatalog
"""
from ILCDIRAC.Core.Utilities.FileUtils import upload
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
from DIRAC import gConfig, exit as dexit
datMan = DataManager()
LOG.notice("Removing process list from file catalog" + path_to_process_list)
res = datMan.removeFile(path_to_process_list)
if not res['OK']:
LOG.error("Could not remove process list from file catalog, do it by hand")
dexit(2)
LOG.notice("Done removing process list from file catalog")
res = upload(os.path.dirname(path_to_process_list) + "/", self.location )
if not res['OK']:
LOG.error("something went wrong in the copy")
dexit(2)
LOG.notice("Putting process list to local processlist directory")
localprocesslistpath = gConfig.getOption("/LocalSite/ProcessListPath", "")
if localprocesslistpath['Value']:
try:
localSvnRepo = "/afs/cern.ch/eng/clic/software/whizard/whizard_195/"
shutil.copy(self.location, localSvnRepo) ## because it does not make a difference if we hardcode it here or in ${DIRAC}/etc/dirac.cfg, yours truly APS, JFS
except OSError, err:
LOG.error("Copy of process list to %s failed with error %s!" % (localSvnRepo, str(err)))
try:
subprocess.call( ["svn","ci", os.path.join( localSvnRepo, os.path.basename(localprocesslistpath['Value'] )), "-m'Process list for whizard version %s'" % appVersion ], shell=False )
except OSError, err:
LOG.error("Commit failed! Error: %s" % str(err))
def uploadProcessListToFileCatalog(self, path_to_process_list, appVersion):
"""Upload the new processList to the FileCatalog
"""
from ILCDIRAC.Core.Utilities.FileUtils import upload
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
from DIRAC import gConfig, exit as dexit
datMan = DataManager()
gLogger.notice("Removing process list from file catalog" + path_to_process_list)
res = datMan.removeFile(path_to_process_list)
if not res['OK']:
gLogger.error("Could not remove process list from file catalog, do it by hand")
dexit(2)
gLogger.notice("Done removing process list from file catalog")
res = upload(os.path.dirname(path_to_process_list) + "/", self.location )
if not res['OK']:
gLogger.error("something went wrong in the copy")
dexit(2)
gLogger.notice("Putting process list to local processlist directory")
localprocesslistpath = gConfig.getOption("/LocalSite/ProcessListPath", "")
if localprocesslistpath['Value']:
try:
localSvnRepo = "/afs/cern.ch/eng/clic/software/whizard/whizard_195/"
shutil.copy(self.location, localSvnRepo) ## because it does not make a difference if we hardcode it here or in ${DIRAC}/etc/dirac.cfg, yours truly APS, JFS
except OSError, err:
gLogger.error("Copy of process list to %s failed with error %s!" % (localSvnRepo, str(err)))
try:
subprocess.call( ["svn","ci", os.path.join( localSvnRepo, os.path.basename(localprocesslistpath['Value'] )), "-m'Process list for whizard version %s'" % appVersion ], shell=False )
except OSError, err:
gLogger.error("Commit failed! Error: %s" % str(err))
def getFilesToStage( lfnList ):
""" Utility that returns out of a list of LFNs those files that are offline,
and those for which at least one copy is online
"""
onlineLFNs = set()
offlineLFNsDict = {}
if not lfnList:
return S_OK( {'onlineLFNs':list( onlineLFNs ), 'offlineLFNs': offlineLFNsDict} )
dm = DataManager()
lfnListReplicas = dm.getActiveReplicas( lfnList, getUrl = False )
if not lfnListReplicas['OK']:
return lfnListReplicas
seToLFNs = dict()
if lfnListReplicas['Value']['Failed']:
return S_ERROR( "Failures in getting replicas" )
for lfn, ld in lfnListReplicas['Value']['Successful'].iteritems():
for se in ld:
seToLFNs.setdefault( se, list() ).append( lfn )
failed = {}
for se, lfnsInSEList in seToLFNs.iteritems():
fileMetadata = StorageElement( se ).getFileMetadata( lfnsInSEList )
if not fileMetadata['OK']:
failed[se] = dict.fromkeys( lfnsInSEList, fileMetadata['Message'] )
else:
failed[se] = fileMetadata['Value']['Failed']
# is there at least one online?
for lfn, mDict in fileMetadata['Value']['Successful'].iteritems():
if mDict['Cached']:
onlineLFNs.add( lfn )
# If the file was found staged, ignore possible errors, but print out errors
if failed:
for se, seFailed in failed.items():
gLogger.error( "Errors when getting files metadata", 'at %s' % se )
for lfn, reason in seFailed.items():
gLogger.info( '%s: %s' % ( lfn, reason ) )
if lfn in onlineLFNs:
failed[se].pop( lfn )
if not failed[se]:
failed.pop( se )
if failed:
return S_ERROR( 'Could not get metadata for %d files' % \
len( set( [lfn for lfnList in failed.values() for lfn in lfnList] ) ) )
offlineLFNs = set( lfnList ) - onlineLFNs
for offlineLFN in offlineLFNs:
ses = lfnListReplicas['Value']['Successful'][offlineLFN].keys()
random.shuffle( ses )
se = ses[0]
offlineLFNsDict.setdefault( se, list() ).append( offlineLFN )
return S_OK( {'onlineLFNs':list( onlineLFNs ), 'offlineLFNs': offlineLFNsDict} )
def getFilesToStage( lfnList ):
""" Utility that returns out of a list of LFNs those files that are offline,
and those for which at least one copy is online
"""
onlineLFNs = set()
offlineLFNsDict = {}
if not lfnList:
return S_OK( {'onlineLFNs':list( onlineLFNs ), 'offlineLFNs': offlineLFNsDict} )
dm = DataManager()
lfnListReplicas = dm.getActiveReplicas( lfnList, getUrl = False )
if not lfnListReplicas['OK']:
return lfnListReplicas
seToLFNs = dict()
if lfnListReplicas['Value']['Failed']:
return S_ERROR( "Failures in getting replicas" )
for lfn, ld in lfnListReplicas['Value']['Successful'].iteritems():
for se in ld:
seToLFNs.setdefault( se, list() ).append( lfn )
failed = {}
for se, lfnsInSEList in seToLFNs.iteritems():
fileMetadata = StorageElement( se ).getFileMetadata( lfnsInSEList )
if not fileMetadata['OK']:
failed[se] = dict.fromkeys( lfnsInSEList, fileMetadata['Message'] )
else:
if fileMetadata['Value']['Failed']:
failed[se] = fileMetadata['Value']['Failed']
# is there at least one online?
for lfn, mDict in fileMetadata['Value']['Successful'].iteritems():
if 'Cached' not in mDict:
failed.setdefault( se, {} )[lfn] = 'No Cached item returned as metadata'
elif mDict['Cached']:
onlineLFNs.add( lfn )
# If the file was found staged, ignore possible errors, but print out errors
for se, seFailed in failed.items():
gLogger.error( "Errors when getting files metadata", 'at %s' % se )
for lfn, reason in seFailed.items():
gLogger.info( '%s: %s' % ( lfn, reason ) )
if lfn in onlineLFNs:
failed[se].pop( lfn )
if not failed[se]:
failed.pop( se )
if failed:
gLogger.error( "Could not get metadata", "for %d files" % len( set( [lfn for lfnList in failed.values() for lfn in lfnList] ) ) )
return S_ERROR( "Could not get metadata for files" )
offlineLFNs = set( lfnList ) - onlineLFNs
for offlineLFN in offlineLFNs:
ses = lfnListReplicas['Value']['Successful'][offlineLFN].keys()
if ses:
offlineLFNsDict.setdefault( random.choice( ses ), list() ).append( offlineLFN )
return S_OK( {'onlineLFNs':list( onlineLFNs ), 'offlineLFNs': offlineLFNsDict} )
def doesFileExist(lfn):
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
dm = DataManager()
result = dm.getActiveReplicas(lfn)
if result[('Value')][('Successful')]:
return True
else:
return False
def __init__(self, soft_category):
""" Constructor
"""
self.SW_SHARED_DIR = 'VO_VO_CTA_IN2P3_FR_SW_DIR'
self.CVMFS_DIR = '/cvmfs/cta.in2p3.fr/software'
self.LFN_ROOT = '/vo.cta.in2p3.fr/software'
self.SOFT_CATEGORY_DICT = soft_category
self.dm = DataManager()
def _replicate(self, lfn, destinationSE, sourceSE="", localCache=""):
dm = DataManager()
result = dm.replicateAndRegister(lfn, destinationSE, sourceSE, '', localCache)
if not result['OK']:
print 'ERROR %s' % (result['Message'])
return result
else:
return S_OK(result['Value']['Successful'][lfn])
def __call__(self):
""" call me maybe """
# # counter for failed files
failedFiles = 0
# # catalog to use
catalog = self.operation.Catalog
dm = DataManager(catalogs=catalog)
# # get waiting files
waitingFiles = self.getWaitingFilesList()
# # loop over files
for opFile in waitingFiles:
gMonitor.addMark("RegisterAtt", 1)
# # get LFN
lfn = opFile.LFN
# # and others
fileTuple = (lfn, opFile.PFN, opFile.Size,
self.operation.targetSEList[0], opFile.GUID,
opFile.Checksum)
# # call DataManager
registerFile = dm.registerFile(fileTuple)
# # check results
if not registerFile["OK"] or lfn in registerFile["Value"]["Failed"]:
gMonitor.addMark("RegisterFail", 1)
self.dataLoggingClient().addFileRecord(lfn, "RegisterFail",
catalog, "",
"RegisterFile")
reason = registerFile.get(
"Message",
registerFile.get("Value", {}).get("Failed",
{}).get(lfn, 'Unknown'))
errorStr = "failed to register LFN %s: %s" % (lfn, reason)
opFile.Error = errorStr
self.log.warn(errorStr)
failedFiles += 1
else:
gMonitor.addMark("RegisterOK", 1)
self.dataLoggingClient().addFileRecord(lfn, "Register",
catalog, "",
"RegisterFile")
self.log.info("file %s has been registered at %s" %
(lfn, catalog))
opFile.Status = "Done"
# # final check
if failedFiles:
self.log.info("all files processed, %s files failed to register" %
failedFiles)
self.operation.Error = "some files failed to register"
return S_ERROR(self.operation.Error)
return S_OK()
def main():
global overwrite
Script.registerSwitch("f", "force", "Force overwrite of existing file",
setOverwrite)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 1 or len(args) > 4:
Script.showHelp(exitCode=1)
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
from DIRAC import gLogger
import DIRAC
exitCode = 0
lfns = []
if len(args) == 1:
inputFileName = args[0]
if os.path.exists(inputFileName):
inputFile = open(inputFileName, "r")
for line in inputFile:
line = line.rstrip()
items = line.split()
items[0] = items[0].replace("LFN:", "").replace("lfn:", "")
lfns.append(getDict(items))
inputFile.close()
else:
gLogger.error("Error: LFN list '%s' missing." % inputFileName)
exitCode = 4
else:
lfns.append(getDict(args))
dm = DataManager()
for lfn in lfns:
if not os.path.exists(lfn["localfile"]):
gLogger.error("File %s must exist locally" % lfn["localfile"])
exitCode = 1
continue
if not os.path.isfile(lfn["localfile"]):
gLogger.error("%s is not a file" % lfn["localfile"])
exitCode = 2
continue
gLogger.notice("\nUploading %s" % lfn["lfn"])
res = dm.putAndRegister(lfn["lfn"],
lfn["localfile"],
lfn["SE"],
lfn["guid"],
overwrite=overwrite)
if not res["OK"]:
exitCode = 3
gLogger.error("Error: failed to upload %s to %s: %s" %
(lfn["lfn"], lfn["SE"], res))
continue
else:
gLogger.notice("Successfully uploaded file to %s" % lfn["SE"])
DIRAC.exit(exitCode)
def web_getSelectedFiles(self):
self.set_header('Content-type', 'text/plain')
arguments = self.request.arguments
gLogger.always("submit: incoming arguments %s to getSelectedFiles" %
arguments)
tmpdir = '/tmp/eiscat/' + str(time.time()) + str(random.random())
dataMgr = DataManager(vo=self.vo)
lfnStr = str(arguments['path'][0])
if not os.path.isdir(tmpdir): os.makedirs(tmpdir)
os.chdir(tmpdir)
for lfn in lfnStr.split(','):
gLogger.always("Data manager get file %s" % lfn)
last_slash = lfn.rfind("/")
pos_relative = lfn.find("/")
pos_relative = lfn.find("/", pos_relative + 1)
pos_relative = lfn.find("/", pos_relative + 1)
pos_relative = pos_relative
pathInZip = lfn[pos_relative:last_slash]
tmpPathInZip = tmpdir + pathInZip
gLogger.always("path in zip %s" % tmpPathInZip)
if not os.path.isdir(tmpPathInZip): os.makedirs(tmpPathInZip)
result = dataMgr.getFile(str(lfn),
destinationDir=str(tmpPathInZip))
if not result["OK"]:
gLogger.error("getSelectedFiles: %s" % result["Message"])
#make zip file
zipname = tmpdir.split('/')[-1] + '.zip'
zf = zipfile.ZipFile('/tmp/eiscat/' + zipname, "w")
gLogger.always("zip file /tmp/eiscat/%s" % zipname)
gLogger.always("start walk in tmpdir %s" % tmpdir)
for absolutePath, dirs, files in os.walk(tmpdir):
gLogger.always("absolute path %s" % absolutePath)
gLogger.always("files %s" % files)
for filename in files:
# relative path form tmpdir current chdir
pos_relative = absolutePath.find("/")
pos_relative = absolutePath.find("/", pos_relative + 1)
pos_relative = absolutePath.find("/", pos_relative + 1)
pos_relative = absolutePath.find("/", pos_relative + 1)
pos_relative = pos_relative + 1
relativePath = absolutePath[pos_relative:]
gLogger.always("relativePath %s, file %s" %
(relativePath, filename))
zf.write(os.path.join(relativePath, filename))
zf.close()
#read zip file
f = open('/tmp/eiscat/' + zipname, "rb")
obj = f.read()
f.close()
#cleanup
shutil.rmtree(tmpdir)
os.remove('/tmp/eiscat/' + zipname)
self.set_header('Content-Disposition',
'attachment; filename="' + zipname)
self.write(obj)
def _get_file(self, lfn):
dm = DataManager()
result = dm.getFile(lfn, "")
if not result['OK']:
return S_ERROR(result['Message'])
if result['Value']['Failed']:
return S_ERROR(result['Value'])
return result
def _remove_file(self, lfn):
dm = DataManager()
res = dm.removeFile([lfn])
if not res['OK']:
gLogger.error("Failed to remove data", res['Message'])
return res
if lfn in res['Value']['Successful']:
return S_OK(res['Value']['Successful'])
return S_ERROR(res['Value']['Failed'])
def __init__(self, soft_category={"corsika_simhessarray": "simulations"}):
""" Constructor
"""
self.SW_SHARED_DIR = 'VO_VO_CTA_IN2P3_FR_SW_DIR'
self.CVMFS_DIR = '/cvmfs/cta.in2p3.fr/software'
self.LFN_ROOT = '/vo.cta.in2p3.fr/software'
# self.SOFT_CATEGORY_DICT = {"corsika_simhessarray":"simulations"}
self.SOFT_CATEGORY_DICT = soft_category
self.dm = DataManager()
def __call__( self ):
""" call me maybe """
# # counter for failed files
failedFiles = 0
# # catalog(s) to use
catalogs = self.operation.Catalog
if catalogs:
catalogs = [ cat.strip() for cat in catalogs.split( ',' ) ]
dm = DataManager( catalogs = catalogs )
# # get waiting files
waitingFiles = self.getWaitingFilesList()
# # loop over files
for opFile in waitingFiles:
gMonitor.addMark( "RegisterAtt", 1 )
# # get LFN
lfn = opFile.LFN
# # and others
fileTuple = ( lfn , opFile.PFN, opFile.Size, self.operation.targetSEList[0], opFile.GUID, opFile.Checksum )
# # call DataManager
registerFile = dm.registerFile( fileTuple )
# # check results
if not registerFile["OK"] or lfn in registerFile["Value"]["Failed"]:
gMonitor.addMark( "RegisterFail", 1 )
# self.dataLoggingClient().addFileRecord( lfn, "RegisterFail", ','.join( catalogs ) if catalogs else "all catalogs", "", "RegisterFile" )
reason = str( registerFile.get( "Message", registerFile.get( "Value", {} ).get( "Failed", {} ).get( lfn, 'Unknown' ) ) )
errorStr = "failed to register LFN"
opFile.Error = "%s: %s" % ( errorStr, reason )
if 'GUID already registered' in reason:
opFile.Status = 'Failed'
self.log.error( errorStr, "%s: %s" % ( lfn, reason ) )
elif 'File already registered with no replicas' in reason:
self.log.warn( errorStr, "%s: %s, will remove it and retry" % ( lfn, reason ) )
dm.removeFile( lfn )
else:
self.log.warn( errorStr, "%s: %s" % ( lfn, reason ) )
failedFiles += 1
else:
gMonitor.addMark( "RegisterOK", 1 )
# self.dataLoggingClient().addFileRecord( lfn, "Register", ','.join( catalogs ) if catalogs else "all catalogs", "", "RegisterFile" )
self.log.verbose( "file %s has been registered at %s" % ( lfn, ','.join( catalogs ) if catalogs else "all catalogs" ) )
opFile.Status = "Done"
# # final check
if failedFiles:
self.log.warn( "all files processed, %s files failed to register" % failedFiles )
self.operation.Error = "some files failed to register"
return S_ERROR( self.operation.Error )
return S_OK()
def __call__( self ):
""" call me maybe """
# # counter for failed files
failedFiles = 0
# # catalog(s) to use
catalogs = self.operation.Catalog
if catalogs:
catalogs = [ cat.strip() for cat in catalogs.split( ',' ) ]
dm = DataManager( catalogs = catalogs )
# # get waiting files
waitingFiles = self.getWaitingFilesList()
# # loop over files
for opFile in waitingFiles:
gMonitor.addMark( "RegisterAtt", 1 )
# # get LFN
lfn = opFile.LFN
# # and others
fileTuple = ( lfn , opFile.PFN, opFile.Size, self.operation.targetSEList[0], opFile.GUID, opFile.Checksum )
# # call DataManager
registerFile = dm.registerFile( fileTuple )
# # check results
if not registerFile["OK"] or lfn in registerFile["Value"]["Failed"]:
gMonitor.addMark( "RegisterFail", 1 )
# self.dataLoggingClient().addFileRecord( lfn, "RegisterFail", ','.join( catalogs ) if catalogs else "all catalogs", "", "RegisterFile" )
reason = str( registerFile.get( "Message", registerFile.get( "Value", {} ).get( "Failed", {} ).get( lfn, 'Unknown' ) ) )
errorStr = "failed to register LFN"
opFile.Error = "%s: %s" % ( errorStr, reason )
if 'GUID already registered' in reason:
opFile.Status = 'Failed'
self.log.error( errorStr, "%s: %s" % ( lfn, reason ) )
elif 'File already registered with no replicas' in reason:
self.log.warn( errorStr, "%s: %s, will remove it and retry" % ( lfn, reason ) )
dm.removeFile( lfn )
else:
self.log.warn( errorStr, "%s: %s" % ( lfn, reason ) )
failedFiles += 1
else:
gMonitor.addMark( "RegisterOK", 1 )
# self.dataLoggingClient().addFileRecord( lfn, "Register", ','.join( catalogs ) if catalogs else "all catalogs", "", "RegisterFile" )
self.log.verbose( "file %s has been registered at %s" % ( lfn, ','.join( catalogs ) if catalogs else "all catalogs" ) )
opFile.Status = "Done"
# # final check
if failedFiles:
self.log.warn( "all files processed, %s files failed to register" % failedFiles )
self.operation.Error = "some files failed to register"
return S_ERROR( self.operation.Error )
return S_OK()
def setUp( self ):
super( FailingUserJobTestCase, self ).setUp()
dm = DataManager()
res = dm.removeFile( ['/lhcb/testCfg/testVer/LOG/00012345/0006/00012345_00067890.tar',
'/lhcb/testCfg/testVer/SIM/00012345/0006/00012345_00067890_1.sim'],
force = True )
if not res['OK']:
print("Could not remove files", res['Message'])
exit( 1 )
def doesFileExist(lfn):
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
dm = DataManager()
result = dm.getActiveReplicas(lfn)
if result[("Value")][("Successful")]:
return True
# print 'File exists.'
else:
return False
def setUp( self ):
super( IntegrationTest, self ).setUp()
dm = DataManager()
res = dm.removeFile( ['/lhcb/testCfg/testVer/LOG/00012345/0006/00012345_00067890.tar',
'/lhcb/testCfg/testVer/SIM/00012345/0006/00012345_00067890_1.sim'],
force = True )
if not res['OK']:
print "Could not remove files", res['Message']
exit( 1 )
def doesFileExist(lfn):
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
dm = DataManager()
result = dm.getActiveReplicas(lfn)
if not result['OK']:
print "ERROR",result['Message']
return False
if lfn in result['Value']['Successful']:
return True
else:
return False
def doesFileExist(lfn):
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
dm = DataManager()
result = dm.getActiveReplicas(lfn)
if not result['OK']:
print "ERROR", result['Message']
return False
if lfn in result['Value']['Successful']:
return True
else:
return False
def initialize(self):
self.fileCatalog = FileCatalog()
self.dm = DataManager()
self.stagerClient = StorageManagerClient()
self.dataIntegrityClient = DataIntegrityClient()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption("shifterProxy", "DataManager")
return S_OK()
def __init__(self, trans, transID, lfns, asIfProd):
self.trans = trans
self.transID = transID
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
self.transClient = TransformationClient()
from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient
self.bk = BookkeepingClient()
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
self.dm = DataManager()
self.asIfProd = asIfProd
(self.transFiles, self.transReplicas) = self.prepareForPlugin(lfns)
def getFilesToStage(lfnList):
""" Utility that returns out of a list of LFNs those files that are offline,
and those for which at least one copy is online
"""
onlineLFNs = set()
offlineLFNsDict = {}
if not lfnList:
return S_OK({
'onlineLFNs': list(onlineLFNs),
'offlineLFNs': offlineLFNsDict
})
dm = DataManager()
lfnListReplicas = dm.getActiveReplicas(lfnList)
if not lfnListReplicas['OK']:
return lfnListReplicas
seObjectsDict = dict()
seToLFNs = dict()
if lfnListReplicas['Value']['Failed']:
return S_ERROR("Failures in getting replicas")
for lfn, ld in lfnListReplicas['Value']['Successful'].iteritems():
for se, _ in ld.iteritems():
seObjectsDict.setdefault(se, StorageElement(se))
seToLFNs.setdefault(se, list()).append(lfn)
for se, lfnsInSEList in seToLFNs.iteritems():
fileMetadata = seObjectsDict[se].getFileMetadata(lfnsInSEList)
if not fileMetadata['OK']:
return fileMetadata
if fileMetadata['Value']['Failed']:
return S_ERROR("Failures in getting file metadata")
# is there at least one online?
for lfn, mDict in fileMetadata['Value']['Successful'].iteritems():
if mDict['Cached']:
onlineLFNs.add(lfn)
offlineLFNs = set(lfnList).difference(onlineLFNs)
for offlineLFN in offlineLFNs:
ses = lfnListReplicas['Value']['Successful'][offlineLFN].keys()
random.shuffle(ses)
se = ses[0]
offlineLFNsDict.setdefault(se, list()).append(offlineLFN)
return S_OK({
'onlineLFNs': list(onlineLFNs),
'offlineLFNs': offlineLFNsDict
})
def __init__(self):
'''
Constructor
'''
super(MoveInFC, self).__init__()
self.enable = False
self.STEP_NUMBER = ''
self.log = gLogger.getSubLogger("MoveInFC")
self.applicationName = 'MoveInFC'
self.repMan = DataManager()
self.listoutput = {}
self.outputpath = ''
def setUp(self):
super(IntegrationTest, self).setUp()
dm = DataManager()
res = dm.removeFile([
'/lhcb/testCfg/testVer/LOG/00012345/0006/00012345_00067890.tar',
'/lhcb/testCfg/testVer/SIM/00012345/0006/00012345_00067890_1.sim'
],
force=True)
if not res['OK']:
print "Could not remove files", res['Message']
exit(1)
def _add_file(self, lfn, localfile, SE, guid=None):
dm = DataManager()
self._create_test_file()
if not os.path.exists(self.options['test_file']):
gLogger.error("File %s must exist locally" % localfile)
if not os.path.isfile(self.options['test_file']):
gLogger.error("%s is not a file" % localfile)
res = dm.putAndRegister(lfn, localfile, SE, guid)
if not res['OK']:
gLogger.error('Error: failed to upload %s to %s' % (lfn, SE))
return S_ERROR(res['Message'])
return S_OK(res['Value']['Successful'][lfn])
def downloadFile(lfn):
""" Download a file using DMS
Keyword arguments:
lfn -- a logical file name
"""
DIRAC.gLogger.info('Downloading ',lfn)
dm = DataManager()
res=dm.getFile(lfn)
if not res['OK']:
DIRAC.gLogger.error ( res['Message'] )
DIRAC.gLogger.error ( 'Could not download %s'%lfn )
DIRAC.exit( -1 )
return DIRAC.S_OK()
def removeOutputData(baseDir, logLevel="INFO"):
gLogger.setLevel(logLevel)
res = getProxyInfo(False, False)
if not res["OK"]:
gLogger.error("Failed to get client proxy information.", res["Message"])
return {"OK": False, "Message": "Failed to get client proxy information: %s" % str(res["Message"]), "RC": 71}
# ######################################################################################################## #
rm = DataManager()
try:
result = rm.cleanLogicalDirectory(baseDir)
except KeyError, ke:
return {"OK": False, "Message": "Caught key error, full stacktrace below\n%s" % str(ke), "RC": 137}
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(("LocalFile: Path to local file containing LFNs",
"LFN: Logical File Names"))
Script.registerArgument(" SE: Storage element")
Script.parseCommandLine()
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
allowUsers = Operations().getValue(
"DataManagement/AllowUserReplicaManagement", False)
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
res = getProxyInfo()
if not res["OK"]:
gLogger.fatal("Can't get proxy info", res["Message"])
dexit(1)
properties = res["Value"].get("groupProperties", [])
if not allowUsers:
if "FileCatalogManagement" not in properties:
gLogger.error(
"You need to use a proxy from a group with FileCatalogManagement"
)
dexit(5)
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
dm = DataManager()
# parseCommandLine show help when mandatory arguments are not specified or incorrect argument
inputFileName, storageElementName = Script.getPositionalArgs(group=True)
if os.path.exists(inputFileName):
inputFile = open(inputFileName, "r")
string = inputFile.read()
lfns = [lfn.strip() for lfn in string.splitlines()]
inputFile.close()
else:
lfns = [inputFileName]
res = dm.removeReplicaFromCatalog(storageElementName, lfns)
if not res["OK"]:
print(res["Message"])
dexit(0)
for lfn in sorted(res["Value"]["Failed"]):
message = res["Value"]["Failed"][lfn]
print("Failed to remove %s replica of %s: %s" %
(storageElementName, lfn, message))
print("Successfully remove %d catalog replicas at %s" %
(len(res["Value"]["Successful"]), storageElementName))
def setUp(self):
super(FailingUserJobTestCase, self).setUp()
dm = DataManager()
res = dm.removeFile(
[
"/lhcb/testCfg/testVer/LOG/00012345/0006/00012345_00067890.tar",
"/lhcb/testCfg/testVer/SIM/00012345/0006/00012345_00067890_1.sim",
],
force=True,
)
if not res["OK"]:
print("Could not remove files", res["Message"])
exit(1)
def __init__(self, url=None, useCertificates=False, **kwargs):
"""c'tor
:param self: self reference
:param bool useCertificates: flag to enable/disable certificates
"""
super(FTSClient, self).__init__(**kwargs)
self.log = gLogger.getSubLogger("DataManagement/FTSClient")
self.serverURL = 'DataManagement/FTSManager' if not url else url
# getting other clients
self.ftsValidator = FTSValidator()
self.dataManager = DataManager()
self.storageFactory = StorageFactory()
def setUp(self):
super(FailingUserJobTestCase, self).setUp()
dm = DataManager()
res = dm.removeFile(
[
"/lhcb/testCfg/testVer/LOG/00012345/0006/00012345_00067890.tar",
"/lhcb/testCfg/testVer/SIM/00012345/0006/00012345_00067890_1.sim",
],
force=True,
)
if not res["OK"]:
print "Could not remove files", res["Message"]
exit(1)
def main():
dm = DataManager()
fileTupleBuffer = []
counter = 0
for f in files:
counter += 1
if not f.startswith('/cefs'):
gLogger.error('File must be under "/cefs"')
return 1
lfn = '/cepc/lustre-ro' + f
result = fcc.isFile(lfn)
if result['OK'] and lfn in result['Value']['Successful'] and result[
'Value']['Successful'][lfn]:
continue
size = os.path.getsize(f)
adler32 = fileAdler(f)
guid = makeGuid()
fileTuple = (lfn, f, size, _se, guid, adler32)
fileTupleBuffer.append(fileTuple)
gLogger.debug('Register to lfn: %s' % lfn)
gLogger.debug('fileTuple: %s' % (fileTuple, ))
if len(fileTupleBuffer) >= _bufferSize:
result = dm.registerFile(fileTupleBuffer)
print('register result', result)
if not result['OK']:
gLogger.error('Register file failed')
return 1
del fileTupleBuffer[:]
gLogger.debug('%s files registered' % counter)
if fileTupleBuffer:
result = dm.registerFile(fileTupleBuffer)
print('register result', result)
if not result['OK']:
gLogger.error('Register file failed')
return 1
del fileTupleBuffer[:]
gLogger.info('Totally %s files registered' % counter)
return 0
def getFilesToStage( lfnList ):
""" Utility that returns out of a list of LFNs those files that are offline,
and those for which at least one copy is online
"""
onlineLFNs = set()
offlineLFNsDict = {}
if not lfnList:
return S_OK( {'onlineLFNs':list( onlineLFNs ), 'offlineLFNs': offlineLFNsDict} )
dm = DataManager()
lfnListReplicas = dm.getActiveReplicas( lfnList )
if not lfnListReplicas['OK']:
return lfnListReplicas
seObjectsDict = dict()
seToLFNs = dict()
if lfnListReplicas['Value']['Failed']:
return S_ERROR( "Failures in getting replicas" )
for lfn, ld in lfnListReplicas['Value']['Successful'].iteritems():
for se, _ in ld.iteritems():
seObjectsDict.setdefault( se, StorageElement( se ) )
seToLFNs.setdefault( se, list() ).append( lfn )
for se, lfnsInSEList in seToLFNs.iteritems():
fileMetadata = seObjectsDict[se].getFileMetadata( lfnsInSEList )
if not fileMetadata['OK']:
return fileMetadata
if fileMetadata['Value']['Failed']:
return S_ERROR( "Failures in getting file metadata" )
# is there at least one online?
for lfn, mDict in fileMetadata['Value']['Successful'].iteritems():
if mDict['Cached']:
onlineLFNs.add( lfn )
offlineLFNs = set( lfnList ).difference( onlineLFNs )
for offlineLFN in offlineLFNs:
ses = lfnListReplicas['Value']['Successful'][offlineLFN].keys()
random.shuffle( ses )
se = ses[0]
offlineLFNsDict.setdefault( se, list() ).append( offlineLFN )
return S_OK( {'onlineLFNs':list( onlineLFNs ), 'offlineLFNs': offlineLFNsDict} )
def main():
Script.parseCommandLine()
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
allowUsers = Operations().getValue(
"DataManagement/AllowUserReplicaManagement", False)
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
res = getProxyInfo()
if not res['OK']:
gLogger.fatal("Can't get proxy info", res['Message'])
dexit(1)
properties = res['Value'].get('groupProperties', [])
if not allowUsers:
if 'FileCatalogManagement' not in properties:
gLogger.error(
"You need to use a proxy from a group with FileCatalogManagement"
)
dexit(5)
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
dm = DataManager()
args = Script.getPositionalArgs()
if len(args) < 2:
Script.showHelp(exitCode=1)
else:
inputFileName = args[0]
storageElementName = args[1]
if os.path.exists(inputFileName):
inputFile = open(inputFileName, 'r')
string = inputFile.read()
lfns = [lfn.strip() for lfn in string.splitlines()]
inputFile.close()
else:
lfns = [inputFileName]
res = dm.removeReplicaFromCatalog(storageElementName, lfns)
if not res['OK']:
print(res['Message'])
dexit(0)
for lfn in sorted(res['Value']['Failed']):
message = res['Value']['Failed'][lfn]
print('Failed to remove %s replica of %s: %s' %
(storageElementName, lfn, message))
print('Successfully remove %d catalog replicas at %s' %
(len(res['Value']['Successful']), storageElementName))
def addDataFiles(args):
fcc = FileCatalogClient()
dm = DataManager(['FileCatalog', 'TSCatalog'])
outputPath = args[0]
outputPattern = args[1]
outputSE = args[2]
metadata = args[3]
metadata = json.loads(metadata)
## Create path
res = fcc.createDirectory(outputPath)
if not res['OK']:
return res
## Set metadata
for key, value in metadata.items():
res = fcc.setMetadata(outputPath, {key: value})
# terrible hack
# if not res['OK']:
# return res
## Upload data files
all_files = glob.glob(outputPattern)
## Check that data files exist
if len(all_files) == 0:
return DIRAC.S_ERROR('No data files found')
for one_file in all_files:
lfn = os.path.join(outputPath, one_file)
msg = 'Try to upload local file: %s \nwith LFN: %s \nto %s' % (
one_file, lfn, outputSE)
DIRAC.gLogger.notice(msg)
res = dm.putAndRegister(lfn, one_file, outputSE)
## Check if failed
if not res['OK']:
DIRAC.gLogger.error(
'Failed to putAndRegister %s \nto %s \nwith message: %s' %
(lfn, outputSE, res['Message']))
return res
elif res['Value']['Failed'].has_key(lfn):
DIRAC.gLogger.error('Failed to putAndRegister %s to %s' %
(lfn, outputSE))
return res
return DIRAC.S_OK()
def __init__(self):
super(WhizardAnalysis, self).__init__()
self.enable = True
self.STEP_NUMBER = ''
self.debug = True
self.SteeringFile = ''
self.OutputFile = ''
self.NumberOfEvents = 1
self.Lumi = 0
self.applicationName = 'whizard'
self.evttype = ""
self.RandomSeed = 0
self.getProcessInFile = False
self.datMan = DataManager()
self.processlist = None
self.parameters = {}
self.susymodel = 0
self.Model = ''
self.genmodel = GeneratorModels()
self.eventstring = ['! ', 'Fatal error:', 'PYSTOP', 'No matrix element available',
'Floating point exception', 'Event generation finished.', " n_events","luminosity",
" sum "]
self.excludeAllButEventString = False
self.steeringparameters = ''
self.options = None
self.optionsdict = {}
self.OptionsDictStr = ''
self.GenLevelCutDictStr = ''
self.genlevelcuts = {}
self.willCut = False
self.useGridFiles = False
def main():
Script.parseCommandLine()
from DIRAC import gLogger
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
args = Script.getPositionalArgs()
if not len(args) == 2:
Script.showHelp(exitCode=1)
else:
inputFileName = args[0]
storageElement = args[1]
if os.path.exists(inputFileName):
inputFile = open(inputFileName, 'r')
string = inputFile.read()
lfns = [lfn.strip() for lfn in string.splitlines()]
inputFile.close()
else:
lfns = [inputFileName]
res = DataManager().getReplicaMetadata(lfns, storageElement)
if not res['OK']:
print('Error:', res['Message'])
DIRACExit(1)
print('%s %s %s %s' % ('File'.ljust(100), 'Migrated'.ljust(8),
'Cached'.ljust(8), 'Size (bytes)'.ljust(10)))
for lfn, metadata in res['Value']['Successful'].items():
print('%s %s %s %s' %
(lfn.ljust(100), str(metadata['Migrated']).ljust(8),
str(metadata.get('Cached', metadata['Accessible'])).ljust(8),
str(metadata['Size']).ljust(10)))
for lfn, reason in res['Value']['Failed'].items():
print('%s %s' % (lfn.ljust(100), reason.ljust(8)))
def cleanMetadataCatalogFiles(self, transID):
""" wipe out files from catalog """
res = self.metadataClient.findFilesByMetadata(
{self.transfidmeta: transID})
if not res['OK']:
return res
fileToRemove = res['Value']
if not fileToRemove:
self.log.info('No files found for transID %s' % transID)
return S_OK()
# Executing with shifter proxy
gConfigurationData.setOptionInCFG(
'/DIRAC/Security/UseServerCertificate', 'false')
res = DataManager().removeFile(fileToRemove, force=True)
gConfigurationData.setOptionInCFG(
'/DIRAC/Security/UseServerCertificate', 'true')
if not res['OK']:
return res
for lfn, reason in res['Value']['Failed'].items():
self.log.error("Failed to remove file found in metadata catalog",
"%s %s" % (lfn, reason))
if res['Value']['Failed']:
return S_ERROR(
"Failed to remove all files found in the metadata catalog")
self.log.info("Successfully removed all files found in the BK")
return S_OK()
def __init__(self): super(OverlayInput, self).__init__() self.enable = True self.STEP_NUMBER = '' self.applicationName = 'OverlayInput' self.curdir = os.getcwd() self.applicationLog = '' self.printoutflag = '' self.prodid = 0 self.detector = '' ##needed for backward compatibility self.detectormodel = "" self.energytouse = '' self.energy = 0 self.nbofeventsperfile = 100 self.lfns = [] self.nbfilestoget = 0 self.BkgEvtType = 'gghad' self.metaEventType = self.BkgEvtType self.BXOverlay = 0 self.ggtohadint = 3.2 self.nbsigeventsperfile = 0 self.nbinputsigfile = 1 self.NbSigEvtsPerJob = 0 self.datMan = DataManager() self.fcc = FileCatalogClient() self.site = DIRAC.siteName() self.useEnergyForFileLookup = True self.machine = 'clic_cdr' self.pathToOverlayFiles = '' self.processorName = ''
def __call__( self ):
""" call me maybe """
# # counter for failed files
failedFiles = 0
# # catalog to use
catalog = self.operation.Catalog
dm = DataManager( catalogs = catalog )
# # get waiting files
waitingFiles = self.getWaitingFilesList()
# # loop over files
for opFile in waitingFiles:
gMonitor.addMark( "RegisterAtt", 1 )
# # get LFN
lfn = opFile.LFN
# # and others
fileTuple = ( lfn , opFile.PFN, opFile.Size, self.operation.targetSEList[0], opFile.GUID, opFile.Checksum )
# # call DataManager
registerFile = dm.registerFile( fileTuple )
# # check results
if not registerFile["OK"] or lfn in registerFile["Value"]["Failed"]:
gMonitor.addMark( "RegisterFail", 1 )
self.dataLoggingClient().addFileRecord( lfn, "RegisterFail", catalog, "", "RegisterFile" )
reason = registerFile.get( "Message", registerFile.get( "Value", {} ).get( "Failed", {} ).get( lfn, 'Unknown' ) )
errorStr = "failed to register LFN %s: %s" % ( lfn, reason )
opFile.Error = errorStr
self.log.warn( errorStr )
failedFiles += 1
else:
gMonitor.addMark( "RegisterOK", 1 )
self.dataLoggingClient().addFileRecord( lfn, "Register", catalog, "", "RegisterFile" )
self.log.info( "file %s has been registered at %s" % ( lfn, catalog ) )
opFile.Status = "Done"
# # final check
if failedFiles:
self.log.info( "all files processed, %s files failed to register" % failedFiles )
self.operation.Error = "some files failed to register"
return S_ERROR( self.operation.Error )
return S_OK()
def __prepareFileForHTTP( self, lfn, key ):
""" proxied preapre file for HTTP """
global HTTP_PATH
res = self.__prepareSecurityDetails()
if not res['OK']:
return res
# Clear the local cache
getFileDir = "%s/%s" % ( HTTP_PATH, key )
os.makedirs(getFileDir)
# Get the file to the cache
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
dataMgr = DataManager()
result = dataMgr.getFile( lfn, destinationDir = getFileDir )
result['CachePath'] = getFileDir
return result
def __copyToExternalSE( self, localFilePath, sbPath ):
"""
Copy uploaded file to external SE
"""
try:
dm = DataManager()
result = dm.put( sbPath, localFilePath, self.__externalSEName )
if not result[ 'OK' ]:
return result
if 'Successful' not in result[ 'Value' ]:
gLogger.verbose( "Oops, no successful transfers there", str( result ) )
return S_ERROR( "RM returned OK to the action but no successful transfers were there" )
okTrans = result[ 'Value' ][ 'Successful' ]
if sbPath not in okTrans:
gLogger.verbose( "Ooops, SB transfer wasn't in the successful ones", str( result ) )
return S_ERROR( "RM returned OK to the action but SB transfer wasn't in the successful ones" )
return S_OK( ( self.__externalSEName, okTrans[ sbPath ] ) )
except Exception, e:
return S_ERROR( "Error while moving sandbox to SE: %s" % str( e ) )
def initialize( self ):
self.fileCatalog = FileCatalog()
self.dm = DataManager()
self.stagerClient = StorageManagerClient()
self.dataIntegrityClient = DataIntegrityClient()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'DataManager' )
return S_OK()
def __init__( self, args = None, apis = None ):
super( SETest, self ).__init__( args, apis )
self.__lfnPath = '/bes/user/z/zhaoxh/'
self.__testFile = 'test.dat'
self.__localPath = '/tmp/'
if 'DataManager' in self.apis:
self.dm = self.apis[ 'DataManager' ]
else:
self.dm = DataManager()
def __init__(self):
'''
Constructor
'''
super(MoveInFC, self).__init__()
self.enable = False
self.STEP_NUMBER = ''
self.applicationName = 'MoveInFC'
self.repMan = DataManager()
self.listoutput = {}
self.outputpath = ''
def finish(self):
"""
after having set all the files, this one does all the job
@return:
"""
rc = 0
rm = DataManager()
for item in self.listFileStaged:
# print("SE '"+self.SE+"' == : '"+str(self.SE == "False")+"'")
if not self.SE:
self.log.info("No SE available for '" + item[0] + "'")
rc += 1
continue
else:
self.log.info("Trying to store '" + item[0] + "' in SE : '" + self.SE + "' ...")
result = rm.putAndRegister(item[1], item[0], self.SE)
if not result["OK"]:
self.log.info("ERROR %s" % (result["Message"]))
self.log.info("Wait 5sec before trying again...")
time.sleep(5)
result = rm.putAndRegister(item[1], item[0], self.SE)
if not result["OK"]:
self.log.info("ERROR %s" % (result["Message"]))
while not result["OK"]:
self.listSEs.remove(self.SE) # make sure not to pick the same SE again.
self.__pickRandomSE()
if not self.SE:
rc += 1
break
self.log.info("Trying with another SE : '" + self.SE + "' . In 5sec...")
time.sleep(5)
result = rm.putAndRegister(item[1], item[0], self.SE)
if result["OK"]:
self.log.info("file stored : '" + item[1] + "' in '" + self.SE + "'")
else:
self.log.error("ERROR : failed to store the file '" + item[1] + "' ...")
rc += 1
return rc
def getProcessList(self):
""" Get the :mod:`ProcessList <ILCDIRAC.Core.Utilities.ProcessList.ProcessList>`
needed by :mod:`Whizard <ILCDIRAC.Interfaces.API.NewInterface.Applications.Whizard>`.
:return: process list object
"""
processlistpath = gConfig.getValue("/LocalSite/ProcessListPath", "")
if not processlistpath:
gLogger.info('Will download the process list locally. To gain time, please put it somewhere and add to \
your dirac.cfg the entry /LocalSite/ProcessListPath pointing to the file')
pathtofile = self.ops.getValue("/ProcessList/Location", "")
if not pathtofile:
gLogger.error("Could not get path to process list")
processlist = ""
else:
datMan = DataManager()
datMan.getFile(pathtofile)
processlist = os.path.basename(pathtofile)
else:
processlist = processlistpath
self.processList = ProcessList(processlist)
return self.processList
def __init__(self): super(Whizard2Analysis, self).__init__() self.enable = True self.STEP_NUMBER = '' self.result = S_ERROR() self.applicationName = 'whizard2' self.startFrom = 0 self.randomSeed = -1 self.whizard2SinFile = '' self.eventstring = ['+++ Generating event'] self.decayProc = ['decay_proc'] self.integratedProcess = '' self.datMan = DataManager()
def initialize( self ):
""" agent initialisation
reading and setting confing opts
:param self: self reference
"""
# # shifter proxy
self.am_setOption( 'shifterProxy', 'DataManager' )
# # transformations types
self.dataProcTTypes = Operations().getValue( 'Transformations/DataProcessing', ['MCSimulation', 'Merge'] )
self.dataManipTTypes = Operations().getValue( 'Transformations/DataManipulation', ['Replication', 'Removal'] )
agentTSTypes = self.am_getOption( 'TransformationTypes', [] )
if agentTSTypes:
self.transformationTypes = sorted( agentTSTypes )
else:
self.transformationTypes = sorted( self.dataProcTTypes + self.dataManipTTypes )
self.log.info( "Will consider the following transformation types: %s" % str( self.transformationTypes ) )
# # directory locations
self.directoryLocations = sorted( self.am_getOption( 'DirectoryLocations', [ 'TransformationDB',
'MetadataCatalog' ] ) )
self.log.info( "Will search for directories in the following locations: %s" % str( self.directoryLocations ) )
# # transformation metadata
self.transfidmeta = self.am_getOption( 'TransfIDMeta', "TransformationID" )
self.log.info( "Will use %s as metadata tag name for TransformationID" % self.transfidmeta )
# # archive periof in days
self.archiveAfter = self.am_getOption( 'ArchiveAfter', 7 ) # days
self.log.info( "Will archive Completed transformations after %d days" % self.archiveAfter )
# # active SEs
self.activeStorages = sorted( self.am_getOption( 'ActiveSEs', [] ) )
self.log.info( "Will check the following storage elements: %s" % str( self.activeStorages ) )
# # transformation log SEs
self.logSE = self.am_getOption( 'TransformationLogSE', 'LogSE' )
self.log.info( "Will remove logs found on storage element: %s" % self.logSE )
# # enable/disable execution, should be using CS option Status?? with default value as 'Active'??
self.enableFlag = self.am_getOption( 'EnableFlag', 'True' )
# # data manager
self.dm = DataManager()
# # transformation client
self.transClient = TransformationClient()
# # wms client
self.wmsClient = WMSClient()
# # request client
self.reqClient = ReqClient()
# # file catalog client
self.metadataClient = FileCatalogClient()
return S_OK()
def downloadFile(tarballURL, app_tar, folder_name):
""" Get the file locally.
"""
#need to make sure the url ends with /, other wise concatenation below returns bad url
if tarballURL[-1] != "/":
tarballURL += "/"
app_tar_base = os.path.basename(app_tar)
if tarballURL.find("http://")>-1:
try :
gLogger.debug("Downloading software", '%s' % (folder_name))
#Copy the file locally, don't try to read from remote, soooo slow
#Use string conversion %s%s to set the address, makes the system more stable
urllib.urlretrieve("%s%s" % (tarballURL, app_tar), app_tar_base)
except IOError as err:
gLogger.exception(str(err))
return S_ERROR('Exception during url retrieve: %s' % str(err))
else:
datMan = DataManager()
resget = datMan.getFile("%s%s" % (tarballURL, app_tar))
if not resget['OK']:
gLogger.error("File could not be downloaded from the grid")
return resget
return S_OK()
def initializeOptimizer( self ):
"""Initialize specific parameters for JobSanityAgent.
"""
self.failedMinorStatus = self.am_getOption( '/FailedJobStatus', 'Input Data Not Available' )
#this will ignore failover SE files
self.checkFileMetadata = self.am_getOption( 'CheckFileMetadata', True )
self.dataManager = DataManager()
self.resourceStatus = ResourceStatus()
self.fc = FileCatalog()
self.seToSiteMapping = {}
self.lastCScheck = 0
self.cacheLength = 600
return S_OK()
def __init__( self, useCertificates = False ):
"""c'tor
:param self: self reference
:param bool useCertificates: flag to enable/disable certificates
"""
Client.__init__( self )
self.log = gLogger.getSubLogger( "DataManagement/FTSClient" )
self.setServer( "DataManagement/FTSManager" )
# getting other clients
self.ftsValidator = FTSValidator()
self.dataManager = DataManager()
self.storageFactory = StorageFactory()
url = PathFinder.getServiceURL( "DataManagement/FTSManager" )
if not url:
raise RuntimeError( "CS option DataManagement/FTSManager URL is not set!" )
self.ftsManager = RPCClient( url )