def __init__(self, operation=None, csPath=None):
"""c'tor
:param self: self reference
:param Operation operation: Operation instance
:param str csPath: CS path for this handler
"""
# # base classes ctor
super(PutAndRegister, self).__init__(operation, csPath)
# # gMonitor stuff
gMonitor.registerActivity("PutAtt", "File put attempts",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("PutFail", "Failed file puts",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("PutOK", "Successful file puts",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RegisterOK",
"Successful file registrations",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RegisterFail", "Failed file registrations",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
self.dm = DataManager()
def cleanMetadataCatalogFiles(self, transID):
""" wipe out files from catalog """
res = self.metadataClient.findFilesByMetadata(
{self.transfidmeta: transID})
if not res['OK']:
return res
fileToRemove = res['Value']
if not fileToRemove:
self.log.info('No files found for transID %s' % transID)
return S_OK()
# Executing with shifter proxy
gConfigurationData.setOptionInCFG(
'/DIRAC/Security/UseServerCertificate', 'false')
res = DataManager().removeFile(fileToRemove, force=True)
gConfigurationData.setOptionInCFG(
'/DIRAC/Security/UseServerCertificate', 'true')
if not res['OK']:
return res
for lfn, reason in res['Value']['Failed'].items():
self.log.error("Failed to remove file found in metadata catalog",
"%s %s" % (lfn, reason))
if res['Value']['Failed']:
return S_ERROR(
"Failed to remove all files found in the metadata catalog")
self.log.info("Successfully removed all files found in the BK")
return S_OK()
def __init__(self, **kwargs):
GridBackend.__init__(self, catalogue_prefix='', **kwargs)
from DIRAC.Core.Base import Script
Script.initialize()
from DIRAC.FrameworkSystem.Client.ProxyManagerClient import ProxyManagerClient
self.pm = ProxyManagerClient()
proxy = self.pm.getUserProxiesInfo()
if not proxy['OK']:
raise BackendException("Proxy error.")
from DIRAC.Interfaces.API.Dirac import Dirac
self.dirac = Dirac()
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
self.fc = FileCatalog()
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
self.dm = DataManager()
self._xattr_cmd = sh.Command('gfal-xattr').bake(_tty_out=False)
self._replica_checksum_cmd = sh.Command('gfal-sum').bake(_tty_out=False)
self._bringonline_cmd = sh.Command('gfal-legacy-bringonline').bake(_tty_out=False)
self._cp_cmd = sh.Command('gfal-copy').bake(_tty_out=False)
self._ls_se_cmd = sh.Command('gfal-ls').bake(color='never', _tty_out=False)
self._move_cmd = sh.Command('gfal-rename').bake(_tty_out=False)
self._mkdir_cmd = sh.Command('gfal-mkdir').bake(_tty_out=False)
self._replicate_cmd = sh.Command('dirac-dms-replicate-lfn').bake(_tty_out=False)
self._add_cmd = sh.Command('dirac-dms-add-file').bake(_tty_out=False)
def main():
Script.parseCommandLine()
args = Script.getPositionalArgs()
if len(args) != 1:
Script.showHelp(exitCode=1)
inputFileName = args[0]
if os.path.exists(inputFileName):
lfns = [lfn.strip().split()[0] for lfn in sorted(open(inputFileName, 'r').read().splitlines())]
else:
lfns = [inputFileName]
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
dm = DataManager()
retVal = 0
for lfn in [lfn for lfn in lfns if lfn]:
gLogger.notice("Cleaning directory %r ... " % lfn)
result = dm.cleanLogicalDirectory(lfn)
if not result['OK']:
gLogger.error('Failed to clean directory', result['Message'])
retVal = -1
else:
if not result['Value']['Failed']:
gLogger.notice('OK')
else:
for folder, message in result['Value']['Failed'].items():
gLogger.error('Failed to clean folder', "%r: %s" % (folder, message))
retVal = -1
DIRACExit(retVal)
def __init__(self):
super(WhizardAnalysis, self).__init__()
self.enable = True
self.STEP_NUMBER = ''
self.debug = True
self.log = gLogger.getSubLogger("WhizardAnalysis")
self.SteeringFile = ''
self.OutputFile = ''
self.NumberOfEvents = 1
self.Lumi = 0
self.applicationName = 'whizard'
self.evttype = ""
self.RandomSeed = 0
self.getProcessInFile = False
self.datMan = DataManager()
self.processlist = None
self.parameters = {}
self.susymodel = 0
self.Model = ''
self.genmodel = GeneratorModels()
self.eventstring = [
'! ', 'Fatal error:', 'PYSTOP', 'No matrix element available',
'Floating point exception', 'Event generation finished.',
" n_events", "luminosity", " sum "
]
self.excludeAllButEventString = False
self.steeringparameters = ''
self.options = None
self.optionsdict = {}
self.OptionsDictStr = ''
self.GenLevelCutDictStr = ''
self.genlevelcuts = {}
self.willCut = False
self.useGridFiles = False
def __init__(self, loggerIn=None):
""" Initialization of module base.
loggerIn is a logger object that can be passed so that the logging will be more clear.
"""
if not loggerIn:
self.log = gLogger.getSubLogger('ModuleBase')
else:
self.log = loggerIn
# These 2 are used in many places, so it's good to have them available here.
self.opsH = Operations()
self.dm = DataManager()
# Some job parameters
self.production_id = 0
self.prod_job_id = 0
self.jobID = 0
self.step_number = 0
self.step_id = 0
self.jobType = ''
self.executable = ''
self.command = None
self.workflowStatus = None
self.stepStatus = None
self.workflow_commons = None
self.step_commons = None
# These are useful objects (see the getFileReporter(), getJobReporter() and getRequestContainer() functions)
self.fileReport = None
self.jobReport = None
self.request = None
def __init__(self, soft_category):
""" Constructor
"""
self.CVMFS_DIR = '/cvmfs/sw.cta-observatory.org/software'
self.LFN_ROOT = '/vo.cta.in2p3.fr/software'
self.SOFT_CATEGORY_DICT = soft_category
self.dm = DataManager()
def main():
Script.parseCommandLine()
from DIRAC import gLogger
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
args = Script.getPositionalArgs()
if not len(args) == 2:
Script.showHelp(exitCode=1)
else:
inputFileName = args[0]
storageElement = args[1]
if os.path.exists(inputFileName):
inputFile = open(inputFileName, 'r')
string = inputFile.read()
lfns = [lfn.strip() for lfn in string.splitlines()]
inputFile.close()
else:
lfns = [inputFileName]
res = DataManager().getReplicaMetadata(lfns, storageElement)
if not res['OK']:
print('Error:', res['Message'])
DIRACExit(1)
print('%s %s %s %s' % ('File'.ljust(100), 'Migrated'.ljust(8),
'Cached'.ljust(8), 'Size (bytes)'.ljust(10)))
for lfn, metadata in res['Value']['Successful'].items():
print('%s %s %s %s' %
(lfn.ljust(100), str(metadata['Migrated']).ljust(8),
str(metadata.get('Cached', metadata['Accessible'])).ljust(8),
str(metadata['Size']).ljust(10)))
for lfn, reason in res['Value']['Failed'].items():
print('%s %s' % (lfn.ljust(100), reason.ljust(8)))
def syncDestinations(upload, source_dir, dest_dir, storage, delete,
nthreads):
"""
Top level wrapper to execute functions
"""
fc = FileCatalog()
dm = DataManager()
result = getContentToSync(upload, fc, source_dir, dest_dir)
if not result['OK']:
return S_ERROR(result['Message'])
if upload:
res = doUpload(fc, dm, result, source_dir, dest_dir, storage,
delete, nthreads)
if not res['OK']:
return S_ERROR('Upload failed: ' + res['Message'])
else:
res = doDownload(dm, result, source_dir, dest_dir, delete,
nthreads)
if not res['OK']:
return S_ERROR('Download failed: ' + res['Message'])
return S_OK('Mirroring successfully finished')
def __copyToExternalSE(self, localFilePath, sbPath):
"""
Copy uploaded file to external SE
"""
try:
dm = DataManager()
result = dm.put(sbPath, localFilePath, self.__externalSEName)
if not result['OK']:
return result
if 'Successful' not in result['Value']:
gLogger.verbose("Oops, no successful transfers there",
str(result))
return S_ERROR(
"RM returned OK to the action but no successful transfers were there"
)
okTrans = result['Value']['Successful']
if sbPath not in okTrans:
gLogger.verbose(
"Ooops, SB transfer wasn't in the successful ones",
str(result))
return S_ERROR(
"RM returned OK to the action but SB transfer wasn't in the successful ones"
)
return S_OK((self.__externalSEName, okTrans[sbPath]))
except Exception as e:
gLogger.error("Error while moving sandbox to SE",
"%s" % repr(e).replace(',)', ')'))
return S_ERROR("Error while moving sandbox to SE")
def __init__(self, plugin, transClient=None, dataManager=None, fc=None):
"""Constructor of the TransformationPlugin.
Instantiate clients, if not given, and set up the PluginUtilities.
"""
super(TransformationPlugin, self).__init__(plugin)
self.data = {}
self.files = False
self.startTime = time.time()
self.valid = False
if transClient is None:
transClient = TransformationClient()
if dataManager is None:
dataManager = DataManager()
if fc is None:
fc = FileCatalog()
self.util = PluginUtilities(plugin,
transClient=transClient,
dataManager=dataManager,
fc=fc)
def initialize(self):
""" agent's initalisation
:param self: self reference
"""
gLogger.info('Initialize')
self.dataManager = DataManager()
self.am_setOption('shifterProxy', 'DataManager')
self.CopyToSE = (self.am_getOption("CopyToSE", 'PNNL-PIC-SRM-SE'))
self.SEDataDirPath = (self.am_getOption("SEDataDirPath",
'/project8/dirac/calib/'))
self.LocalDataDirPath = (self.am_getOption("LocalDataDirPath",
'/data_claude/'))
self.DIRACCfgSEPath = 'Resources/StorageElements'
self.dryRun = bool(self.am_getOption("DryRun", True))
self.fc = FileCatalogClient()
gLogger.info("DryRun: " + str(self.dryRun))
gLogger.info("CopyToSE: " + str(self.CopyToSE))
### This defines which calibration directories to consider
self.calibDirs = ['rf_bkgd', 'esr']
### This defines which sub dir under calib dir to replicate
self.rawDataDir = 'raw'
self.acceptableFileSuffix = [
'-esr.json', '.json', '.dpt', '.root', '.Setup'
]
return S_OK()
def __init__(self): super(OverlayInput, self).__init__() self.enable = True self.STEP_NUMBER = '' self.log = gLogger.getSubLogger( "OverlayInput" ) self.applicationName = 'OverlayInput' self.curdir = os.getcwd() self.applicationLog = '' self.printoutflag = '' self.prodid = 0 self.detector = '' ##needed for backward compatibility self.detectormodel = "" self.energytouse = '' self.energy = 0 self.nbofeventsperfile = 100 self.lfns = [] self.nbfilestoget = 0 self.BkgEvtType = 'gghad' self.metaEventType = self.BkgEvtType self.BXOverlay = 0 self.ggtohadint = 3.2 self.nbsigeventsperfile = 0 self.nbinputsigfile = 1 self.NbSigEvtsPerJob = 0 self.datMan = DataManager() self.fcc = FileCatalogClient() self.site = DIRAC.siteName() self.useEnergyForFileLookup = True self.machine = 'clic_cdr' self.pathToOverlayFiles = '' self.processorName = ''
def setUp(self):
result = getProxyInfo()
if result['Value']['group'] not in ['lhcb_user', 'dirac_user']:
print "GET A USER GROUP"
exit(1)
result = ResourceStatus().getElementStatus('PIC-USER',
'StorageElement',
'WriteAccess')
if result['Value']['PIC-USER']['WriteAccess'].lower() != 'banned':
print "BAN PIC-USER in writing! and then restart this test"
exit(1)
res = DataManager().getReplicas([
'/lhcb/user/f/fstagni/test/testInputFileSingleLocation.txt',
'/lhcb/user/f/fstagni/test/testInputFile.txt'
])
if not res['OK']:
print "DATAMANAGER.getRepicas failure: %s" % res['Message']
exit(1)
if res['Value']['Failed']:
print "DATAMANAGER.getRepicas failed for something: %s" % res[
'Value']['Failed']
exit(1)
replicas = res['Value']['Successful']
if replicas['/lhcb/user/f/fstagni/test/testInputFile.txt'].keys() != [
'CERN-USER', 'IN2P3-USER'
]:
print "/lhcb/user/f/fstagni/test/testInputFile.txt locations are not correct"
if replicas[
'/lhcb/user/f/fstagni/test/testInputFileSingleLocation.txt'].keys(
) != ['CERN-USER']:
print "/lhcb/user/f/fstagni/test/testInputFileSingleLocation.txt locations are not correct"
def cleanCatalogContents( self, directory ):
""" wipe out everything from catalog under folder :directory:
:param self: self reference
:params str directory: folder name
"""
self.log.verbose("Cleaning Catalog contents")
res = self.__getCatalogDirectoryContents( [directory] )
if not res['OK']:
return res
filesFound = res['Value']
if not filesFound:
self.log.info( "No files are registered in the catalog directory %s" % directory )
return S_OK()
self.log.info( "Attempting to remove %d possible remnants from the catalog and storage" % len( filesFound ) )
# Executing with shifter proxy
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' )
res = DataManager().removeFile( filesFound, force = True )
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true' )
if not res['OK']:
return res
realFailure = False
for lfn, reason in res['Value']['Failed'].items():
if "File does not exist" in str( reason ):
self.log.warn( "File %s not found in some catalog: " % ( lfn ) )
else:
self.log.error( "Failed to remove file found in the catalog", "%s %s" % ( lfn, reason ) )
realFailure = True
if realFailure:
return S_ERROR( "Failed to remove all files found in the catalog" )
return S_OK()
def uploadProcessListToFileCatalog(self, path_to_process_list, appVersion):
"""Upload the new processList to the FileCatalog
"""
from ILCDIRAC.Core.Utilities.FileUtils import upload
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
from DIRAC import gConfig, exit as dexit
datMan = DataManager()
gLogger.notice("Removing process list from file catalog" + path_to_process_list)
res = datMan.removeFile(path_to_process_list)
if not res['OK']:
gLogger.error("Could not remove process list from file catalog, do it by hand")
dexit(2)
gLogger.notice("Done removing process list from file catalog")
res = upload(os.path.dirname(path_to_process_list) + "/", self.location )
if not res['OK']:
gLogger.error("something went wrong in the copy")
dexit(2)
gLogger.notice("Putting process list to local processlist directory")
localprocesslistpath = gConfig.getOption("/LocalSite/ProcessListPath", "")
if localprocesslistpath['Value']:
try:
localSvnRepo = "/afs/cern.ch/eng/clic/software/whizard/whizard_195/"
shutil.copy(self.location, localSvnRepo) ## because it does not make a difference if we hardcode it here or in ${DIRAC}/etc/dirac.cfg, yours truly APS, JFS
except OSError, err:
gLogger.error("Copy of process list to %s failed with error %s!" % (localSvnRepo, str(err)))
try:
subprocess.call( ["svn","ci", os.path.join( localSvnRepo, os.path.basename(localprocesslistpath['Value'] )), "-m'Process list for whizard version %s'" % appVersion ], shell=False )
except OSError, err:
gLogger.error("Commit failed! Error: %s" % str(err))
def getFilesToStage( lfnList ):
""" Utility that returns out of a list of LFNs those files that are offline,
and those for which at least one copy is online
"""
onlineLFNs = set()
offlineLFNsDict = {}
if not lfnList:
return S_OK( {'onlineLFNs':list( onlineLFNs ), 'offlineLFNs': offlineLFNsDict} )
dm = DataManager()
lfnListReplicas = dm.getActiveReplicas( lfnList, getUrl = False )
if not lfnListReplicas['OK']:
return lfnListReplicas
seToLFNs = dict()
if lfnListReplicas['Value']['Failed']:
return S_ERROR( "Failures in getting replicas" )
for lfn, ld in lfnListReplicas['Value']['Successful'].iteritems():
for se in ld:
seToLFNs.setdefault( se, list() ).append( lfn )
failed = {}
for se, lfnsInSEList in seToLFNs.iteritems():
fileMetadata = StorageElement( se ).getFileMetadata( lfnsInSEList )
if not fileMetadata['OK']:
failed[se] = dict.fromkeys( lfnsInSEList, fileMetadata['Message'] )
else:
failed[se] = fileMetadata['Value']['Failed']
# is there at least one online?
for lfn, mDict in fileMetadata['Value']['Successful'].iteritems():
if mDict['Cached']:
onlineLFNs.add( lfn )
# If the file was found staged, ignore possible errors, but print out errors
if failed:
for se, seFailed in failed.items():
gLogger.error( "Errors when getting files metadata", 'at %s' % se )
for lfn, reason in seFailed.items():
gLogger.info( '%s: %s' % ( lfn, reason ) )
if lfn in onlineLFNs:
failed[se].pop( lfn )
if not failed[se]:
failed.pop( se )
if failed:
return S_ERROR( 'Could not get metadata for %d files' % \
len( set( [lfn for lfnList in failed.values() for lfn in lfnList] ) ) )
offlineLFNs = set( lfnList ) - onlineLFNs
for offlineLFN in offlineLFNs:
ses = lfnListReplicas['Value']['Successful'][offlineLFN].keys()
random.shuffle( ses )
se = ses[0]
offlineLFNsDict.setdefault( se, list() ).append( offlineLFN )
return S_OK( {'onlineLFNs':list( onlineLFNs ), 'offlineLFNs': offlineLFNsDict} )
def __init__(self):
"""initialize the member of class"""
self.jobReader_ = JobReader()
self.replicaReader_ = ReplicaReader()
self.bkClient_ = BookkeepingDatabaseClient()
self.dm_ = DataManager()
self.fileTypeCache = {}
def _getClients( self ):
""" returns the clients used in the threads
"""
threadTransformationClient = TransformationClient()
threadDataManager = DataManager()
return {'TransformationClient': threadTransformationClient,
'DataManager': threadDataManager}
def __init__(self, soft_category):
""" Constructor
"""
self.SW_SHARED_DIR = 'VO_VO_CTA_IN2P3_FR_SW_DIR'
self.CVMFS_DIR = '/cvmfs/cta.in2p3.fr/software'
self.LFN_ROOT = '/vo.cta.in2p3.fr/software'
self.SOFT_CATEGORY_DICT = soft_category
self.dm = DataManager()
def __call__(self):
""" call me maybe """
# # counter for failed files
failedFiles = 0
# # catalog to use
catalog = self.operation.Catalog
dm = DataManager(catalogs=catalog)
# # get waiting files
waitingFiles = self.getWaitingFilesList()
# # loop over files
for opFile in waitingFiles:
gMonitor.addMark("RegisterAtt", 1)
# # get LFN
lfn = opFile.LFN
# # and others
fileTuple = (lfn, opFile.PFN, opFile.Size,
self.operation.targetSEList[0], opFile.GUID,
opFile.Checksum)
# # call DataManager
registerFile = dm.registerFile(fileTuple)
# # check results
if not registerFile["OK"] or lfn in registerFile["Value"]["Failed"]:
gMonitor.addMark("RegisterFail", 1)
self.dataLoggingClient().addFileRecord(lfn, "RegisterFail",
catalog, "",
"RegisterFile")
reason = registerFile.get(
"Message",
registerFile.get("Value", {}).get("Failed",
{}).get(lfn, 'Unknown'))
errorStr = "failed to register LFN %s: %s" % (lfn, reason)
opFile.Error = errorStr
self.log.warn(errorStr)
failedFiles += 1
else:
gMonitor.addMark("RegisterOK", 1)
self.dataLoggingClient().addFileRecord(lfn, "Register",
catalog, "",
"RegisterFile")
self.log.info("file %s has been registered at %s" %
(lfn, catalog))
opFile.Status = "Done"
# # final check
if failedFiles:
self.log.info("all files processed, %s files failed to register" %
failedFiles)
self.operation.Error = "some files failed to register"
return S_ERROR(self.operation.Error)
return S_OK()
def main():
global overwrite
Script.registerSwitch("f", "force", "Force overwrite of existing file",
setOverwrite)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 1 or len(args) > 4:
Script.showHelp(exitCode=1)
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
from DIRAC import gLogger
import DIRAC
exitCode = 0
lfns = []
if len(args) == 1:
inputFileName = args[0]
if os.path.exists(inputFileName):
inputFile = open(inputFileName, "r")
for line in inputFile:
line = line.rstrip()
items = line.split()
items[0] = items[0].replace("LFN:", "").replace("lfn:", "")
lfns.append(getDict(items))
inputFile.close()
else:
gLogger.error("Error: LFN list '%s' missing." % inputFileName)
exitCode = 4
else:
lfns.append(getDict(args))
dm = DataManager()
for lfn in lfns:
if not os.path.exists(lfn["localfile"]):
gLogger.error("File %s must exist locally" % lfn["localfile"])
exitCode = 1
continue
if not os.path.isfile(lfn["localfile"]):
gLogger.error("%s is not a file" % lfn["localfile"])
exitCode = 2
continue
gLogger.notice("\nUploading %s" % lfn["lfn"])
res = dm.putAndRegister(lfn["lfn"],
lfn["localfile"],
lfn["SE"],
lfn["guid"],
overwrite=overwrite)
if not res["OK"]:
exitCode = 3
gLogger.error("Error: failed to upload %s to %s: %s" %
(lfn["lfn"], lfn["SE"], res))
continue
else:
gLogger.notice("Successfully uploaded file to %s" % lfn["SE"])
DIRAC.exit(exitCode)
def __init__(self, soft_category={"corsika_simhessarray": "simulations"}):
""" Constructor
"""
self.SW_SHARED_DIR = 'VO_VO_CTA_IN2P3_FR_SW_DIR'
self.CVMFS_DIR = '/cvmfs/cta.in2p3.fr/software'
self.LFN_ROOT = '/vo.cta.in2p3.fr/software'
# self.SOFT_CATEGORY_DICT = {"corsika_simhessarray":"simulations"}
self.SOFT_CATEGORY_DICT = soft_category
self.dm = DataManager()
def __init__(self,
interactive=True,
transClient=None,
dm=None,
fc=None,
dic=None):
""" c'tor
interactive: Data Manager (True) or DIRAC Agente (False)
transClient: TransformationClient() if None, else transClient params
dm: DataManager() if None, else dm params
fc: FileCatalog() if None, else fc params
One object for every production/directoriesList...
"""
self.interactive = interactive
self.transClient = TransformationClient(
) if transClient is None else transClient
self.dm = dm if dm else DataManager()
self.fc = fc if fc else FileCatalog()
self.dic = dic if dic else DataIntegrityClient()
self.dirac = Dirac()
# Base elements from which to start the consistency checks
self._prod = 0
self._bkQuery = None
self._fileType = []
self._fileTypesExcluded = []
self._lfns = []
self.noLFC = False
self.directories = []
# Accessory elements
self.runsList = []
self.runStatus = None
self.fromProd = None
self.transType = ''
self.cachedReplicas = {}
self.prcdWithDesc = []
self.prcdWithoutDesc = []
self.prcdWithMultDesc = []
self.nonPrcdWithDesc = []
self.nonPrcdWithoutDesc = []
self.nonPrcdWithMultDesc = []
self.descForPrcdLFNs = []
self.descForNonPrcdLFNs = []
self.removedFiles = []
self.absentLFNsInFC = []
self.existLFNsNoSE = {}
self.existLFNsBadReplicas = {}
self.existLFNsBadFiles = {}
self.existLFNsNotExisting = {}
self.commonAncestors = {}
self.multipleDescendants = {}
self.ancestors = {}
self._verbose = False
def web_getSelectedFiles(self):
self.set_header('Content-type', 'text/plain')
arguments = self.request.arguments
gLogger.always("submit: incoming arguments %s to getSelectedFiles" %
arguments)
tmpdir = '/tmp/eiscat/' + str(time.time()) + str(random.random())
dataMgr = DataManager(vo=self.vo)
lfnStr = str(arguments['path'][0])
if not os.path.isdir(tmpdir): os.makedirs(tmpdir)
os.chdir(tmpdir)
for lfn in lfnStr.split(','):
gLogger.always("Data manager get file %s" % lfn)
last_slash = lfn.rfind("/")
pos_relative = lfn.find("/")
pos_relative = lfn.find("/", pos_relative + 1)
pos_relative = lfn.find("/", pos_relative + 1)
pos_relative = pos_relative
pathInZip = lfn[pos_relative:last_slash]
tmpPathInZip = tmpdir + pathInZip
gLogger.always("path in zip %s" % tmpPathInZip)
if not os.path.isdir(tmpPathInZip): os.makedirs(tmpPathInZip)
result = dataMgr.getFile(str(lfn),
destinationDir=str(tmpPathInZip))
if not result["OK"]:
gLogger.error("getSelectedFiles: %s" % result["Message"])
#make zip file
zipname = tmpdir.split('/')[-1] + '.zip'
zf = zipfile.ZipFile('/tmp/eiscat/' + zipname, "w")
gLogger.always("zip file /tmp/eiscat/%s" % zipname)
gLogger.always("start walk in tmpdir %s" % tmpdir)
for absolutePath, dirs, files in os.walk(tmpdir):
gLogger.always("absolute path %s" % absolutePath)
gLogger.always("files %s" % files)
for filename in files:
# relative path form tmpdir current chdir
pos_relative = absolutePath.find("/")
pos_relative = absolutePath.find("/", pos_relative + 1)
pos_relative = absolutePath.find("/", pos_relative + 1)
pos_relative = absolutePath.find("/", pos_relative + 1)
pos_relative = pos_relative + 1
relativePath = absolutePath[pos_relative:]
gLogger.always("relativePath %s, file %s" %
(relativePath, filename))
zf.write(os.path.join(relativePath, filename))
zf.close()
#read zip file
f = open('/tmp/eiscat/' + zipname, "rb")
obj = f.read()
f.close()
#cleanup
shutil.rmtree(tmpdir)
os.remove('/tmp/eiscat/' + zipname)
self.set_header('Content-Disposition',
'attachment; filename="' + zipname)
self.write(obj)
def __call__( self ):
""" call me maybe """
# # counter for failed files
failedFiles = 0
# # catalog(s) to use
catalogs = self.operation.Catalog
if catalogs:
catalogs = [ cat.strip() for cat in catalogs.split( ',' ) ]
dm = DataManager( catalogs = catalogs )
# # get waiting files
waitingFiles = self.getWaitingFilesList()
# # loop over files
for opFile in waitingFiles:
gMonitor.addMark( "RegisterAtt", 1 )
# # get LFN
lfn = opFile.LFN
# # and others
fileTuple = ( lfn , opFile.PFN, opFile.Size, self.operation.targetSEList[0], opFile.GUID, opFile.Checksum )
# # call DataManager
registerFile = dm.registerFile( fileTuple )
# # check results
if not registerFile["OK"] or lfn in registerFile["Value"]["Failed"]:
gMonitor.addMark( "RegisterFail", 1 )
# self.dataLoggingClient().addFileRecord( lfn, "RegisterFail", ','.join( catalogs ) if catalogs else "all catalogs", "", "RegisterFile" )
reason = str( registerFile.get( "Message", registerFile.get( "Value", {} ).get( "Failed", {} ).get( lfn, 'Unknown' ) ) )
errorStr = "failed to register LFN"
opFile.Error = "%s: %s" % ( errorStr, reason )
if 'GUID already registered' in reason:
opFile.Status = 'Failed'
self.log.error( errorStr, "%s: %s" % ( lfn, reason ) )
elif 'File already registered with no replicas' in reason:
self.log.warn( errorStr, "%s: %s, will remove it and retry" % ( lfn, reason ) )
dm.removeFile( lfn )
else:
self.log.warn( errorStr, "%s: %s" % ( lfn, reason ) )
failedFiles += 1
else:
gMonitor.addMark( "RegisterOK", 1 )
# self.dataLoggingClient().addFileRecord( lfn, "Register", ','.join( catalogs ) if catalogs else "all catalogs", "", "RegisterFile" )
self.log.verbose( "file %s has been registered at %s" % ( lfn, ','.join( catalogs ) if catalogs else "all catalogs" ) )
opFile.Status = "Done"
# # final check
if failedFiles:
self.log.warn( "all files processed, %s files failed to register" % failedFiles )
self.operation.Error = "some files failed to register"
return S_ERROR( self.operation.Error )
return S_OK()
def initialize(self):
"""
specify the tests which need to be executed.
"""
self.apis['DataManager'] = DataManager()
self.apis[
'ResourceManagementIHEPClient'] = ResourceManagementIHEPClient()
return S_OK()
def setUp( self ):
super( FailingUserJobTestCase, self ).setUp()
dm = DataManager()
res = dm.removeFile( ['/lhcb/testCfg/testVer/LOG/00012345/0006/00012345_00067890.tar',
'/lhcb/testCfg/testVer/SIM/00012345/0006/00012345_00067890_1.sim'],
force = True )
if not res['OK']:
print("Could not remove files", res['Message'])
exit( 1 )
def __getDataManager(self, vo):
if vo in self.__dataManDict:
return self.__dataManDict[vo]
try:
self.__dataManDict[vo] = DataManager(vo=vo)
except Exception:
msg = "Failed to create DataManager"
self.log.exception(msg)
return None
return self.__dataManDict[vo]
def doesFileExist(lfn):
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
dm = DataManager()
result = dm.getActiveReplicas(lfn)
if not result['OK']:
print "ERROR", result['Message']
return False
if lfn in result['Value']['Successful']:
return True
else:
return False
