def getSoftwareEnviron(package, environ=None):
"""
Check shared and local Area for the given package and return
the new environ dictionary
"""
if environ == None:
environ = dict(os.environ)
elif type(environ) == DictType:
environ = dict(environ)
else:
return DIRAC.S_ERROR('environ argument must be a dictionary')
gLogger.notice('Getting environment for', package)
for area in [workingArea(), sharedArea(), localArea()]:
if area:
if not checkSoftwarePackage(package, area)['OK']:
continue
fileName = _getEnvFileName(package, area)
base = os.path.splitext(fileName)[0]
gLogger.info('Sourcing file to get environment:', fileName)
result = DIRAC.sourceEnv(TIMEOUT, [base], environ)
if not result['OK']:
return result
if result['stdout']:
gLogger.info(result['stdout'])
return DIRAC.S_OK(result['outputEnv'])
# If we get here it means the package has not been found
return DIRAC.S_ERROR('Failed to get Environment')
def getMatchingFiles(args):
""" simple wrapper to download data files matching a pattern
Keyword arguments:
args -- a list of arguments in order []
"""
# check command line
res=None
if len(args)!=1:
res=DIRAC.S_ERROR()
res['Message'] = 'just give the function you wish to use: sub5, sub2sub5, HB9SCT'
return res
# now do something
funcName = args[0]
# What shall we verify ?
if funcName == "sub5":
res=getSub5()
elif funcName == "sub2sub5":
res=getSub2Sub5()
elif funcName == "HB9SCT":
res=getHB9SCT()
else:
res=DIRAC.S_ERROR()
res['Message'] = 'Function "%s" not known'% funcName
return res
def verifyMerging(nbFiles=10, minSize=5000.):
""" Verify a PROD3 simtel merging step
Keyword arguments:
nbFiles -- number of output files expected
minSize -- minimum file size
"""
DIRAC.gLogger.info('Verifying Merging step')
# get list of output files
outputFiles=glob.glob('Data/sim_telarray/cta-prod3/0.0deg/Data/*simtel.gz')
# check the number of output files --- could be done by telescope type
N=len(outputFiles)
if N != nbFiles :
DIRAC.gLogger.error( 'Wrong number of Simtel Merged files : %s instead of %s' % ( N, nbFiles ) )
cleanFiles( outputFiles )
return DIRAC.S_ERROR()
# check the file size --- could be done by telescope type
for afile in outputFiles:
sizekb=os.path.getsize(afile)/1024.
if sizekb < minSize:
DIRAC.gLogger.error( '%s\n File size too small : %s < %s kb' % ( afile, sizekb, minSize ) )
cleanFiles( outputFiles )
return DIRAC.S_ERROR()
DIRAC.gLogger.info( 'Good files found:\n%s'%'\n'.join(outputFiles) )
return DIRAC.S_OK()
def execute(self):
"""
The method called by the Workflow framework
"""
from DIRAC.Core.Utilities.Subprocess import systemCall
from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron
ret = self.__checkInputs()
if not ret['OK']:
return ret
ret = getSoftwareEnviron(self.softwarePackage)
if not ret['OK']:
error = ret['Message']
self.log.error(error, self.softwarePackage)
return DIRAC.S_ERROR(' '.join([error, str(self.softwarePackage)]))
envdispEnviron = ret['Value']
cmdTuple = [self.edExe]
cmdTuple.extend(self.edArguments)
self.log.notice('Executing command tuple:', cmdTuple)
ret = systemCall(0, cmdTuple, self.sendOutput, env=envdispEnviron)
if not ret['OK']:
self.log.error('Failed to execute evndisp:', ret['Message'])
return DIRAC.S_ERROR('Can not execute evndisp')
status, stdout, stderr = ret['Value']
self.log.notice('evndisp status is:', status)
return status
def verify_corsika():
""" Verify a generic Corsika log file
"""
DIRAC.gLogger.info('Verifying Corsika log file')
# get list of output files
log_file = glob.glob('Data/corsika/run*/run*.log')
if len(log_file) != 1:
DIRAC.gLogger.error('no log file found!')
return DIRAC.S_ERROR()
# check EOR tag
tag = '=== END OF RUN ==='
content = open(log_file[0]).read()
if content.find(tag) < 0:
DIRAC.gLogger.error('"%s" tag not found!' % tag)
corsika_files = list(os.popen('find . -iname "*corsika.*z*"'))
if len(corsika_files) > 0:
clean_output_file(corsika_files)
simtel_files = list(os.popen('find . -iname "*simtel.*z*"'))
if len(simtel_files) > 0:
clean_output_file(simtel_files)
log_gz_files = list(os.popen('find . -iname "*log.*z*"'))
if len(log_gz_files) > 0:
clean_output_file(log_gz_files)
log_hist_files = list(os.popen('find . -iname "*log_hist.tar"'))
if len(log_hist_files) > 0:
clean_output_file(log_hist_files)
return DIRAC.S_ERROR()
return DIRAC.S_OK()
def __addInstanceHistory(self,
instanceID,
status,
load=0.0,
jobs=0,
transferredFiles=0,
transferredBytes=0):
"""
Insert a History Record
"""
try:
load = float(load)
except:
return DIRAC.S_ERROR("Load has to be a float value")
try:
jobs = int(jobs)
except:
return DIRAC.S_ERROR("Jobs has to be an integer value")
try:
transferredFiles = int(transferredFiles)
except:
return DIRAC.S_ERROR(
"Transferred files has to be an integer value")
self._insert('vm_History', [
'VMInstanceID', 'Status', 'Load', 'Update', 'Jobs',
'TransferredFiles', 'TransferredBytes'
], [
instanceID, status, load,
DIRAC.Time.toString(), jobs, transferredFiles, transferredBytes
])
return
def __getStatus(self, object, id):
"""
Check and return status of Images and Instances by ID
returns:
S_OK(Status) if Status is valid and not Error
S_ERROR(ErrorMessage) otherwise
"""
(tableName, validStates, idName) = self.__getTypeTuple(object)
if not tableName:
return DIRAC.S_ERROR('Unknown DB object: %s' % object)
ret = self._getFields(tableName, ['Status', 'ErrorMessage'], [idName],
[id])
if not ret['OK']:
return ret
if not ret['Value']:
return DIRAC.S_ERROR('Unknown %s = %s' % (idName, id))
(status, msg) = ret['Value'][0]
if not status in validStates:
return self.__setError(object, id, 'Invalid Status: %s' % status)
if status == validStates[-1]:
return DIRAC.S_ERROR(msg)
return DIRAC.S_OK(status)
def __getSubmittedInstanceID(self, imageName):
"""
Retrieve and InstanceID associated to a submitted Instance for a given Image
"""
(tableName, validStates, idName) = self.__getTypeTuple('Image')
imageID = self._getFields(tableName, [idName], ['Name'], [imageName])
if not imageID['OK']:
return imageID
if not imageID['Value']:
return DIRAC.S_ERROR('Unknown Image = %s' % imageName)
if len(imageID['Value']) <> 1:
return DIRAC.S_ERROR('Image name "%s" is not unique' % imageName)
imageID = imageID['Value'][0][0]
imageIDName = idName
(tableName, validStates, idName) = self.__getTypeTuple('Instance')
instanceID = self._getFields(tableName, [idName],
[imageIDName, 'Status'],
[imageID, 'Submitted'])
if not instanceID['OK']:
return instanceID
if not instanceID['Value']:
return DIRAC.S_ERROR('No Submitted instance of "%s" found' %
imageName)
return DIRAC.S_OK(instanceID['Value'][0][0])
def getImageDict(imageName):
"""
Return from CS a Dictionary with Image definition
"""
imagesCSPath = '/Resources/VirtualMachines/Images'
definedImages = DIRAC.gConfig.getSections(imagesCSPath)
if not definedImages['OK']:
return definedImages
if imageName not in definedImages['Value']:
return DIRAC.S_ERROR('Image "%s" not defined' % imageName)
imageCSPath = '%s/%s' % (imagesCSPath, imageName)
imageDict = {}
flavor = DIRAC.gConfig.getValue('%s/Flavor' % imageCSPath, '')
if not flavor:
return DIRAC.S_ERROR('Missing Flavor for image "%s"' % imageName)
for option, value in DIRAC.gConfig.getOptionsDict(
imageCSPath)['Value'].items():
imageDict[option] = value
imageRequirementsDict = DIRAC.gConfig.getOptionsDict('%s/Requirements' %
imageCSPath)
if not imageRequirementsDict['OK']:
return DIRAC.S_ERROR('Missing Requirements for image "%s"' % imageName)
if 'CPUTime' in imageRequirementsDict['Value']:
imageRequirementsDict['Value']['CPUTime'] = int(
imageRequirementsDict['Value']['CPUTime'])
imageDict['Requirements'] = imageRequirementsDict['Value']
return DIRAC.S_OK(imageDict)
def verify(args):
""" simple wrapper to put and register all PROD3 files
Keyword arguments:
args -- a list of arguments in order []
"""
# check command line
if len(args) < 3 :
res=DIRAC.S_ERROR()
res['Message'] = 'verify requires at least nbFiles and fileSize as arguments'
return res
# now do something
stepType = args[0]
nbFiles = int(args[1])
fileSize = float(args[2])
if len(args)==4 :
path = args[3]
# What shall we verify ?
if stepType == "corsika":
res=verifyCorsika(nbFiles, fileSize)
elif stepType == "simtel":
res=verifySimtel(nbFiles, fileSize)
elif stepType == "merging":
res=verifyMerging(nbFiles, fileSize)
elif stepType == "analysisinputs":
res = verifyAnalysisInputs( fileSize )
elif stepType == "generic":
res = verifyGeneric( nbFiles, fileSize, path )
else:
res=DIRAC.S_ERROR()
res['Message'] = 'Do not know how to verify "%s"'% stepType
return res
def verify_corsika():
""" Verify a generic Corsika log file
"""
DIRAC.gLogger.info('Verifying Corsika log file')
# get list of output files
log_file = glob.glob('Data/corsika/run*/run*.log')
if len(log_file) != 1:
DIRAC.gLogger.error('"=== END OF RUN ===" not found!')
return DIRAC.S_ERROR()
# check EOR tag
tag = '=== END OF RUN ==='
content = open(log_file[0]).read()
if content.find(tag) < 0:
DIRAC.gLogger.error('"%s" tag not found!' % tag)
corsika_files = glob.glob('Data/corsika/run*/*corsika.*z*')
if len(corsika_files) > 0:
clean_output_file(corsika_files)
simtel_files = glob.glob('Data/sim_telarray/*/*/Data/*simtel.*z*')
if len(simtel_files) > 0:
clean_output_file(simtel_files)
return DIRAC.S_ERROR()
return DIRAC.S_OK()
def testMultiThreading(tries):
import random
DIRAC.gLogger.info('Testing MySQL MultiThreading')
DIRAC.gLogger.info('First adding 10 K records')
if not DB.checktable()['OK']:
return DIRAC.S_ERROR()
if not DB.filltable(10000)['OK']:
return DIRAC.S_ERROR()
i = 0
# overthread = 0
DIRAC.gLogger.info('Now querying 100 K in MultiThread mode')
while i < tries:
if not i % 1000:
DIRAC.gLogger.info('Query:', i)
overthread = 0
i += 1
id = int(random.uniform(0, 10000)) + 1
t = threading.Thread(target=testRetrieve, args=(id, ))
semaphore.acquire()
t.start()
n = threading.activeCount()
while n > 1:
DIRAC.gLogger.info('Waiting for Treads to end:', n)
n = threading.activeCount()
time.sleep(0.1)
DIRAC.gLogger.info('Total retrieved values', Success)
DIRAC.gLogger.info('Total Errors', Error)
return DIRAC.S_OK((Success, Error))
def verifyCorsika( nbFiles = 6, minSize = 50. ):
""" Verify a PROD3 corsika step
Keyword arguments:
nbFiles -- number of output files expected
minSize -- minimum file size
"""
DIRAC.gLogger.info('Verifying Corsika step')
# get list of output files
outputFiles=glob.glob('Data/corsika/run*/*corsika.gz')
# check the number of output files
N=len(outputFiles)
if N != nbFiles :
DIRAC.gLogger.error( 'Wrong number of Corsika files : %s instead of %s' % ( N, nbFiles ) )
cleanFiles( outputFiles )
return DIRAC.S_ERROR()
# check the file size
for afile in outputFiles:
sizekb=os.path.getsize(afile)/1024.
if sizekb < minSize:
DIRAC.gLogger.error( '%s\n File size too small : %s < %s kb' % ( afile, sizekb, minSize ) )
cleanFiles( outputFiles )
return DIRAC.S_ERROR()
DIRAC.gLogger.info( 'Good files found:\n%s'%'\n'.join(outputFiles) )
return DIRAC.S_OK()
def verifySimtel(nbFiles=31, minSize=50.):
""" Verify a PROD3 simtel step
Keyword arguments:
nbFiles -- number of output files expected
minSize -- minimum file size
"""
DIRAC.gLogger.info('Verifying Simtel step')
# get list of output files
outputFiles = glob.glob('Data/simtel_tmp/Data/*simtel.gz')
# check the number of output files --- could be done by telescope type
N = len(outputFiles)
if N != nbFiles:
DIRAC.gLogger.error('Wrong number of Simtel files : %s instead of %s' %
(N, nbFiles))
cleanFiles(outputFiles)
return DIRAC.S_ERROR()
# check the file size --- could be done by telescope type
for afile in outputFiles:
sizekb = os.path.getsize(afile) / 1024.
if sizekb < minSize:
DIRAC.gLogger.error('%s\n File size too small : %s < %s kb' %
(afile, sizekb, minSize))
cleanFiles(outputFiles)
return DIRAC.S_ERROR()
return DIRAC.S_OK()
def createDstFileSystAndMD():
# Creating INDEXES in DFC DB
dstDirMDFields = {}
dstDirMDFields['dstProdVersion'] = 'VARCHAR(128)'
createIndexes(dstDirMDFields)
# Adding Directory level metadata Values to DFC
dstDirMD = {}
dstDirMD['dstProdVersion'] = dstProdVersion
res = createDirAndInsertMD(dstDirPath, dstDirMD)
if res != DIRAC.S_OK:
return DIRAC.S_ERROR('MD Error: Problem creating Dst Directory MD ')
dstDataDirPath = os.path.join(dstDirPath, 'Data')
dstDataDirMD = {}
dstDataDirMD['outputType'] = 'dstData'
res = createDirAndInsertMD(dstDataDirPath, dstDataDirMD)
if res != DIRAC.S_OK:
return DIRAC.S_ERROR('Problem creating Dst Data Directory MD ')
dstHistoDirPath = os.path.join(dstDirPath, 'Histograms')
dstHistoDirMD = {}
dstHistoDirMD['outputType'] = 'dstHisto'
res = createDirAndInsertMD(dstHistoDirPath, dstHistoDirMD)
if res != DIRAC.S_OK:
return DIRAC.S_ERROR('Problem creating Dst Histo Directory MD ')
return DIRAC.S_OK('Dst Directory MD successfully created')
def verifyGeneric(nbFiles=1, minSize=50., path='Data/*'):
""" Verify a PROD3 generic step
Keyword arguments:
nbFiles -- number of output files expected
minSize -- minimum file size
"""
DIRAC.gLogger.info('Verifying generic step output')
# get list of output files
outputFiles = glob.glob(path)
# check the number of output files
N = len(outputFiles)
if N != nbFiles:
DIRAC.gLogger.error('Wrong number of output files : %s instead of %s' %
(N, nbFiles))
clean_output_file(outputFiles)
return DIRAC.S_ERROR()
# check the file size
for afile in outputFiles:
sizekb = os.path.getsize(afile) / 1024.
if sizekb < minSize:
DIRAC.gLogger.error('%s\n File size too small : %s < %s kb' %
(afile, sizekb, minSize))
clean_output_file(outputFiles)
return DIRAC.S_ERROR()
DIRAC.gLogger.info('Good files found:\n%s' % '\n'.join(outputFiles))
return DIRAC.S_OK()
def createCorsikaFileSystAndMD():
# before creating indexes, it would be fine to know what are those that already exist in the DB
# Creating INDEXES in DFC DB
corsikaDirMDFields = {}
corsikaDirMDFields['prodName'] = 'VARCHAR(128)'
corsikaDirMDFields['thetaP'] = 'float'
corsikaDirMDFields['phiP'] = 'float'
corsikaDirMDFields['altitude'] = 'float'
corsikaDirMDFields['particle'] = 'VARCHAR(128)'
corsikaDirMDFields['energyInfo'] = 'VARCHAR(128)'
corsikaDirMDFields['viewCone'] = 'float'
corsikaDirMDFields['corsikaProdVersion'] = 'VARCHAR(128)'
corsikaDirMDFields['nbShowers'] = 'int'
corsikaDirMDFields['outputType'] = 'VARCHAR(128)'
corsikaDirMDFields['runNumSeries'] = 'int'
createIndexes(corsikaDirMDFields)
# Adding Directory level metadata Values to DFC
corsikaDirMD = {}
corsikaDirMD['thetaP'] = thetaP
corsikaDirMD['phiP'] = phiP
corsikaDirMD['altitude'] = obslev
corsikaDirMD['energyInfo'] = energyInfo
corsikaDirMD['corsikaProdVersion'] = corsikaProdVersion
res = createDirAndInsertMD(corsikaDirPath, corsikaDirMD)
if res != DIRAC.S_OK:
return DIRAC.S_ERROR('Problem creating Corsika Directory MD ')
corsikaParticleDirMD = {}
corsikaParticleDirMD['particle'] = particle
corsikaParticleDirMD['viewCone'] = viewCone
res = createDirAndInsertMD(corsikaParticleDirPath, corsikaParticleDirMD)
if res != DIRAC.S_OK:
return DIRAC.S_ERROR('Problem creating Corsika Particle Directory MD ')
corsikaParticleDataDirPath = os.path.join(corsikaParticleDirPath, 'Data')
corsikaParticleDataDirMD = {}
corsikaParticleDataDirMD['outputType'] = 'corsikaData'
res = createDirAndInsertMD(corsikaParticleDataDirPath,
corsikaParticleDataDirMD)
if res != DIRAC.S_OK:
return DIRAC.S_ERROR(
'Problem creating Corsika Particle Data Directory MD ')
corsikaParticleLogDirPath = os.path.join(corsikaParticleDirPath, 'Log')
corsikaParticleLogDirMD = {}
corsikaParticleLogDirMD['outputType'] = 'corsikaLog'
res = createDirAndInsertMD(corsikaParticleLogDirPath,
corsikaParticleLogDirMD)
if res != DIRAC.S_OK:
return DIRAC.S_ERROR(
'Problem creating Corsika Particle Log Directory MD ')
return DIRAC.S_OK('Corsika Directory MD successfully created')
def execute(self):
"""
The method called by the Workflow framework
"""
from DIRAC.Core.Utilities.Subprocess import systemCall
from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea
ret = self.__checkInputs()
if not ret['OK']:
return ret
ret = getSoftwareEnviron(self.softwarePackage)
if not ret['OK']:
error = ret['Message']
self.log.error(error, self.softwarePackage)
return DIRAC.S_ERROR(' '.join([error, str(self.softwarePackage)]))
hapEnviron = ret['Value']
hessroot = hapEnviron['HESSROOT']
rootlogon_file = hessroot + '/rootlogon.C'
cp_cmd = 'cp ' + rootlogon_file + ' .'
os.system(cp_cmd)
fileName = hessroot + self.rootMacro
if fileName[-1] == '+':
# If the macro has to be compiled there is an extra "+" at the end of its name
fileName = fileName[:-1]
if not os.path.isfile(fileName):
error = 'Hap Root macro file does not exist:'
self.log.error(error, fileName)
return DIRAC.S_ERROR(' '.join([error, fileName]))
fileName = hessroot + self.rootMacro
cmdTuple = ['root', '-b', '-q']
cmdTuple += [
'%s( %s )' %
(fileName, ', '.join(self.rootArguments).replace("'", '"'))
]
self.log.notice('Executing command tuple:', cmdTuple)
ret = systemCall(0, cmdTuple, self.sendOutput, env=hapEnviron)
if not ret['OK']:
self.log.error('Failed to execute Root:', ret['Message'])
return DIRAC.S_ERROR('Can not execute Hap root macro')
return DIRAC.S_OK()
def __setError(self, object, id, reason):
"""
"""
(tableName, validStates, idName) = self.__getTypeTuple(object)
if not tableName:
return DIRAC.S_ERROR('Unknown DB object: %s' % object)
cmd = 'UPDATE `%s` SET Status = "%s", ErrorMessage = "%s", LastUpdate = UTC_TIMESTAMP() WHERE %s = %s' % (
tableName, validStates[-1], reason, idName, id)
ret = self._update(cmd)
if not ret['OK']:
return ret
return DIRAC.S_ERROR(reason)
def execute( self ):
"""
The method called by the Workflow framework
"""
from DIRAC.Core.Utilities.Subprocess import systemCall
from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron
ret = self.__checkInputs()
if not ret['OK']:
return ret
ret = getSoftwareEnviron( self.softwarePackage )
if not ret['OK']:
error = ret['Message']
self.log.error( error, self.softwarePackage )
return DIRAC.S_ERROR( ' '.join( [ error, str( self.softwarePackage ) ] ) )
rootEnviron = ret['Value']
fileName = self.rootMacro
if fileName[-1] == '+':
# If the macro has to be compiled there is an extra "+" at the end of its name
fileName = fileName[:-1]
if not os.path.isfile( fileName ):
error = 'Root macro file does not exist:'
self.log.error( error, fileName )
return DIRAC.S_ERROR( ' '.join( [ error, fileName ] ) )
cmdTuple = ['root', '-b', '-q']
cmdTuple += ['%s( %s )' % ( self.rootMacro, ', '.join( self.rootArguments ).replace( "'", '"' ) ) ]
self.log.info( 'Executing command tuple:', cmdTuple )
ret = systemCall( 0, cmdTuple, env = rootEnviron )
if not ret['OK']:
self.log.error( 'Failed to execute Root:', ret['Message'] )
return DIRAC.S_ERROR( 'Can not execute root' )
status, stdout, stderr = ret['Value']
if status:
self.log.error( 'Root execution reports Error:', status )
self.log.error( stdout )
self.log.error( stderr )
return DIRAC.S_ERROR( 'Failed root Execution' )
self.log.info( 'Root stdout:' )
self.log.info( stdout )
return DIRAC.S_OK()
def putAndRegister(self,
lfn,
localfile,
filemetadata,
DataType='SimtelProd'):
""" put and register one file and set metadata
"""
# ## Get the list of Production SE
res = self._getSEList('ProductionOutputs', DataType)
if res['OK']:
ProductionSEList = res['Value']
else:
return res
# ## Get the list of Failover SE
res = self._getSEList('ProductionOutputsFailover', DataType)
if res['OK']:
FailoverSEList = res['Value']
else:
return res
# ## Upload file to a Production SE
res = self._putAndRegisterToSEList(lfn, localfile, ProductionSEList)
if not res['OK']:
DIRAC.gLogger.error(
'Failed to upload file to any Production SE: %s' %
ProductionSEList)
# ## Upload file to a Failover SE
res = self._putAndRegisterToSEList(lfn, localfile, FailoverSEList)
if not res['OK']:
return DIRAC.S_ERROR(
'Failed to upload file to any Failover SE: %s' %
FailoverSEList)
# ## Set file metadata: jobID, subarray, sct
if res['OK']:
fmd = json.loads(filemetadata)
if os.environ.has_key('JOBID'):
fmd.update({'jobID': os.environ['JOBID']})
filename = os.path.basename(localfile)
# set subarray and sct md from filename
p = re.compile('subarray-\d+')
if p.search(filename) != None:
subarray = p.search(filename).group()
fmd.update({'subarray': subarray})
sct = 'False'
p = re.compile('nosct')
psct = re.compile('sct')
if p.search(filename) == None and psct.search(filename) != None:
sct = 'True'
# ## Set sct flag only for production data
res = self.fcc.getFileUserMetadata(lfn)
if DataType == 'SimtelProd' and res['Value'][
'outputType'] == 'Data':
fmd.update({'sct': sct})
res = self.fc.setMetadata(lfn, fmd)
if not res['OK']:
return res
return DIRAC.S_OK()
def install_software(self, tar_lfn, target_dir='.'):
""" install software package in the current directory
"""
DIRAC.gLogger.notice('Installing package at %s'%tar_lfn)
# Download the tar file
DIRAC.gLogger.notice('Trying to download package:', tar_lfn)
res = self.dm.getFile(tar_lfn)
if not res['OK']:
return res
if tar_lfn in res['Value']['Successful']:
DIRAC.gLogger.notice(' Package downloaded successfully:', tar_lfn)
else:
error = 'Failed to download package:', tar_lfn
return DIRAC.S_ERROR(error)
# Extract the tar file to the target directory
tar_mode = "r|*"
tar = tarfile.open(tar_lfn, tar_mode)
for tarInfo in tar:
tar.extract(tarInfo, target_dir)
tar.close()
os.unlink(tar_lfn)
# Done
DIRAC.gLogger.notice('Package %s installed successfully at:\n%s'
%(tar_lfn, target_dir))
return DIRAC.S_OK(target_dir)
def _search_software(self, package, version, compiler, use_cvmfs):
''' Look for sotfware package
'''
# software package category
category = self.SOFT_CATEGORY_DICT[package]
# look for software on cvmfs
if use_cvmfs:
package_dir = os.path.join(self.CVMFS_DIR, 'centos7',
compiler, category, package, version)
if os.path.isdir(package_dir):
DIRAC.gLogger.notice('Found package %s version %s at:\n%s' %
(package, version, package_dir))
return DIRAC.S_OK({'Source':'cvmfs', 'Path':package_dir})
else:
DIRAC.gLogger.warn('%s\n not found on cvmfs'%package_dir)
# look for tarball in the Dirac file catalog
else:
package_dir = os.path.join(self.LFN_ROOT, 'centos7',
compiler, category, package, version)
DIRAC.gLogger.notice('Looking for tarball in %s'%package_dir)
results = self.dm.getFilesFromDirectory(package_dir)
try:
first_file_path = results['Value'][0]
if first_file_path[-7:] == '.tar.gz':
results = self.dm.getActiveReplicas(first_file_path)
if results['OK']:
return DIRAC.S_OK({'Source':'tarball', 'Path':package_dir})
except:
DIRAC.gLogger.warn('No usual tarball found in the directory')
return DIRAC.S_ERROR('Could not find package %s / %s / %s in any location'
% (package, version, compiler))
def checkSoftwarePackage(self,
package,
version,
arch="sl6-gcc44",
area=None):
""" check if the software package is installed in any software area
Keyword arguments:
package -- package name as the directory name
version -- software version as the directory name
arch -- architecture as the directory name
"""
# if area is specified, just get the shared area
if area:
areaList = self._getSharedArea()
else:
areaList = self._getSoftwareAreas()
if len(areaList) == 0:
DIRAC.gLogger.warn('No software area is available')
# ## look for the package directory in the software areas
for area in areaList:
package_dir = os.path.join(area, arch,
self.SOFT_CATEGORY_DICT[package],
package, version)
if os.path.isdir(package_dir):
DIRAC.gLogger.notice('Found package %s version %s at:\n%s' %
(package, version, area))
return DIRAC.S_OK(package_dir)
return DIRAC.S_ERROR(
'Could not find package %s version %s in any location' %
(package, version))
def _getSEList( self, SEType = 'ProductionOutputs', DataType = 'SimtelProd' ):
""" get from CS the list of available SE for data upload
"""
opsHelper = Operations()
optionName = os.path.join( SEType, DataType )
SEList = opsHelper.getValue( optionName , [] )
SEList = List.randomize( SEList )
DIRAC.gLogger.notice( 'List of %s SE: %s ' % ( SEType, SEList ) )
# # Check if the local SE is in the list. If yes try it first by reversing list order
localSEList = []
res = getSEsForSite( DIRAC.siteName() )
if res['OK']:
localSEList = res['Value']
retainedlocalSEList = []
for localSE in localSEList:
if localSE in SEList:
DIRAC.gLogger.notice( 'The local Storage Element is an available SE: ', localSE )
retainedlocalSEList.append( localSE )
SEList.remove( localSE )
SEList = retainedlocalSEList + SEList
if len( SEList ) == 0:
return DIRAC.S_ERROR( 'Error in building SEList' )
return DIRAC.S_OK( SEList )
def _putAndRegisterToSEList( self, lfn, localfile, SEList ):
""" put and register one file to one SE in the SEList
"""
# ## Try to upload file to a SE in the list
for SE in SEList:
msg = 'Try to upload local file: %s \nwith LFN: %s \nto %s' % ( localfile, lfn, SE )
DIRAC.gLogger.notice( msg )
res = self.dm.putAndRegister( lfn, localfile, SE )
DIRAC.gLogger.notice(res)
# ## check if failed
if not res['OK']:
DIRAC.gLogger.error( 'Failed to putAndRegister %s \nto %s \nwith message: %s' % ( lfn, SE, res['Message'] ) )
DIRAC.gLogger.notice('Trying to clean up %s' % lfn)
res = self.dm.removeFile(lfn)
if res['OK']:
DIRAC.gLogger.notice('Successfully removed %s \n that was not supposed to have been uploaded successfully' % lfn)
continue
elif res['Value']['Failed'].has_key( lfn ):
DIRAC.gLogger.error( 'Failed to putAndRegister %s to %s' % ( lfn, SE ) )
DIRAC.gLogger.notice('Trying to clean up %s' % lfn)
res = self.dm.removeFile(lfn)
if res['OK']:
DIRAC.gLogger.notice('Successfully removed %s \n that was not supposed to have been uploaded successfully' % lfn)
continue
else:
return DIRAC.S_OK()
return DIRAC.S_ERROR()
def __insertInstance(self, imageName, instanceName):
"""
Attempts to insert a new Instance for the given Image
"""
image = self.__getImageID(imageName)
if not image['OK']:
return image
imageID = image['Value']
(tableName, validStates, idName) = self.__getTypeTuple('Instance')
fields = ['Name', 'VMImageID', 'Status', 'LastUpdate']
values = [instanceName, imageID, validStates[0], DIRAC.Time.toString()]
result = getImageDict(imageName)
if not result['OK']:
return result
imgDict = result['Value']
if 'MaxAllowedPrice' in imgDict:
fields.append('MaxAllowedPrice')
values.append(imgDict['MaxAllowedPrice'])
instance = self._insert(tableName, fields, values)
if instance['OK'] and 'lastRowId' in instance:
self.__addInstanceHistory(instance['lastRowId'], validStates[0])
return DIRAC.S_OK(instance['lastRowId'])
if not instance['OK']:
return instance
return DIRAC.S_ERROR('Failed to insert new Instance')
def setProxyStrength(self, arg):
try:
self.proxyStrength = int(arg)
except:
print "Can't parse %s bits! Is it a number?" % arg
return DIRAC.S_ERROR("Can't parse strength argument")
return DIRAC.S_OK()
def execute(self):
"""
Check if requested packages are available in shared Area
otherwise install them locally
"""
# For the moment CTA does not use any kind if "platform" only slc5 64bit is supported ???
if not self.packs:
# There is nothing to do
return DIRAC.S_OK()
if self.sharedArea:
gLogger.notice('Using Shared Area at:', self.sharedArea)
if self.localArea:
gLogger.notice('Using Local Area at:', self.localArea)
for package in self.packs:
gLogger.notice('Checking:', package)
if self.sharedArea:
if checkSoftwarePackage(package, self.sharedArea)['OK']:
gLogger.notice('Package found in Shared Area:', package)
continue
if self.localArea:
if checkSoftwarePackage(package, self.localArea)['OK']:
gLogger.notice('Package found in Local Area:', package)
continue
if installSoftwarePackage(package, self.localArea)['OK']:
continue
gLogger.error('Check Failed for software package:', package)
return DIRAC.S_ERROR('%s not available' % package)
return DIRAC.S_OK()
def getInstancesByStatus(self, status):
"""
Get dictionary of Image Names with InstanceIDs in given status
"""
if status not in self.validInstanceStates:
return DIRAC.S_ERROR('Status %s is not known' % status)
(tableName, validStates, idName) = self.__getTypeTuple('Instance')
runningInstances = self._getFields(tableName,
['VMImageID', 'UniqueID'],
['Status'], [status])
if not runningInstances['OK']:
return runningInstances
runningInstances = runningInstances['Value']
instancesDict = {}
imagesDict = {}
for imageID, uniqueID in runningInstances:
if not imageID in imagesDict:
(tableName, validStates, idName) = self.__getTypeTuple('Image')
imageName = self._getFields(tableName, ['Name'], [idName],
[imageID])
if not imageName['OK']:
continue
imagesDict[imageID] = imageName['Value'][0][0]
if not imagesDict[imageID] in instancesDict:
instancesDict[imagesDict[imageID]] = []
instancesDict[imagesDict[imageID]].append(uniqueID)
return DIRAC.S_OK(instancesDict)
