def getSoftwareEnviron(package, environ=None): """ Check shared and local Area for the given package and return the new environ dictionary """ if environ == None: environ = dict(os.environ) elif type(environ) == DictType: environ = dict(environ) else: return DIRAC.S_ERROR('environ argument must be a dictionary') gLogger.notice('Getting environment for', package) for area in [workingArea(), sharedArea(), localArea()]: if area: if not checkSoftwarePackage(package, area)['OK']: continue fileName = _getEnvFileName(package, area) base = os.path.splitext(fileName)[0] gLogger.info('Sourcing file to get environment:', fileName) result = DIRAC.sourceEnv(TIMEOUT, [base], environ) if not result['OK']: return result if result['stdout']: gLogger.info(result['stdout']) return DIRAC.S_OK(result['outputEnv']) # If we get here it means the package has not been found return DIRAC.S_ERROR('Failed to get Environment')
def getMatchingFiles(args): """ simple wrapper to download data files matching a pattern Keyword arguments: args -- a list of arguments in order [] """ # check command line res=None if len(args)!=1: res=DIRAC.S_ERROR() res['Message'] = 'just give the function you wish to use: sub5, sub2sub5, HB9SCT' return res # now do something funcName = args[0] # What shall we verify ? if funcName == "sub5": res=getSub5() elif funcName == "sub2sub5": res=getSub2Sub5() elif funcName == "HB9SCT": res=getHB9SCT() else: res=DIRAC.S_ERROR() res['Message'] = 'Function "%s" not known'% funcName return res
def verifyMerging(nbFiles=10, minSize=5000.): """ Verify a PROD3 simtel merging step Keyword arguments: nbFiles -- number of output files expected minSize -- minimum file size """ DIRAC.gLogger.info('Verifying Merging step') # get list of output files outputFiles=glob.glob('Data/sim_telarray/cta-prod3/0.0deg/Data/*simtel.gz') # check the number of output files --- could be done by telescope type N=len(outputFiles) if N != nbFiles : DIRAC.gLogger.error( 'Wrong number of Simtel Merged files : %s instead of %s' % ( N, nbFiles ) ) cleanFiles( outputFiles ) return DIRAC.S_ERROR() # check the file size --- could be done by telescope type for afile in outputFiles: sizekb=os.path.getsize(afile)/1024. if sizekb < minSize: DIRAC.gLogger.error( '%s\n File size too small : %s < %s kb' % ( afile, sizekb, minSize ) ) cleanFiles( outputFiles ) return DIRAC.S_ERROR() DIRAC.gLogger.info( 'Good files found:\n%s'%'\n'.join(outputFiles) ) return DIRAC.S_OK()
def execute(self): """ The method called by the Workflow framework """ from DIRAC.Core.Utilities.Subprocess import systemCall from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron ret = self.__checkInputs() if not ret['OK']: return ret ret = getSoftwareEnviron(self.softwarePackage) if not ret['OK']: error = ret['Message'] self.log.error(error, self.softwarePackage) return DIRAC.S_ERROR(' '.join([error, str(self.softwarePackage)])) envdispEnviron = ret['Value'] cmdTuple = [self.edExe] cmdTuple.extend(self.edArguments) self.log.notice('Executing command tuple:', cmdTuple) ret = systemCall(0, cmdTuple, self.sendOutput, env=envdispEnviron) if not ret['OK']: self.log.error('Failed to execute evndisp:', ret['Message']) return DIRAC.S_ERROR('Can not execute evndisp') status, stdout, stderr = ret['Value'] self.log.notice('evndisp status is:', status) return status
def verify_corsika(): """ Verify a generic Corsika log file """ DIRAC.gLogger.info('Verifying Corsika log file') # get list of output files log_file = glob.glob('Data/corsika/run*/run*.log') if len(log_file) != 1: DIRAC.gLogger.error('no log file found!') return DIRAC.S_ERROR() # check EOR tag tag = '=== END OF RUN ===' content = open(log_file[0]).read() if content.find(tag) < 0: DIRAC.gLogger.error('"%s" tag not found!' % tag) corsika_files = list(os.popen('find . -iname "*corsika.*z*"')) if len(corsika_files) > 0: clean_output_file(corsika_files) simtel_files = list(os.popen('find . -iname "*simtel.*z*"')) if len(simtel_files) > 0: clean_output_file(simtel_files) log_gz_files = list(os.popen('find . -iname "*log.*z*"')) if len(log_gz_files) > 0: clean_output_file(log_gz_files) log_hist_files = list(os.popen('find . -iname "*log_hist.tar"')) if len(log_hist_files) > 0: clean_output_file(log_hist_files) return DIRAC.S_ERROR() return DIRAC.S_OK()
def __addInstanceHistory(self, instanceID, status, load=0.0, jobs=0, transferredFiles=0, transferredBytes=0): """ Insert a History Record """ try: load = float(load) except: return DIRAC.S_ERROR("Load has to be a float value") try: jobs = int(jobs) except: return DIRAC.S_ERROR("Jobs has to be an integer value") try: transferredFiles = int(transferredFiles) except: return DIRAC.S_ERROR( "Transferred files has to be an integer value") self._insert('vm_History', [ 'VMInstanceID', 'Status', 'Load', 'Update', 'Jobs', 'TransferredFiles', 'TransferredBytes' ], [ instanceID, status, load, DIRAC.Time.toString(), jobs, transferredFiles, transferredBytes ]) return
def __getStatus(self, object, id): """ Check and return status of Images and Instances by ID returns: S_OK(Status) if Status is valid and not Error S_ERROR(ErrorMessage) otherwise """ (tableName, validStates, idName) = self.__getTypeTuple(object) if not tableName: return DIRAC.S_ERROR('Unknown DB object: %s' % object) ret = self._getFields(tableName, ['Status', 'ErrorMessage'], [idName], [id]) if not ret['OK']: return ret if not ret['Value']: return DIRAC.S_ERROR('Unknown %s = %s' % (idName, id)) (status, msg) = ret['Value'][0] if not status in validStates: return self.__setError(object, id, 'Invalid Status: %s' % status) if status == validStates[-1]: return DIRAC.S_ERROR(msg) return DIRAC.S_OK(status)
def __getSubmittedInstanceID(self, imageName): """ Retrieve and InstanceID associated to a submitted Instance for a given Image """ (tableName, validStates, idName) = self.__getTypeTuple('Image') imageID = self._getFields(tableName, [idName], ['Name'], [imageName]) if not imageID['OK']: return imageID if not imageID['Value']: return DIRAC.S_ERROR('Unknown Image = %s' % imageName) if len(imageID['Value']) <> 1: return DIRAC.S_ERROR('Image name "%s" is not unique' % imageName) imageID = imageID['Value'][0][0] imageIDName = idName (tableName, validStates, idName) = self.__getTypeTuple('Instance') instanceID = self._getFields(tableName, [idName], [imageIDName, 'Status'], [imageID, 'Submitted']) if not instanceID['OK']: return instanceID if not instanceID['Value']: return DIRAC.S_ERROR('No Submitted instance of "%s" found' % imageName) return DIRAC.S_OK(instanceID['Value'][0][0])
def getImageDict(imageName): """ Return from CS a Dictionary with Image definition """ imagesCSPath = '/Resources/VirtualMachines/Images' definedImages = DIRAC.gConfig.getSections(imagesCSPath) if not definedImages['OK']: return definedImages if imageName not in definedImages['Value']: return DIRAC.S_ERROR('Image "%s" not defined' % imageName) imageCSPath = '%s/%s' % (imagesCSPath, imageName) imageDict = {} flavor = DIRAC.gConfig.getValue('%s/Flavor' % imageCSPath, '') if not flavor: return DIRAC.S_ERROR('Missing Flavor for image "%s"' % imageName) for option, value in DIRAC.gConfig.getOptionsDict( imageCSPath)['Value'].items(): imageDict[option] = value imageRequirementsDict = DIRAC.gConfig.getOptionsDict('%s/Requirements' % imageCSPath) if not imageRequirementsDict['OK']: return DIRAC.S_ERROR('Missing Requirements for image "%s"' % imageName) if 'CPUTime' in imageRequirementsDict['Value']: imageRequirementsDict['Value']['CPUTime'] = int( imageRequirementsDict['Value']['CPUTime']) imageDict['Requirements'] = imageRequirementsDict['Value'] return DIRAC.S_OK(imageDict)
def verify(args): """ simple wrapper to put and register all PROD3 files Keyword arguments: args -- a list of arguments in order [] """ # check command line if len(args) < 3 : res=DIRAC.S_ERROR() res['Message'] = 'verify requires at least nbFiles and fileSize as arguments' return res # now do something stepType = args[0] nbFiles = int(args[1]) fileSize = float(args[2]) if len(args)==4 : path = args[3] # What shall we verify ? if stepType == "corsika": res=verifyCorsika(nbFiles, fileSize) elif stepType == "simtel": res=verifySimtel(nbFiles, fileSize) elif stepType == "merging": res=verifyMerging(nbFiles, fileSize) elif stepType == "analysisinputs": res = verifyAnalysisInputs( fileSize ) elif stepType == "generic": res = verifyGeneric( nbFiles, fileSize, path ) else: res=DIRAC.S_ERROR() res['Message'] = 'Do not know how to verify "%s"'% stepType return res
def verify_corsika(): """ Verify a generic Corsika log file """ DIRAC.gLogger.info('Verifying Corsika log file') # get list of output files log_file = glob.glob('Data/corsika/run*/run*.log') if len(log_file) != 1: DIRAC.gLogger.error('"=== END OF RUN ===" not found!') return DIRAC.S_ERROR() # check EOR tag tag = '=== END OF RUN ===' content = open(log_file[0]).read() if content.find(tag) < 0: DIRAC.gLogger.error('"%s" tag not found!' % tag) corsika_files = glob.glob('Data/corsika/run*/*corsika.*z*') if len(corsika_files) > 0: clean_output_file(corsika_files) simtel_files = glob.glob('Data/sim_telarray/*/*/Data/*simtel.*z*') if len(simtel_files) > 0: clean_output_file(simtel_files) return DIRAC.S_ERROR() return DIRAC.S_OK()
def testMultiThreading(tries): import random DIRAC.gLogger.info('Testing MySQL MultiThreading') DIRAC.gLogger.info('First adding 10 K records') if not DB.checktable()['OK']: return DIRAC.S_ERROR() if not DB.filltable(10000)['OK']: return DIRAC.S_ERROR() i = 0 # overthread = 0 DIRAC.gLogger.info('Now querying 100 K in MultiThread mode') while i < tries: if not i % 1000: DIRAC.gLogger.info('Query:', i) overthread = 0 i += 1 id = int(random.uniform(0, 10000)) + 1 t = threading.Thread(target=testRetrieve, args=(id, )) semaphore.acquire() t.start() n = threading.activeCount() while n > 1: DIRAC.gLogger.info('Waiting for Treads to end:', n) n = threading.activeCount() time.sleep(0.1) DIRAC.gLogger.info('Total retrieved values', Success) DIRAC.gLogger.info('Total Errors', Error) return DIRAC.S_OK((Success, Error))
def verifyCorsika( nbFiles = 6, minSize = 50. ): """ Verify a PROD3 corsika step Keyword arguments: nbFiles -- number of output files expected minSize -- minimum file size """ DIRAC.gLogger.info('Verifying Corsika step') # get list of output files outputFiles=glob.glob('Data/corsika/run*/*corsika.gz') # check the number of output files N=len(outputFiles) if N != nbFiles : DIRAC.gLogger.error( 'Wrong number of Corsika files : %s instead of %s' % ( N, nbFiles ) ) cleanFiles( outputFiles ) return DIRAC.S_ERROR() # check the file size for afile in outputFiles: sizekb=os.path.getsize(afile)/1024. if sizekb < minSize: DIRAC.gLogger.error( '%s\n File size too small : %s < %s kb' % ( afile, sizekb, minSize ) ) cleanFiles( outputFiles ) return DIRAC.S_ERROR() DIRAC.gLogger.info( 'Good files found:\n%s'%'\n'.join(outputFiles) ) return DIRAC.S_OK()
def verifySimtel(nbFiles=31, minSize=50.): """ Verify a PROD3 simtel step Keyword arguments: nbFiles -- number of output files expected minSize -- minimum file size """ DIRAC.gLogger.info('Verifying Simtel step') # get list of output files outputFiles = glob.glob('Data/simtel_tmp/Data/*simtel.gz') # check the number of output files --- could be done by telescope type N = len(outputFiles) if N != nbFiles: DIRAC.gLogger.error('Wrong number of Simtel files : %s instead of %s' % (N, nbFiles)) cleanFiles(outputFiles) return DIRAC.S_ERROR() # check the file size --- could be done by telescope type for afile in outputFiles: sizekb = os.path.getsize(afile) / 1024. if sizekb < minSize: DIRAC.gLogger.error('%s\n File size too small : %s < %s kb' % (afile, sizekb, minSize)) cleanFiles(outputFiles) return DIRAC.S_ERROR() return DIRAC.S_OK()
def createDstFileSystAndMD(): # Creating INDEXES in DFC DB dstDirMDFields = {} dstDirMDFields['dstProdVersion'] = 'VARCHAR(128)' createIndexes(dstDirMDFields) # Adding Directory level metadata Values to DFC dstDirMD = {} dstDirMD['dstProdVersion'] = dstProdVersion res = createDirAndInsertMD(dstDirPath, dstDirMD) if res != DIRAC.S_OK: return DIRAC.S_ERROR('MD Error: Problem creating Dst Directory MD ') dstDataDirPath = os.path.join(dstDirPath, 'Data') dstDataDirMD = {} dstDataDirMD['outputType'] = 'dstData' res = createDirAndInsertMD(dstDataDirPath, dstDataDirMD) if res != DIRAC.S_OK: return DIRAC.S_ERROR('Problem creating Dst Data Directory MD ') dstHistoDirPath = os.path.join(dstDirPath, 'Histograms') dstHistoDirMD = {} dstHistoDirMD['outputType'] = 'dstHisto' res = createDirAndInsertMD(dstHistoDirPath, dstHistoDirMD) if res != DIRAC.S_OK: return DIRAC.S_ERROR('Problem creating Dst Histo Directory MD ') return DIRAC.S_OK('Dst Directory MD successfully created')
def verifyGeneric(nbFiles=1, minSize=50., path='Data/*'): """ Verify a PROD3 generic step Keyword arguments: nbFiles -- number of output files expected minSize -- minimum file size """ DIRAC.gLogger.info('Verifying generic step output') # get list of output files outputFiles = glob.glob(path) # check the number of output files N = len(outputFiles) if N != nbFiles: DIRAC.gLogger.error('Wrong number of output files : %s instead of %s' % (N, nbFiles)) clean_output_file(outputFiles) return DIRAC.S_ERROR() # check the file size for afile in outputFiles: sizekb = os.path.getsize(afile) / 1024. if sizekb < minSize: DIRAC.gLogger.error('%s\n File size too small : %s < %s kb' % (afile, sizekb, minSize)) clean_output_file(outputFiles) return DIRAC.S_ERROR() DIRAC.gLogger.info('Good files found:\n%s' % '\n'.join(outputFiles)) return DIRAC.S_OK()
def createCorsikaFileSystAndMD(): # before creating indexes, it would be fine to know what are those that already exist in the DB # Creating INDEXES in DFC DB corsikaDirMDFields = {} corsikaDirMDFields['prodName'] = 'VARCHAR(128)' corsikaDirMDFields['thetaP'] = 'float' corsikaDirMDFields['phiP'] = 'float' corsikaDirMDFields['altitude'] = 'float' corsikaDirMDFields['particle'] = 'VARCHAR(128)' corsikaDirMDFields['energyInfo'] = 'VARCHAR(128)' corsikaDirMDFields['viewCone'] = 'float' corsikaDirMDFields['corsikaProdVersion'] = 'VARCHAR(128)' corsikaDirMDFields['nbShowers'] = 'int' corsikaDirMDFields['outputType'] = 'VARCHAR(128)' corsikaDirMDFields['runNumSeries'] = 'int' createIndexes(corsikaDirMDFields) # Adding Directory level metadata Values to DFC corsikaDirMD = {} corsikaDirMD['thetaP'] = thetaP corsikaDirMD['phiP'] = phiP corsikaDirMD['altitude'] = obslev corsikaDirMD['energyInfo'] = energyInfo corsikaDirMD['corsikaProdVersion'] = corsikaProdVersion res = createDirAndInsertMD(corsikaDirPath, corsikaDirMD) if res != DIRAC.S_OK: return DIRAC.S_ERROR('Problem creating Corsika Directory MD ') corsikaParticleDirMD = {} corsikaParticleDirMD['particle'] = particle corsikaParticleDirMD['viewCone'] = viewCone res = createDirAndInsertMD(corsikaParticleDirPath, corsikaParticleDirMD) if res != DIRAC.S_OK: return DIRAC.S_ERROR('Problem creating Corsika Particle Directory MD ') corsikaParticleDataDirPath = os.path.join(corsikaParticleDirPath, 'Data') corsikaParticleDataDirMD = {} corsikaParticleDataDirMD['outputType'] = 'corsikaData' res = createDirAndInsertMD(corsikaParticleDataDirPath, corsikaParticleDataDirMD) if res != DIRAC.S_OK: return DIRAC.S_ERROR( 'Problem creating Corsika Particle Data Directory MD ') corsikaParticleLogDirPath = os.path.join(corsikaParticleDirPath, 'Log') corsikaParticleLogDirMD = {} corsikaParticleLogDirMD['outputType'] = 'corsikaLog' res = createDirAndInsertMD(corsikaParticleLogDirPath, corsikaParticleLogDirMD) if res != DIRAC.S_OK: return DIRAC.S_ERROR( 'Problem creating Corsika Particle Log Directory MD ') return DIRAC.S_OK('Corsika Directory MD successfully created')
def execute(self): """ The method called by the Workflow framework """ from DIRAC.Core.Utilities.Subprocess import systemCall from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea ret = self.__checkInputs() if not ret['OK']: return ret ret = getSoftwareEnviron(self.softwarePackage) if not ret['OK']: error = ret['Message'] self.log.error(error, self.softwarePackage) return DIRAC.S_ERROR(' '.join([error, str(self.softwarePackage)])) hapEnviron = ret['Value'] hessroot = hapEnviron['HESSROOT'] rootlogon_file = hessroot + '/rootlogon.C' cp_cmd = 'cp ' + rootlogon_file + ' .' os.system(cp_cmd) fileName = hessroot + self.rootMacro if fileName[-1] == '+': # If the macro has to be compiled there is an extra "+" at the end of its name fileName = fileName[:-1] if not os.path.isfile(fileName): error = 'Hap Root macro file does not exist:' self.log.error(error, fileName) return DIRAC.S_ERROR(' '.join([error, fileName])) fileName = hessroot + self.rootMacro cmdTuple = ['root', '-b', '-q'] cmdTuple += [ '%s( %s )' % (fileName, ', '.join(self.rootArguments).replace("'", '"')) ] self.log.notice('Executing command tuple:', cmdTuple) ret = systemCall(0, cmdTuple, self.sendOutput, env=hapEnviron) if not ret['OK']: self.log.error('Failed to execute Root:', ret['Message']) return DIRAC.S_ERROR('Can not execute Hap root macro') return DIRAC.S_OK()
def __setError(self, object, id, reason): """ """ (tableName, validStates, idName) = self.__getTypeTuple(object) if not tableName: return DIRAC.S_ERROR('Unknown DB object: %s' % object) cmd = 'UPDATE `%s` SET Status = "%s", ErrorMessage = "%s", LastUpdate = UTC_TIMESTAMP() WHERE %s = %s' % ( tableName, validStates[-1], reason, idName, id) ret = self._update(cmd) if not ret['OK']: return ret return DIRAC.S_ERROR(reason)
def execute( self ): """ The method called by the Workflow framework """ from DIRAC.Core.Utilities.Subprocess import systemCall from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron ret = self.__checkInputs() if not ret['OK']: return ret ret = getSoftwareEnviron( self.softwarePackage ) if not ret['OK']: error = ret['Message'] self.log.error( error, self.softwarePackage ) return DIRAC.S_ERROR( ' '.join( [ error, str( self.softwarePackage ) ] ) ) rootEnviron = ret['Value'] fileName = self.rootMacro if fileName[-1] == '+': # If the macro has to be compiled there is an extra "+" at the end of its name fileName = fileName[:-1] if not os.path.isfile( fileName ): error = 'Root macro file does not exist:' self.log.error( error, fileName ) return DIRAC.S_ERROR( ' '.join( [ error, fileName ] ) ) cmdTuple = ['root', '-b', '-q'] cmdTuple += ['%s( %s )' % ( self.rootMacro, ', '.join( self.rootArguments ).replace( "'", '"' ) ) ] self.log.info( 'Executing command tuple:', cmdTuple ) ret = systemCall( 0, cmdTuple, env = rootEnviron ) if not ret['OK']: self.log.error( 'Failed to execute Root:', ret['Message'] ) return DIRAC.S_ERROR( 'Can not execute root' ) status, stdout, stderr = ret['Value'] if status: self.log.error( 'Root execution reports Error:', status ) self.log.error( stdout ) self.log.error( stderr ) return DIRAC.S_ERROR( 'Failed root Execution' ) self.log.info( 'Root stdout:' ) self.log.info( stdout ) return DIRAC.S_OK()
def putAndRegister(self, lfn, localfile, filemetadata, DataType='SimtelProd'): """ put and register one file and set metadata """ # ## Get the list of Production SE res = self._getSEList('ProductionOutputs', DataType) if res['OK']: ProductionSEList = res['Value'] else: return res # ## Get the list of Failover SE res = self._getSEList('ProductionOutputsFailover', DataType) if res['OK']: FailoverSEList = res['Value'] else: return res # ## Upload file to a Production SE res = self._putAndRegisterToSEList(lfn, localfile, ProductionSEList) if not res['OK']: DIRAC.gLogger.error( 'Failed to upload file to any Production SE: %s' % ProductionSEList) # ## Upload file to a Failover SE res = self._putAndRegisterToSEList(lfn, localfile, FailoverSEList) if not res['OK']: return DIRAC.S_ERROR( 'Failed to upload file to any Failover SE: %s' % FailoverSEList) # ## Set file metadata: jobID, subarray, sct if res['OK']: fmd = json.loads(filemetadata) if os.environ.has_key('JOBID'): fmd.update({'jobID': os.environ['JOBID']}) filename = os.path.basename(localfile) # set subarray and sct md from filename p = re.compile('subarray-\d+') if p.search(filename) != None: subarray = p.search(filename).group() fmd.update({'subarray': subarray}) sct = 'False' p = re.compile('nosct') psct = re.compile('sct') if p.search(filename) == None and psct.search(filename) != None: sct = 'True' # ## Set sct flag only for production data res = self.fcc.getFileUserMetadata(lfn) if DataType == 'SimtelProd' and res['Value'][ 'outputType'] == 'Data': fmd.update({'sct': sct}) res = self.fc.setMetadata(lfn, fmd) if not res['OK']: return res return DIRAC.S_OK()
def install_software(self, tar_lfn, target_dir='.'): """ install software package in the current directory """ DIRAC.gLogger.notice('Installing package at %s'%tar_lfn) # Download the tar file DIRAC.gLogger.notice('Trying to download package:', tar_lfn) res = self.dm.getFile(tar_lfn) if not res['OK']: return res if tar_lfn in res['Value']['Successful']: DIRAC.gLogger.notice(' Package downloaded successfully:', tar_lfn) else: error = 'Failed to download package:', tar_lfn return DIRAC.S_ERROR(error) # Extract the tar file to the target directory tar_mode = "r|*" tar = tarfile.open(tar_lfn, tar_mode) for tarInfo in tar: tar.extract(tarInfo, target_dir) tar.close() os.unlink(tar_lfn) # Done DIRAC.gLogger.notice('Package %s installed successfully at:\n%s' %(tar_lfn, target_dir)) return DIRAC.S_OK(target_dir)
def _search_software(self, package, version, compiler, use_cvmfs): ''' Look for sotfware package ''' # software package category category = self.SOFT_CATEGORY_DICT[package] # look for software on cvmfs if use_cvmfs: package_dir = os.path.join(self.CVMFS_DIR, 'centos7', compiler, category, package, version) if os.path.isdir(package_dir): DIRAC.gLogger.notice('Found package %s version %s at:\n%s' % (package, version, package_dir)) return DIRAC.S_OK({'Source':'cvmfs', 'Path':package_dir}) else: DIRAC.gLogger.warn('%s\n not found on cvmfs'%package_dir) # look for tarball in the Dirac file catalog else: package_dir = os.path.join(self.LFN_ROOT, 'centos7', compiler, category, package, version) DIRAC.gLogger.notice('Looking for tarball in %s'%package_dir) results = self.dm.getFilesFromDirectory(package_dir) try: first_file_path = results['Value'][0] if first_file_path[-7:] == '.tar.gz': results = self.dm.getActiveReplicas(first_file_path) if results['OK']: return DIRAC.S_OK({'Source':'tarball', 'Path':package_dir}) except: DIRAC.gLogger.warn('No usual tarball found in the directory') return DIRAC.S_ERROR('Could not find package %s / %s / %s in any location' % (package, version, compiler))
def checkSoftwarePackage(self, package, version, arch="sl6-gcc44", area=None): """ check if the software package is installed in any software area Keyword arguments: package -- package name as the directory name version -- software version as the directory name arch -- architecture as the directory name """ # if area is specified, just get the shared area if area: areaList = self._getSharedArea() else: areaList = self._getSoftwareAreas() if len(areaList) == 0: DIRAC.gLogger.warn('No software area is available') # ## look for the package directory in the software areas for area in areaList: package_dir = os.path.join(area, arch, self.SOFT_CATEGORY_DICT[package], package, version) if os.path.isdir(package_dir): DIRAC.gLogger.notice('Found package %s version %s at:\n%s' % (package, version, area)) return DIRAC.S_OK(package_dir) return DIRAC.S_ERROR( 'Could not find package %s version %s in any location' % (package, version))
def _getSEList( self, SEType = 'ProductionOutputs', DataType = 'SimtelProd' ): """ get from CS the list of available SE for data upload """ opsHelper = Operations() optionName = os.path.join( SEType, DataType ) SEList = opsHelper.getValue( optionName , [] ) SEList = List.randomize( SEList ) DIRAC.gLogger.notice( 'List of %s SE: %s ' % ( SEType, SEList ) ) # # Check if the local SE is in the list. If yes try it first by reversing list order localSEList = [] res = getSEsForSite( DIRAC.siteName() ) if res['OK']: localSEList = res['Value'] retainedlocalSEList = [] for localSE in localSEList: if localSE in SEList: DIRAC.gLogger.notice( 'The local Storage Element is an available SE: ', localSE ) retainedlocalSEList.append( localSE ) SEList.remove( localSE ) SEList = retainedlocalSEList + SEList if len( SEList ) == 0: return DIRAC.S_ERROR( 'Error in building SEList' ) return DIRAC.S_OK( SEList )
def _putAndRegisterToSEList( self, lfn, localfile, SEList ): """ put and register one file to one SE in the SEList """ # ## Try to upload file to a SE in the list for SE in SEList: msg = 'Try to upload local file: %s \nwith LFN: %s \nto %s' % ( localfile, lfn, SE ) DIRAC.gLogger.notice( msg ) res = self.dm.putAndRegister( lfn, localfile, SE ) DIRAC.gLogger.notice(res) # ## check if failed if not res['OK']: DIRAC.gLogger.error( 'Failed to putAndRegister %s \nto %s \nwith message: %s' % ( lfn, SE, res['Message'] ) ) DIRAC.gLogger.notice('Trying to clean up %s' % lfn) res = self.dm.removeFile(lfn) if res['OK']: DIRAC.gLogger.notice('Successfully removed %s \n that was not supposed to have been uploaded successfully' % lfn) continue elif res['Value']['Failed'].has_key( lfn ): DIRAC.gLogger.error( 'Failed to putAndRegister %s to %s' % ( lfn, SE ) ) DIRAC.gLogger.notice('Trying to clean up %s' % lfn) res = self.dm.removeFile(lfn) if res['OK']: DIRAC.gLogger.notice('Successfully removed %s \n that was not supposed to have been uploaded successfully' % lfn) continue else: return DIRAC.S_OK() return DIRAC.S_ERROR()
def __insertInstance(self, imageName, instanceName): """ Attempts to insert a new Instance for the given Image """ image = self.__getImageID(imageName) if not image['OK']: return image imageID = image['Value'] (tableName, validStates, idName) = self.__getTypeTuple('Instance') fields = ['Name', 'VMImageID', 'Status', 'LastUpdate'] values = [instanceName, imageID, validStates[0], DIRAC.Time.toString()] result = getImageDict(imageName) if not result['OK']: return result imgDict = result['Value'] if 'MaxAllowedPrice' in imgDict: fields.append('MaxAllowedPrice') values.append(imgDict['MaxAllowedPrice']) instance = self._insert(tableName, fields, values) if instance['OK'] and 'lastRowId' in instance: self.__addInstanceHistory(instance['lastRowId'], validStates[0]) return DIRAC.S_OK(instance['lastRowId']) if not instance['OK']: return instance return DIRAC.S_ERROR('Failed to insert new Instance')
def setProxyStrength(self, arg): try: self.proxyStrength = int(arg) except: print "Can't parse %s bits! Is it a number?" % arg return DIRAC.S_ERROR("Can't parse strength argument") return DIRAC.S_OK()
def execute(self): """ Check if requested packages are available in shared Area otherwise install them locally """ # For the moment CTA does not use any kind if "platform" only slc5 64bit is supported ??? if not self.packs: # There is nothing to do return DIRAC.S_OK() if self.sharedArea: gLogger.notice('Using Shared Area at:', self.sharedArea) if self.localArea: gLogger.notice('Using Local Area at:', self.localArea) for package in self.packs: gLogger.notice('Checking:', package) if self.sharedArea: if checkSoftwarePackage(package, self.sharedArea)['OK']: gLogger.notice('Package found in Shared Area:', package) continue if self.localArea: if checkSoftwarePackage(package, self.localArea)['OK']: gLogger.notice('Package found in Local Area:', package) continue if installSoftwarePackage(package, self.localArea)['OK']: continue gLogger.error('Check Failed for software package:', package) return DIRAC.S_ERROR('%s not available' % package) return DIRAC.S_OK()
def getInstancesByStatus(self, status): """ Get dictionary of Image Names with InstanceIDs in given status """ if status not in self.validInstanceStates: return DIRAC.S_ERROR('Status %s is not known' % status) (tableName, validStates, idName) = self.__getTypeTuple('Instance') runningInstances = self._getFields(tableName, ['VMImageID', 'UniqueID'], ['Status'], [status]) if not runningInstances['OK']: return runningInstances runningInstances = runningInstances['Value'] instancesDict = {} imagesDict = {} for imageID, uniqueID in runningInstances: if not imageID in imagesDict: (tableName, validStates, idName) = self.__getTypeTuple('Image') imageName = self._getFields(tableName, ['Name'], [idName], [imageID]) if not imageName['OK']: continue imagesDict[imageID] = imageName['Value'][0][0] if not imagesDict[imageID] in instancesDict: instancesDict[imagesDict[imageID]] = [] instancesDict[imagesDict[imageID]].append(uniqueID) return DIRAC.S_OK(instancesDict)