def setReplicaFlag(self, arg):
""" Setter """
if arg.lower() in ('yes', 'no', 'all'):
self.options['ReplicaFlag'] = arg
else:
gLogger.error('Unknown replica flag: %s' % arg)
return DIRAC.exit(1)
return DIRAC.S_OK()
def setVisibility(self, arg):
""" Setter """
if arg.lower() in ('yes', 'no', 'all'):
self.options['Visibility'] = arg
else:
gLogger.error('Unknown visibility flag: %s' % arg)
return DIRAC.exit(1)
return DIRAC.S_OK()
def setProxyLifeTime(self, arg):
try:
fields = [f.strip() for f in arg.split(":")]
self.proxyLifeTime = int(fields[0]) * 3600 + int(fields[1]) * 60
except ValueError:
print "Can't parse %s time! Is it a HH:MM?" % arg
return DIRAC.S_ERROR("Can't parse time argument")
return DIRAC.S_OK()
def listtable(self, entries):
for i in xrange(1, entries + 1):
retDict = self._getFields('MyDB_testTable', [],
inFields=['Status'],
inValues=[i])
if not retDict['OK']:
return retDict
return DIRAC.S_OK(i)
def setServer(optionValue):
global configurationServer
configurationServer = optionValue
DIRAC.gConfig.setOptionValue('/DIRAC/Configuration/Servers',
configurationServer)
DIRAC.gConfig.setOptionValue(cfgInstallPath('ConfigurationServer'),
configurationServer)
return DIRAC.S_OK()
def setSoftwarePackage(self, softwarePackage=None):
"""
Define the software package or try to get it from the job.info file
This method can be used when executing outside a Workflow
"""
if softwarePackage:
self.softwarePackage = softwarePackage
return DIRAC.S_OK()
try:
infoFile = open('job.info')
for line in infoFile.readlines():
if line.find('SoftwarePackages') > -1:
self.softwarePackage = line.split('=')[-1].strip()
break
return DIRAC.S_OK()
except Exception:
return DIRAC.S_ERROR('Can not set softwarePackage')
def _checkemptydir(self, path):
""" check that the directory is not empty
"""
if len(glob.glob(path)) == 0:
error = 'Empty directory: %s' % path
return DIRAC.S_ERROR(error)
else:
return DIRAC.S_OK()
def setProxyLifeTime(self, arg):
try:
fields = [f.strip() for f in arg.split(":")]
self.proxyLifeTime = int(fields[0]) * 3600 + int(fields[1]) * 60
except BaseException:
gLogger.notice("Can't parse %s time! Is it a HH:MM?" % arg)
return DIRAC.S_ERROR("Can't parse time argument")
return DIRAC.S_OK()
def setSiteName(optionValue):
global siteName
siteName = optionValue
DIRAC.gLogger.debug('/LocalSite/Site =', siteName)
Script.localCfg.addDefaultEntry('/LocalSite/Site', siteName)
DIRAC.__siteName = False
DIRAC.gConfig.setOptionValue(cfgInstallPath('SiteName'), siteName)
return DIRAC.S_OK()
def clean_output_file(output_files):
""" Delete Local Files
"""
DIRAC.gLogger.info('Deleting Local Files')
for afile in output_files:
DIRAC.gLogger.warn('Remove local File %s' % afile)
os.remove(afile)
return DIRAC.S_OK()
def installDIRACScripts(self, package_dir):
""" copy prod3 DIRAC scripts in the current directory
"""
cmd = 'cp ' + os.path.join(package_dir, 'dirac_*') + ' .'
if not os.system(cmd):
return DIRAC.S_OK()
else:
return DIRAC.S_ERROR('Failed to install DIRAC scripts')
def setServer(optionValue):
global configurationServer
configurationServer = optionValue
Script.localCfg.addDefaultEntry('/DIRAC/Configuration/Servers',
configurationServer)
DIRAC.gConfig.setOptionValue(cfgInstallPath('ConfigurationServer'),
configurationServer)
return DIRAC.S_OK()
def putAndRegisterPROD3(args):
""" simple wrapper to put and register all PROD3 files
Keyword arguments:
args -- a list of arguments in order []
"""
metadata = args[0]
metadatafield = args[1]
filemetadata = args[2]
inputpath = args[3]
basepath = args[4]
start_run_nb = args[5]
catalogs = ['DIRACFileCatalog']
# # Create MD structure
prod3dm = Prod3DataManager(catalogs)
res = prod3dm.createMDStructure(metadata, metadatafield, basepath,
'tel_sim')
if res['OK']:
path = res['Value']
else:
return res
# # Upload data files
datadir = os.path.join(inputpath, 'Data/*')
res = prod3dm._checkemptydir(datadir)
if not res['OK']:
return res
# ## Add start_run_number to run_number and update filemetadata
fmd = json.loads(filemetadata)
run_number = int(fmd['runNumber']) + int(start_run_nb)
fmd['runNumber'] = '%08d' % run_number
fmdjson = json.dumps(fmd)
# ## Get the run directory
# runpath = prod3dm._getRunPath( fmdjson )
runpath = prod3dm._getRunPath(fmd)
for localfile in glob.glob(datadir):
filename = os.path.basename(localfile)
lfn = os.path.join(path, 'Data', runpath, filename)
res = prod3dm.putAndRegister(lfn, localfile, fmdjson)
if not res['OK']:
return res
# ## Upload log files
tarname = filename.split('___cta-prod3')[0] + '.tar.gz'
res = prod3dm.createTarLogFiles(inputpath, tarname)
if not res['OK']:
return DIRAC.S_ERROR('prod3dm.createTarLogFiles failed')
lfn = os.path.join(path, 'Log', runpath, tarname)
# res = prod3dm.putAndRegister( lfn, tarname, filemetadata )
res = prod3dm.putAndRegister(lfn, tarname, fmdjson)
if not res['OK']:
return res
return DIRAC.S_OK()
def checkRecord(self):
result = self.getValue("ExecTime")
if not result["OK"]:
return result
execTime = result["Value"]
if execTime > 33350400: # 1 year
return DIRAC.S_ERROR(
"OOps. More than 1 year of cpu time smells fishy!")
return DIRAC.S_OK()
def dumpSetupScriptPath(self,
package_dir,
textfilename='setup_script_path.txt'):
""" dump the path to setupPackage.sh in a one line ascii file
to be read and source by the following script
"""
script_path = os.path.join(package_dir, 'setupPackage.sh')
open(textfilename, 'w').writelines(script_path + '\n')
return DIRAC.S_OK()
def createCorsikaFileSystAndMD():
# before creating indexes, it would be fine to know what are those that already exist in the DB
# Creating INDEXES in DFC DB
corsikaDirMDFields = {}
corsikaDirMDFields['prodName'] = 'VARCHAR(128)'
corsikaDirMDFields['thetaP'] = 'float'
corsikaDirMDFields['phiP'] = 'float'
corsikaDirMDFields['altitude'] = 'float'
corsikaDirMDFields['particle'] = 'VARCHAR(128)'
corsikaDirMDFields['energyInfo'] = 'VARCHAR(128)'
corsikaDirMDFields['viewCone'] = 'float'
corsikaDirMDFields['corsikaProdVersion'] = 'VARCHAR(128)'
corsikaDirMDFields['nbShowers'] = 'int'
corsikaDirMDFields['outputType'] = 'VARCHAR(128)'
corsikaDirMDFields['runNumSeries'] = 'int'
createIndexes(corsikaDirMDFields)
# Adding Directory level metadata Values to DFC
corsikaDirMD = {}
corsikaDirMD['thetaP'] = thetaP
corsikaDirMD['phiP'] = phiP
corsikaDirMD['altitude'] = obslev
corsikaDirMD['energyInfo'] = energyInfo
corsikaDirMD['corsikaProdVersion'] = corsikaProdVersion
res = createDirAndInsertMD(corsikaDirPath, corsikaDirMD)
if res != DIRAC.S_OK:
return DIRAC.S_ERROR('Problem creating Corsika Directory MD ')
corsikaParticleDirMD = {}
corsikaParticleDirMD['particle'] = particle
corsikaParticleDirMD['viewCone'] = viewCone
res = createDirAndInsertMD(corsikaParticleDirPath, corsikaParticleDirMD)
if res != DIRAC.S_OK:
return DIRAC.S_ERROR('Problem creating Corsika Particle Directory MD ')
corsikaParticleDataDirPath = os.path.join(corsikaParticleDirPath, 'Data')
corsikaParticleDataDirMD = {}
corsikaParticleDataDirMD['outputType'] = 'corsikaData'
res = createDirAndInsertMD(corsikaParticleDataDirPath,
corsikaParticleDataDirMD)
if res != DIRAC.S_OK:
return DIRAC.S_ERROR(
'Problem creating Corsika Particle Data Directory MD ')
corsikaParticleLogDirPath = os.path.join(corsikaParticleDirPath, 'Log')
corsikaParticleLogDirMD = {}
corsikaParticleLogDirMD['outputType'] = 'corsikaLog'
res = createDirAndInsertMD(corsikaParticleLogDirPath,
corsikaParticleLogDirMD)
if res != DIRAC.S_OK:
return DIRAC.S_ERROR(
'Problem creating Corsika Particle Log Directory MD ')
return DIRAC.S_OK('Corsika Directory MD successfully created')
def cleanLocalFiles ( self, datadir, pattern = '*' ):
""" remove files matching pattern in datadir
"""
for localfile in glob.glob( os.path.join( datadir, pattern ) ):
DIRAC.gLogger.notice( 'Removing local file: ', localfile )
os.remove( localfile )
return DIRAC.S_OK()
def checkSoftwarePackage(package, area):
"""
Check if the requested version of package is installed under the given area
"""
fileName = _getEnvFileName(package, area)
softwareDir = _getSoftwareDir(package, area)
try:
gLogger.notice('Looking for Environment file', fileName)
if os.path.exists(fileName):
return DIRAC.S_OK()
else:
gLogger.notice('Looking for Software dir', softwareDir)
if os.path.isdir(softwareDir):
return DIRAC.S_OK()
except Exception:
pass
return DIRAC.S_ERROR()
def removeSoftwarePackage(self, packagedir):
""" remove Software Package
"""
cmd = 'rm -Rf ' + packagedir
if not os.system(cmd):
return DIRAC.S_OK()
else:
error = 'Failed to remove %s' % packagedir
return DIRAC.S_ERROR(error)
def checktable(self):
retDict = self._update('DROP TABLE IF EXISTS `MyDB_testTable`')
if not retDict['OK']: return retDict
retDict = self._update('CREATE TABLE `MyDB_testTable` ( '
'`ID` INTEGER NOT NULL AUTO_INCREMENT, '
'`LastUpdate` TIMESTAMP, '
'`Status` varchar(128), '
'PRIMARY KEY (`ID`) )')
if not retDict['OK']: return retDict
return DIRAC.S_OK()
def install_dirac_scripts(self, package_dir):
""" copy DIRAC scripts in the current directory
"""
dirac_scripts = glob.glob(os.path.join(package_dir, 'dirac_*'))
try:
for one_file in dirac_scripts:
shutil.copy2(one_file, '.')
return DIRAC.S_OK()
except shutil.Error as error:
return DIRAC.S_ERROR('Failed to install DIRAC scripts:\n%s'%error)
def _getSharedArea(self):
""" get Shared Area
"""
if os.environ.has_key(self.SW_SHARED_DIR):
shared_area = os.path.join(os.environ[self.SW_SHARED_DIR],
'software')
else:
return DIRAC.S_ERROR('Shared area not found')
return DIRAC.S_OK(shared_area)
def setupSoftware(args):
""" setup a given software package
to be used in main
Keyword arguments:
args -- a list of arguments in order [package, version, arch]
"""
# check number of arguments
if len(args) < 2:
Script.showHelp()
return DIRAC.S_ERROR('Wrong number of arguments')
# get arguments
package = args[0]
version = args[1]
program_category = 'simulations'
if len(args) >= 3:
program_category = args[2]
arch = "sl6-gcc44"
if len(args) == 4:
arch = args[3]
soft_category = {package: program_category}
prod3swm = Prod3SoftwareManager(soft_category)
# check if and where Package is installed
res = prod3swm.checkSoftwarePackage(package, version, arch)
if not res['OK']:
res = prod3swm.installSoftwarePackage(package, version, arch)
if not res['OK']:
return res
else:
package_dir = res['Value']
prod3swm.dumpSetupScriptPath(package_dir)
return DIRAC.S_OK()
# # dump the SetupScriptPath to be sourced by DIRAC scripts
# ## copy DIRAC scripts in the current directory
package_dir = res['Value']
prod3swm.dumpSetupScriptPath(package_dir)
res = prod3swm.installDIRACScripts(package_dir)
if not res['OK']:
return res
return DIRAC.S_OK()
def __setLastLoadAndUptime(self, instanceID, load, uptime):
if not uptime:
result = self._query(
"SELECT MAX( UNIX_TIMESTAMP( `Update` ) ) - MIN( UNIX_TIMESTAMP( `Update` ) ) FROM `vm_History` WHERE VMInstanceID = %d GROUP BY VMInstanceID"
% instanceID)
if result['OK'] and len(result['Value']) > 0:
uptime = int(result['Value'][0][0])
sqlUpdate = "UPDATE `vm_Instances` SET `Uptime` = %d, `Load` = %f WHERE `VMInstanceID` = %d" % (
uptime, load, instanceID)
self._update(sqlUpdate)
return DIRAC.S_OK()
def execute(self):
"""
The method called by the Workflow framework
"""
from DIRAC.Core.Utilities.Subprocess import systemCall
from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea
ret = self.__checkInputs()
if not ret['OK']:
return ret
ret = getSoftwareEnviron(self.softwarePackage)
if not ret['OK']:
error = ret['Message']
self.log.error(error, self.softwarePackage)
return DIRAC.S_ERROR(' '.join([error, str(self.softwarePackage)]))
hapEnviron = ret['Value']
hessroot = hapEnviron['HESSROOT']
rootlogon_file = hessroot + '/rootlogon.C'
cp_cmd = 'cp ' + rootlogon_file + ' .'
os.system(cp_cmd)
fileName = hessroot + self.rootMacro
if fileName[-1] == '+':
# If the macro has to be compiled there is an extra "+" at the end of its name
fileName = fileName[:-1]
if not os.path.isfile(fileName):
error = 'Hap Root macro file does not exist:'
self.log.error(error, fileName)
return DIRAC.S_ERROR(' '.join([error, fileName]))
fileName = hessroot + self.rootMacro
cmdTuple = ['root', '-b', '-q']
cmdTuple += [
'%s( %s )' %
(fileName, ', '.join(self.rootArguments).replace("'", '"'))
]
self.log.notice('Executing command tuple:', cmdTuple)
ret = systemCall(0, cmdTuple, self.sendOutput, env=hapEnviron)
if not ret['OK']:
self.log.error('Failed to execute Root:', ret['Message'])
return DIRAC.S_ERROR('Can not execute Hap root macro')
return DIRAC.S_OK()
def __checkInputs(self):
"""
Check the data members for the expected Input Parameters
"""
for inputPar in INPUTPARAMETERS:
if not inputPar in dir(self):
error = 'Input Parameter not defined:'
self.log.error(error, inputPar)
return DIRAC.S_ERROR(' '.join([error, inputPar]))
return DIRAC.S_OK()
def setDirectory(self, arg):
""" Setter """
if os.path.exists(arg) and not os.path.isdir(arg):
with open(arg, 'r') as inFile:
directories = [
line.split()[0] for line in inFile.read().splitlines()
if line.strip()
]
else:
directories = arg.split(',')
self.options.setdefault('Directory', set()).update(directories)
return DIRAC.S_OK()
def setIssuer(self, optionValue):
# If the user selects the issuer, it means that there will be authorization through tokens
os.environ["DIRAC_USE_ACCESS_TOKEN"] = "True"
# Allow the user to enter only the server domain
if not optionValue.startswith("http://") or not optionValue.startswith(
"https://"):
optionValue = f"https://{optionValue.lstrip('/')}"
if len(optionValue.strip("/").split("/")) == 3:
optionValue = f"{optionValue.rstrip('/')}/auth"
self.issuer = optionValue
DIRAC.gConfig.setOptionValue("/DIRAC/Security/Authorization/issuer",
self.issuer)
return DIRAC.S_OK()
def downloadFile(lfn):
""" Download a file using DMS
Keyword arguments:
lfn -- a logical file name
"""
DIRAC.gLogger.info('Downloading ',lfn)
dm = DataManager()
res=dm.getFile(lfn)
if not res['OK']:
DIRAC.gLogger.error ( res['Message'] )
DIRAC.gLogger.error ( 'Could not download %s'%lfn )
DIRAC.exit( -1 )
return DIRAC.S_OK()
def setProductions(self, arg):
"""
Parse production numbers
"""
prods = []
if arg.upper() == "ALL":
self.options['Productions'] = arg
return DIRAC.S_OK()
try:
for prod in arg.split(','):
if prod.find(":") > 0:
pr = prod.split(":")
for i in xrange(int(pr[0]), int(pr[1]) + 1):
prods.append(i)
else:
prods.append(prod)
self.options['Productions'] = [int(prod) for prod in prods]
except ValueError as e:
gLogger.error("Invalid production switch value: %s" % arg, repr(e))
self.options['Productions'] = ['Invalid']
return DIRAC.S_ERROR()
return DIRAC.S_OK()
