def master_configure(self):
exePath = fullpath(self.exe)
if not os.path.isfile(exePath):
raise ApplicationConfigurationError\
( None, "exe = '%s' not found" % self.exe )
listFile = fullpath(self.imageList)
if not os.path.isfile(listFile):
raise ApplicationConfigurationError\
( None, "imageList file = '%s' not found" % self.imageList )
imageFile = open(listFile)
imageDir = fullpath(imageFile.readline().strip().rstrip(os.sep))
inputList = imageFile.readlines()
imageFile.close()
if not imageDir:
raise ApplicationConfigurationError\
( None, "Top-level image directory '%s' not found" % imageDir )
outDir = fullpath(self.outDir)
if not outDir:
outDir = imageDir
if not os.path.isdir(outDir):
os.makedirs(outDir)
imageList = []
for filename in inputList:
relativePath = filename.strip().strip(os.sep)
imagePath = fullpath(os.path.join(imageDir, relativePath))
if os.path.exists(imagePath):
imageList.append(relativePath)
resultDir = os.path.join(os.path.dirname(imagePath), "vseg")
if os.path.isdir(resultDir):
shutil.rmtree(resultDir)
else:
logger.warning("Specified image file '%s' not found" %
imagePath)
logger.warning("File will be skipped")
if len(imageList) >= self.maxImage:
if abs(self.maxImage) == self.maxImage:
break
appDict = \
{
"exePath" : exePath,
"imageDir" : imageDir,
"outDir" : outDir,
"imageList" : imageList
}
return (False, appDict)
def master_configure( self ):
listFile = fullpath( self.docList )
if not os.path.isfile( listFile ):
raise ApplicationConfigurationError\
( None, "docList file = '%s' not found" % self.docList )
docFile = open( listFile )
docDir = docFile.readline().strip().rstrip( os.sep )
inputList = []
if docDir:
docDir = fullpath( docDir )
inputList = docFile.readlines()
docFile.close()
outDir = fullpath( self.outDir )
if not outDir:
outDir = docDir
tagFile = ""
else:
outDir = fullpath( self.outDir )
tagFile = listFile
if not outDir:
raise ApplicationConfigurationError\
( None, "Output directory not defined" )
if not os.path.isdir( outDir ):
os.makedirs( outDir )
docList = []
for input in inputList:
url = input.split()[ 0 ]
relativePath = input[ len( url ) : ].strip().strip( os.sep )
docList.append( ( url, relativePath ) )
if len( docList ) >= self.maxDoc:
if abs( self.maxDoc ) == self.maxDoc:
break
if self.softwareDir:
softwareDir = fullpath( self.softwareDir )
else:
softwareDir = ""
appDict = \
{
"version" : self.version,
"docDir" : docDir,
"outDir" : outDir,
"docList" : docList,
"tagFile" : tagFile,
"softwareDir" : softwareDir
}
return( False, appDict )
def master_configure(self):
listFile = fullpath(self.docList)
if not os.path.isfile(listFile):
raise ApplicationConfigurationError\
( None, "docList file = '%s' not found" % self.docList )
docFile = open(listFile)
docDir = docFile.readline().strip().rstrip(os.sep)
inputList = []
if docDir:
docDir = fullpath(docDir)
inputList = docFile.readlines()
docFile.close()
outDir = fullpath(self.outDir)
if not outDir:
outDir = docDir
tagFile = ""
else:
outDir = fullpath(self.outDir)
tagFile = listFile
if not outDir:
raise ApplicationConfigurationError\
( None, "Output directory not defined" )
if not os.path.isdir(outDir):
os.makedirs(outDir)
docList = []
for input in inputList:
url = input.split()[0]
relativePath = input[len(url):].strip().strip(os.sep)
docList.append((url, relativePath))
if len(docList) >= self.maxDoc:
if abs(self.maxDoc) == self.maxDoc:
break
if self.softwareDir:
softwareDir = fullpath(self.softwareDir)
else:
softwareDir = ""
appDict = \
{
"version" : self.version,
"docDir" : docDir,
"outDir" : outDir,
"docList" : docList,
"tagFile" : tagFile,
"softwareDir" : softwareDir
}
return (False, appDict)
def master_configure( self ):
exePath = fullpath( self.exe )
if not os.path.isfile( exePath ):
raise ApplicationConfigurationError\
( None, "exe = '%s' not found" % self.exe )
listFile = fullpath( self.imageList )
if not os.path.isfile( listFile ):
raise ApplicationConfigurationError\
( None, "imageList file = '%s' not found" % self.imageList )
imageFile = open( listFile )
imageDir = fullpath( imageFile.readline().strip().rstrip( os.sep ) )
inputList = imageFile.readlines()
imageFile.close()
if not imageDir:
raise ApplicationConfigurationError\
( None, "Top-level image directory '%s' not found" % imageDir )
outDir = fullpath( self.outDir )
if not outDir:
outDir = imageDir
if not os.path.isdir( outDir ):
os.makedirs( outDir )
imageList = []
for filename in inputList:
relativePath = filename.strip().strip( os.sep )
imagePath = fullpath( os.path.join( imageDir, relativePath ) )
if os.path.exists( imagePath ):
imageList.append( relativePath )
resultDir = os.path.join( os.path.dirname( imagePath ), "vseg" )
if os.path.isdir( resultDir ):
shutil.rmtree( resultDir )
else:
logger.warning( "Specified image file '%s' not found" % imagePath )
logger.warning( "File will be skipped" )
if len( imageList ) >= self.maxImage:
if abs( self.maxImage ) == self.maxImage:
break
appDict = \
{
"exePath" : exePath,
"imageDir" : imageDir,
"outDir" : outDir,
"imageList" : imageList
}
return( False, appDict )
def _check_inputs(self):
"""Checks the validity of user's entries for BenderScript schema"""
if not self.scripts and not self.imports :
raise ApplicationConfigurationError("Application scripts/imports are not defined")
if isinstance ( self.scripts , str ) : self.scripts = [ File ( self.scripts ) ]
if isinstance ( self.imports , str ) : self.imports = [ File ( self.imports ) ]
for f in self.scripts : f.name = fullpath ( f.name )
for f in self.imports : f.name = fullpath ( f.name )
def unpack( self, job = None, tarFile = None ):
if not job:
job = self._getParent()
if not tarFile:
tarFile = fullpath( os.path.join( job.outputdir, "%s.tar.gz" \
% self.tarfile ) )
if not os.path.exists( tarFile ):
logger.warning( "Output not found for job %s" % job.id )
logger.warning( "Setting job status to failed" )
job.updateStatus( "failed" )
return False
outDir = job.application.outDir
if not outDir:
listFile = fullpath( job.application.imageList )
if not os.path.isfile( listFile ):
logger.warning( "imageList file = '%s' not found for job %s" % \
( job.application.imageList, job.id ) )
logger.warning( "Setting job status to failed" )
job.updateStatus( "failed" )
return False
imageFile = open( listFile )
job.application.outDir = imageFile.readline().strip().rstrip( os.sep )
imageFile.close()
outDir = fullpath( job.application.outDir )
if not os.path.isdir( outDir ):
os.makedirs( outDir )
tmpStore = os.path.join( job.outputdir, "tmp" )
if not os.path.isdir( tmpStore ):
os.makedirs( tmpStore )
untarCommand = "tar -C %s -zxf %s" % ( tmpStore, tarFile )
status = os.system( untarCommand )
if ( 0 != status ):
logger.warning( "Problems untarring output for job %s" % job.id )
logger.warning( "Setting job status to failed" )
job.updateStatus( "failed" )
return False
copyCommand = "cp -r %s/* %s" % ( tmpStore, outDir )
status = os.system( copyCommand )
if ( 0 != status ):
logger.warning( "Problems copying output for job %s" % job.id )
logger.warning( "Setting job status to failed" )
job.updateStatus( "failed" )
return False
return True
def _check_inputs(self):
"""Checks the validity of user's entries for BenderScript schema"""
if not self.scripts and not self.imports :
raise ApplicationConfigurationError("Application scripts/imports are not defined")
if isinstance ( self.scripts , str ) : self.scripts = [ File ( self.scripts ) ]
if isinstance ( self.imports , str ) : self.imports = [ File ( self.imports ) ]
for f in self.scripts : f.name = fullpath ( f.name )
for f in self.imports : f.name = fullpath ( f.name )
def _check_inputs(self):
"""Checks the validity of some of user's entries for Gaudi schema"""
# Warn user that no optsfiles given
if len(self.optsfile) == 0:
logger.warning("The 'optsfile' is not set. I hope this is OK!")
else:
# Check for non-exting optsfiles defined
nonexistentOptFiles = []
for f in self.optsfile:
from Ganga.GPIDev.Lib.File.File import File
if type(f) is str:
myFile = File(f)
else:
myFile = f
myFile.name = fullpath(myFile.name)
if not os.path.isfile(myFile.name):
nonexistentOptFiles.append(myFile)
if len(nonexistentOptFiles):
tmpmsg = "The 'optsfile' attribute contains non-existent file/s: ["
for _f in nonexistentOptFiles:
tmpmsg += "'%s', " % _f.name
msg = tmpmsg[:-2] + ']'
raise ApplicationConfigurationError(msg)
def _check_inputs(self):
"""Checks the validity of some of user's entries for Gaudi schema"""
# Warn user that no optsfiles given
if len(self.optsfile) == 0:
logger.warning("The 'optsfile' is not set. I hope this is OK!")
else:
# Check for non-exting optsfiles defined
nonexistentOptFiles = []
for f in self.optsfile:
from Ganga.GPIDev.Lib.File.File import File
if type(f) is str:
myFile = File(f)
else:
myFile = f
myFile.name = fullpath(myFile.name)
if not os.path.isfile(myFile.name):
nonexistentOptFiles.append(myFile)
if len(nonexistentOptFiles):
tmpmsg = "The 'optsfile' attribute contains non-existent file/s: ["
for _f in nonexistentOptFiles:
tmpmsg += "'%s', " % _f.name
msg = tmpmsg[:-2] + ']'
raise ApplicationConfigurationError(msg)
def download( self, job = None, tarFile = None ):
if not job:
job = self._getParent()
if not tarFile:
tarFile = fullpath( os.path.join( job.outputdir, "%s.tar.gz" \
% self.tarfile ) )
if ( "LCG" == job.backend._name ):
gridUrl = \
os.path.join( job.outputdata.getGridStorage(), "%s.tar.gz" \
% self.tarfile )
cp = "globus-url-copy %s file:%s" % ( gridUrl, tarFile )
status = shell.cmd1( cmd = cp, allowed_exit = range( 1000 ) )[ 0 ]
if ( 0 != status ):
logger.warning( "Problems retrieving output for job %s" % job.id )
logger.warning( "Setting job status to failed" )
job.updateStatus( "failed" )
return False
rm = "edg-gridftp-rm %s" % gridUrl
shell.cmd1( cmd = rm, allowed_exit = range( 1000 ) )
rmdir = "edg-gridftp-rmdir %s" % os.path.dirname( gridUrl )
shell.cmd1( cmd = rmdir, allowed_exit = range( 1000 ) )
return True
def _check_inputs(self):
"""Checks the validity of user's entries for Ostap schema"""
if not self.scripts and not self.commands and not self.arguments :
raise ApplicationConfigurationError(None, "Application scripts are not defined")
if isinstance ( self.scripts , str ) : self.scripts = [ File ( self.scripts ) ]
for f in self.scripts : f.name = fullpath ( f.name )
def _check_inputs(self):
"""Checks the validity of user's entries for Ostap schema"""
if not self.scripts and not self.commands and not self.arguments:
raise ApplicationConfigurationError(
None, "Application scripts are not defined")
if isinstance(self.scripts, str): self.scripts = [File(self.scripts)]
for f in self.scripts:
f.name = fullpath(f.name)
def checkDiskQuota():
import subprocess
from Ganga.Utility.files import fullpath
repo_partition = getLocalRoot()
repo_partition = os.path.realpath(repo_partition)
work_partition = getLocalWorkspace()
work_partition = os.path.realpath(work_partition)
folders_to_check = [repo_partition, work_partition]
home_dir = os.environ['HOME']
to_remove = []
for partition in folders_to_check:
if not os.path.exists(partition):
if home_dir not in folders_to_check:
folders_to_check.append(home_dir)
to_remove.append(partition)
for folder in to_remove:
folders_to_check.remove(folder)
for data_partition in folders_to_check:
if fullpath(data_partition).find('/afs') == 0:
quota = subprocess.Popen(['fs', 'quota', '%s' % data_partition], stdout=subprocess.PIPE)
output = quota.communicate()[0]
logger.debug("fs quota %s:\t%s" % (str(data_partition), str(output)))
else:
df = subprocess.Popen(["df", '-Pk', data_partition], stdout=subprocess.PIPE)
output = df.communicate()[0]
try:
global partition_warning
global partition_critical
quota_percent = output.split('%')[0]
if int(quota_percent) >= partition_warning:
logger.warning("WARNING: You're running low on disk space, Ganga may stall on launch or fail to download job output")
logger.warning("WARNING: Please free some disk space on: %s" % str(data_partition))
if int(quota_percent) >= partition_critical and config['force_start'] is False:
logger.error("You are crtitically low on disk space!")
logger.error("To prevent repository corruption and data loss we won't start Ganga.")
logger.error("Either set your config variable 'force_start' in .gangarc to enable starting and ignore this check.")
logger.error("Or, make sure you have more than %s percent free disk space on: %s" %(str(100-partition_critical), str(data_partition)))
raise GangaException("Not Enough Disk Space!!!")
except GangaException as err:
raise err
except Exception as err:
logger.debug("Error checking disk partition: %s" % str(err))
return
def checkDiskQuota():
import subprocess
repo_partition = getLocalRoot()
repo_partition = os.path.realpath(repo_partition)
work_partition = getLocalWorkspace()
work_partition = os.path.realpath(work_partition)
folders_to_check = [repo_partition, work_partition]
home_dir = os.environ['HOME']
to_remove = []
for partition in folders_to_check:
if not os.path.exists(partition):
if home_dir not in folders_to_check:
folders_to_check.append(home_dir)
to_remove.append(partition)
for folder in to_remove:
folders_to_check.remove(folder)
for data_partition in folders_to_check:
if fullpath(data_partition, True).find('/afs') == 0:
quota = subprocess.Popen(['fs', 'quota', '%s' % data_partition], stdout=subprocess.PIPE)
output = quota.communicate()[0]
logger.debug("fs quota %s:\t%s" % (str(data_partition), str(output)))
else:
df = subprocess.Popen(["df", '-Pk', data_partition], stdout=subprocess.PIPE)
output = df.communicate()[0]
try:
global partition_warning
global partition_critical
quota_percent = output.split('%')[0]
if int(quota_percent) >= partition_warning:
logger.warning("WARNING: You're running low on disk space, Ganga may stall on launch or fail to download job output")
logger.warning("WARNING: Please free some disk space on: %s" % str(data_partition))
if int(quota_percent) >= partition_critical and config['force_start'] is False:
logger.error("You are crtitically low on disk space!")
logger.error("To prevent repository corruption and data loss we won't start Ganga.")
logger.error("Either set your config variable 'force_start' in .gangarc to enable starting and ignore this check.")
logger.error("Or, make sure you have more than %s percent free disk space on: %s" %(str(100-partition_critical), str(data_partition)))
raise GangaException("Not Enough Disk Space!!!")
except GangaException as err:
raise err
except Exception as err:
logger.debug("Error checking disk partition: %s" % str(err))
return
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig):
local = VansegLocal.prepare\
( self, app, appsubconfig, appmasterconfig, jobmasterconfig )
inbox = []
inbox.extend(local.inputbox)
exeFile = File(fullpath(app.exe))
if not exeFile in inbox:
inbox.append(exeFile)
requirements = LCGRequirements()
lcg = LCGJobConfig( exe = local.exe, inputbox = inbox,\
outputbox = local.outputbox, requirements = requirements )
return lcg
def _check_inputs(self):
"""Checks the validity of user's entries for GaudiPython schema"""
if len(self.script) == 0:
logger.warning("No script defined. Will use a default "
'script which is probably not what you want.')
self.script = [File(os.path.join(
os.path.dirname(inspect.getsourcefile(GaudiPython)),
'options/GaudiPythonExample.py'))]
else:
for f in self.script:
f.name = fullpath(f.name)
return
def prepare( self, app, appsubconfig, appmasterconfig, jobmasterconfig ):
local = VansegLocal.prepare\
( self, app, appsubconfig, appmasterconfig, jobmasterconfig )
inbox = []
inbox.extend( local.inputbox )
exeFile = File( fullpath( app.exe ) )
if not exeFile in inbox:
inbox.append( exeFile )
requirements = LCGRequirements()
lcg = LCGJobConfig( exe = local.exe, inputbox = inbox,\
outputbox = local.outputbox, requirements = requirements )
return lcg
def _check_inputs(self):
"""Checks the validity of user's entries for GaudiPython schema"""
# Always check for None OR empty
if self.module.name == None:
raise ApplicationConfigurationError(
None, "Application Module not requested")
elif self.module.name == "":
raise ApplicationConfigurationError(
None, "Application Module not requested")
else:
# Always check we've been given a FILE!
self.module.name = fullpath(self.module.name)
if not os.path.isfile(self.module.name):
msg = 'Module file %s not found.' % self.module.name
raise ApplicationConfigurationError(None, msg)
def _check_inputs(self):
"""Checks the validity of user's entries for GaudiPython schema"""
if len(self.script) == 0:
logger.warning("No script defined. Will use a default "
'script which is probably not what you want.')
self.script = [
File(
os.path.join(
os.path.dirname(inspect.getsourcefile(GaudiPython)),
'options/GaudiPythonExample.py'))
]
else:
for f in self.script:
f.name = fullpath(f.name)
return
def master_configure( self ):
listFile = fullpath( self.imageList )
if not os.path.isfile( listFile ):
raise ApplicationConfigurationError\
( None, "imageList file = '%s' not found" % self.imageList )
imageFile = open( listFile )
imageDir = imageFile.readline().strip().rstrip( os.sep )
inputList = []
if imageDir:
imageDir = fullpath( imageDir )
inputList = imageFile.readlines()
imageFile.close()
outDir = fullpath( self.outDir )
if not outDir:
outDir = imageDir
tagFile = ""
else:
outDir = fullpath( self.outDir )
tagFile = listFile
if not outDir:
raise ApplicationConfigurationError\
( None, "Output directory not defined" )
if not os.path.isdir( outDir ):
os.makedirs( outDir )
imageList = []
for input in inputList:
url = input.split()[ 0 ]
relativePath = input[ len( url ) : ].strip().strip( os.sep )
imageList.append( ( url, relativePath ) )
if len( imageList ) >= self.maxImage:
if abs( self.maxImage ) == self.maxImage:
break
appDict = \
{
"version" : self.version,
"classifierDir" : fullpath( self.classifierDir ),
"libList" : [],
"imageDir" : imageDir,
"outDir" : outDir,
"imageList" : imageList,
"tagFile" : tagFile
}
for lib in self.libList:
appDict[ "libList" ].append( fullpath( lib ) )
return( False, appDict )
def master_configure( self ):
listFile = fullpath( self.imageList )
if not os.path.isfile( listFile ):
raise ApplicationConfigurationError\
( None, "imageList file = '%s' not found" % self.imageList )
imageFile = open( listFile )
imageDir = imageFile.readline().strip().rstrip( os.sep )
inputList = []
if imageDir:
imageDir = fullpath( imageDir )
inputList = imageFile.readlines()
imageFile.close()
outDir = fullpath( self.outDir )
if not outDir:
outDir = imageDir
tagFile = ""
else:
outDir = fullpath( self.outDir )
tagFile = listFile
if not outDir:
raise ApplicationConfigurationError\
( None, "Output directory not defined" )
if not os.path.isdir( outDir ):
os.makedirs( outDir )
imageList = []
for input in inputList:
url = input.split()[ 0 ]
relativePath = input[ len( url ) : ].strip().strip( os.sep )
imageList.append( ( url, relativePath ) )
if len( imageList ) >= self.maxImage:
if abs( self.maxImage ) == self.maxImage:
break
appDict = \
{
"version" : self.version,
"classifierDir" : fullpath( self.classifierDir ),
"libList" : [],
"imageDir" : imageDir,
"outDir" : outDir,
"imageList" : imageList,
"tagFile" : tagFile
}
for lib in self.libList:
appDict[ "libList" ].append( fullpath( lib ) )
return( False, appDict )
def checkDiskQuota():
import subprocess
from Ganga.Utility.files import fullpath
repo_partition = getLocalRoot()
repo_partition = os.path.realpath(repo_partition)
work_partition = getLocalWorkspace()
work_partition = os.path.realpath(work_partition)
folders_to_check = [repo_partition, work_partition]
home_dir = os.environ["HOME"]
to_remove = []
for partition in folders_to_check:
if not os.path.exists(partition):
if home_dir not in folders_to_check:
folders_to_check.append(home_dir)
to_remove.append(partition)
for folder in to_remove:
folders_to_check.remove(folder)
for data_partition in folders_to_check:
if fullpath(data_partition).find("/afs") == 0:
quota = subprocess.Popen(["fs", "quota", "%s" % data_partition], stdout=subprocess.PIPE)
output = quota.communicate()[0]
logger.debug("fs quota %s:\t%s" % (str(data_partition), str(output)))
else:
df = subprocess.Popen(["df", "-Pk", data_partition], stdout=subprocess.PIPE)
output = df.communicate()[0]
try:
quota_percent = output.split("%")[0]
if int(quota_percent) >= 95:
logger.warning(
"WARNING: You're running low on disk space, Ganga may stall on launch or fail to download job output"
)
logger.warning("WARNING: Please free some disk space on: %s" % str(data_partition))
except Exception as err:
logger.debug("Error checking disk partition: %s" % str(err))
return
def _check_inputs(self):
"""Checks the validity of user's entries for GaudiPython schema"""
# Always check for None OR empty
#logger.info("self.module: %s" % str(self.module))
if isType(self.module, str):
self.module = File(self.module)
if self.module.name == None:
raise ApplicationConfigurationError(
"Application Module not requested")
elif self.module.name == "":
raise ApplicationConfigurationError(
"Application Module not requested")
else:
# Always check we've been given a FILE!
self.module.name = fullpath(self.module.name)
if not os.path.isfile(self.module.name):
msg = 'Module file %s not found.' % self.module.name
raise ApplicationConfigurationError(msg)
def __setattr__(self, attr, value):
"""
This overloads the baseclass setter method and allows for dynamic evaluation of a parameter on assignment
Args:
attr (str): Name of the attribute which is being assigned for this class
value (unknown): The raw value which is being passed to this class for assigning to the attribute
"""
actual_value = value
if attr == 'directory':
if value:
actual_value = path.abspath(fullpath(expandfilename(value)))
elif attr == 'options':
if isinstance(value, str):
new_file = allComponentFilters['gangafiles'](value, None)
actual_value = [ new_file ]
elif isinstance(value, IGangaFile):
actual_value = [ value ]
elif not isinstance(value, (list, tuple, GangaList, type(None))):
logger.warning("Possibly setting wrong type for options: '%s'" % type(value))
super(GaudiExec, self).__setattr__(attr, actual_value)
def fill( self ):
hostname = commands.getoutput( "hostname -f" )
job = self._getParent()
tarFile = fullpath( os.path.join( job.outputdir, "%s.tar.gz" \
% self.tarfile ) )
downloadStartTime = "%.6f" % time.time()
statusOK = self.download( job = job, tarFile = tarFile )
downloadEndTime = "%.6f" % time.time()
unpackStartTime = "%.6f" % time.time()
if statusOK:
statusOK = self.unpack( job = job, tarFile = tarFile )
unpackEndTime = "%.6f" % time.time()
cleanupStartTime = "%.6f" % time.time()
if statusOK:
statusOK = self.cleanup( job = job )
cleanupEndTime = "%.6f" % time.time()
lineList = \
[
"Hostname: %s" % hostname,
"Download_start: %s" % downloadStartTime,
"Download_end: %s" % downloadEndTime,
"Unpack_start: %s" % unpackStartTime,
"Unpack_end: %s" % unpackEndTime,
"Cleanup_start: %s" % cleanupStartTime,
"Cleanup_end: %s" % cleanupEndTime,
]
outString = "\n".join( lineList )
outfile = open( os.path.join( job.outputdir, "retrieve.dat" ), "w" )
outfile.write( outString )
outfile.close()
return None
def requiresAfsToken():
# Were we executed from within an AFS folder
return fullpath(getLocalRoot(), True).find('/afs') == 0
import platform
import re
c = re.compile('\S+-redhat-(?P<ver>\S+)-\S+')
r = c.match(platform.platform())
if r and r.group('ver').split('.')[0] == '5':
platfstring = platf5
return platfstring[arch]
# guess defaults if not defined
if not _defaultExternalHome:
import os.path
import Ganga
from Ganga.Utility.files import fullpath
p = fullpath(Ganga.__file__)
for i in range(5):
p = os.path.dirname(p)
_defaultExternalHome = os.path.join(p, 'external')
if not _defaultPlatform:
_defaultPlatform = detectPlatform()
from Ganga.Utility.Setup import PackageSetup
# The setup object
setup = PackageSetup(_externalPackages)
def standardSetup(setup=setup):
""" Perform automatic initialization of the environment of the package.
The gangaDir argument is only used by the core package, other packages should have no arguments.
import platform
import re
c = re.compile('\S+-redhat-(?P<ver>\S+)-\S+')
r = c.match(platform.platform())
if r and r.group('ver').split('.')[0] == '5':
platfstring = platf5
return platfstring[arch]
# guess defaults if not defined
if not _defaultExternalHome:
import os.path
import Ganga
from Ganga.Utility.files import fullpath
p = fullpath(Ganga.__file__)
for i in range(5):
p = os.path.dirname(p)
_defaultExternalHome = os.path.join(p, 'external')
if not _defaultPlatform:
_defaultPlatform = detectPlatform()
from Ganga.Utility.Setup import PackageSetup
# The setup object
setup = PackageSetup(_externalPackages)
def standardSetup(setup=setup):
""" Perform automatic initialization of the environment of the package.
def requiresAfsToken():
return fullpath(getLocalRoot(), True).find('/afs') == 0
def get_user_dlls(appname,version,user_release_area,platform,shell):
user_ra = user_release_area
update_cmtproject_path(user_release_area)
full_user_ra = fullpath(user_ra) # expand any symbolic links
# Work our way through the CMTPATH until we find a cmt directory
if not shell.env.has_key('CMTPATH'): return [], [], []
projectdirs = shell.env['CMTPATH'].split(os.pathsep)
#appveruser = os.path.join(appname + '_' + version,'cmt')
appveruser = os.path.join(version,'cmt')
#appverrelease = os.path.join(appname.upper(),appname.upper() + '_' + version,
appverrelease = os.path.join('cmt')
for projectdir in projectdirs:
dir = fullpath(os.path.join(projectdir,appveruser))
logger.error('Looking for projectdir %s' % dir)
if os.path.exists(dir):
break
dir = fullpath(os.path.join(projectdir,appverrelease))
logger.error('Looking for projectdir %s' % dir)
if os.path.exists(dir):
break
logger.error('Using the CMT directory %s for identifying projects' % dir)
rc, showProj, m = shell.cmd1('cd ' + dir +';cmt show projects',
capture_stderr=True)
logger.error(showProj)
libs=[]
merged_pys = []
subdir_pys = {}
project_areas = []
py_project_areas = []
for line in showProj.split('\n'):
for entry in line.split():
if entry.startswith(user_ra) or entry.startswith(full_user_ra):
tmp = entry.rstrip('\)')
libpath = fullpath(os.path.join(tmp,'InstallArea',platform,'lib'))
logger.debug(libpath)
project_areas.append(libpath)
pypath = fullpath(os.path.join(tmp,'InstallArea','python'))
logger.debug(pypath)
py_project_areas.append(pypath)
pypath = fullpath(os.path.join(tmp,'InstallArea',platform,'python'))
logger.debug(pypath)
py_project_areas.append(pypath)
# savannah 47793 (remove multiple copies of the same areas)
project_areas = unique(project_areas)
py_project_areas = unique(py_project_areas)
ld_lib_path = []
if shell.env.has_key('LD_LIBRARY_PATH'):
ld_lib_path = shell.env['LD_LIBRARY_PATH'].split(':')
project_areas_dict = {}
for area in project_areas:
if area in ld_lib_path:
project_areas_dict[area] = ld_lib_path.index(area)
else:
project_areas_dict[area] = 666
from operator import itemgetter
sorted_project_areas = []
for item in sorted(project_areas_dict.items(),key=itemgetter(1)):
sorted_project_areas.append(item[0])
lib_names = []
for libpath in sorted_project_areas:
if os.path.exists(libpath):
for f in os.listdir(libpath):
if lib_names.count(f) > 0: continue
fpath = os.path.join(libpath,f)
if os.path.exists(fpath):
lib_names.append(f)
libs.append(fpath)
else:
logger.warning("File %s in %s does not exist. Skipping...",
str(f),str(libpath))
for pypath in py_project_areas:
if os.path.exists( pypath):
for f in os.listdir( pypath):
confDB_path = os.path.join( pypath, f)
if confDB_path.endswith( '.py'):
if os.path.exists( confDB_path):
merged_pys.append( confDB_path)
else:
logger.warning("File %s in %s does not exist. Skipping...",
str(f),str(confDB_path))
elif os.path.isdir(confDB_path):
pyfiles = []
for g in os.listdir(confDB_path):
file_path = os.path.join(confDB_path, g)
if (file_path.endswith('.py')):
if os.path.exists(file_path):
pyfiles.append(file_path)
else:
logger.warning("File %s in %s does not exist. Skipping...",
str(g),str(f))
subdir_pys[ f] = pyfiles
logger.debug("%s",pprint.pformat( libs))
logger.debug("%s",pprint.pformat( merged_pys))
logger.debug("%s",pprint.pformat( subdir_pys))
return libs, merged_pys, subdir_pys
def getLocalRoot():
from Ganga.Utility.files import fullpath
import Ganga.Core.FileWorkspace
return fullpath(Ganga.Core.FileWorkspace.gettop())
def requiresAfsToken():
return fullpath(getLocalRoot(), True).find('/afs') == 0
def get_user_dlls(appname, version, user_release_area, platform, env):
user_ra = user_release_area
update_project_path(user_release_area)
from Ganga.Utility.files import fullpath
full_user_ra = fullpath(user_ra) # expand any symbolic links
# Work our way through the CMTPROJECTPATH until we find a cmt directory
if 'CMTPROJECTPATH' not in env:
return [], [], []
projectdirs = env['CMTPROJECTPATH'].split(os.pathsep)
appveruser = os.path.join(appname + '_' + version, 'cmt')
appverrelease = os.path.join(appname.upper(),
appname.upper() + '_' + version, 'cmt')
for projectdir in projectdirs:
projectDir = fullpath(os.path.join(projectdir, appveruser))
logger.debug('Looking for projectdir %s' % projectDir)
if os.path.exists(projectDir):
break
projectDir = fullpath(os.path.join(projectdir, appverrelease))
logger.debug('Looking for projectdir %s' % projectDir)
if os.path.exists(projectDir):
break
logger.debug('Using the CMT directory %s for identifying projects' %
projectDir)
# rc, showProj, m = shell.cmd1('cd ' + projectDir +';cmt show projects',
# capture_stderr=True)
from GangaGaudi.Lib.Applications.GaudiUtils import shellEnv_cmd
rc, showProj, m = shellEnv_cmd('cmt show projects', env, projectDir)
logger.debug(showProj)
libs = []
merged_pys = []
subdir_pys = {}
project_areas = []
py_project_areas = []
for line in showProj.split('\n'):
for entry in line.split():
if entry.startswith(user_ra) or entry.startswith(full_user_ra):
tmp = entry.rstrip('\)')
libpath = fullpath(
os.path.join(tmp, 'InstallArea', platform, 'lib'))
logger.debug(libpath)
project_areas.append(libpath)
pypath = fullpath(os.path.join(tmp, 'InstallArea', 'python'))
logger.debug(pypath)
py_project_areas.append(pypath)
pypath = fullpath(
os.path.join(tmp, 'InstallArea', platform, 'python'))
logger.debug(pypath)
py_project_areas.append(pypath)
# savannah 47793 (remove multiple copies of the same areas)
from Ganga.Utility.util import unique
project_areas = unique(project_areas)
py_project_areas = unique(py_project_areas)
ld_lib_path = []
if 'LD_LIBRARY_PATH' in env:
ld_lib_path = env['LD_LIBRARY_PATH'].split(':')
project_areas_dict = {}
for area in project_areas:
if area in ld_lib_path:
project_areas_dict[area] = ld_lib_path.index(area)
else:
project_areas_dict[area] = 666
from operator import itemgetter
sorted_project_areas = []
for item in sorted(project_areas_dict.items(), key=itemgetter(1)):
sorted_project_areas.append(item[0])
lib_names = []
for libpath in sorted_project_areas:
if os.path.exists(libpath):
for f in os.listdir(libpath):
if lib_names.count(f) > 0:
continue
fpath = os.path.join(libpath, f)
if os.path.exists(fpath):
lib_names.append(f)
libs.append(fpath)
else:
logger.warning("File %s in %s does not exist. Skipping...",
str(f), str(libpath))
for pypath in py_project_areas:
if os.path.exists(pypath):
from GangaGaudi.Lib.Applications.GaudiUtils import pyFileCollector
from Ganga.Utility.Config import getConfig
configGaudi = getConfig('GAUDI')
pyFileCollector(pypath, merged_pys, subdir_pys,
configGaudi['pyFileCollectionDepth'])
import pprint
logger.debug("%s", pprint.pformat(libs))
logger.debug("%s", pprint.pformat(merged_pys))
logger.debug("%s", pprint.pformat(subdir_pys))
return libs, merged_pys, subdir_pys
def requiresAfsToken():
from Ganga.Utility.files import fullpath
return fullpath(getLocalRoot()).find("/afs") == 0
def requiresAfsToken():
# Were we executed from within an AFS folder
return fullpath(getLocalRoot(), True).find('/afs') == 0
def get_user_dlls(appname, version, user_release_area, platform, env):
user_ra = user_release_area
update_project_path(user_release_area)
from Ganga.Utility.files import fullpath
full_user_ra = fullpath(user_ra) # expand any symbolic links
# Work our way through the CMTPROJECTPATH until we find a cmt directory
if 'CMTPROJECTPATH' not in env:
return [], [], []
projectdirs = env['CMTPROJECTPATH'].split(os.pathsep)
appveruser = os.path.join(appname + '_' + version, 'cmt')
appverrelease = os.path.join(
appname.upper(), appname.upper() + '_' + version, 'cmt')
for projectdir in projectdirs:
projectDir = fullpath(os.path.join(projectdir, appveruser))
logger.debug('Looking for projectdir %s' % projectDir)
if os.path.exists(projectDir):
break
projectDir = fullpath(os.path.join(projectdir, appverrelease))
logger.debug('Looking for projectdir %s' % projectDir)
if os.path.exists(projectDir):
break
logger.debug('Using the CMT directory %s for identifying projects' % projectDir)
# rc, showProj, m = shell.cmd1('cd ' + projectDir +';cmt show projects',
# capture_stderr=True)
from GangaGaudi.Lib.Applications.GaudiUtils import shellEnv_cmd
rc, showProj, m = shellEnv_cmd('cmt show projects', env, projectDir)
logger.debug(showProj)
libs = []
merged_pys = []
subdir_pys = {}
project_areas = []
py_project_areas = []
for line in showProj.split('\n'):
for entry in line.split():
if entry.startswith(user_ra) or entry.startswith(full_user_ra):
tmp = entry.rstrip('\)')
libpath = fullpath(os.path.join(tmp, 'InstallArea', platform, 'lib'))
logger.debug(libpath)
project_areas.append(libpath)
pypath = fullpath(os.path.join(tmp, 'InstallArea', 'python'))
logger.debug(pypath)
py_project_areas.append(pypath)
pypath = fullpath(os.path.join(tmp, 'InstallArea', platform, 'python'))
logger.debug(pypath)
py_project_areas.append(pypath)
# savannah 47793 (remove multiple copies of the same areas)
from Ganga.Utility.util import unique
project_areas = unique(project_areas)
py_project_areas = unique(py_project_areas)
ld_lib_path = []
if 'LD_LIBRARY_PATH' in env:
ld_lib_path = env['LD_LIBRARY_PATH'].split(':')
project_areas_dict = {}
for area in project_areas:
if area in ld_lib_path:
project_areas_dict[area] = ld_lib_path.index(area)
else:
project_areas_dict[area] = 666
from operator import itemgetter
sorted_project_areas = []
for item in sorted(project_areas_dict.items(), key=itemgetter(1)):
sorted_project_areas.append(item[0])
lib_names = []
for libpath in sorted_project_areas:
if os.path.exists(libpath):
for f in os.listdir(libpath):
if lib_names.count(f) > 0:
continue
fpath = os.path.join(libpath, f)
if os.path.exists(fpath):
lib_names.append(f)
libs.append(fpath)
else:
logger.warning("File %s in %s does not exist. Skipping...", str(f), str(libpath))
for pypath in py_project_areas:
if os.path.exists(pypath):
from GangaGaudi.Lib.Applications.GaudiUtils import pyFileCollector
from Ganga.Utility.Config import getConfig
configGaudi = getConfig('GAUDI')
pyFileCollector(
pypath, merged_pys, subdir_pys, configGaudi['pyFileCollectionDepth'])
import pprint
logger.debug("%s", pprint.pformat(libs))
logger.debug("%s", pprint.pformat(merged_pys))
logger.debug("%s", pprint.pformat(subdir_pys))
return libs, merged_pys, subdir_pys