def load_associations():
config = makeConfig(
"File_Associations", 'Default associations between file types and file-viewing commands. The name identifies the extension and the value the commans. New extensions can be added. A single & after the command indicates that the process will be started in the background. A && after the command indicates that a new terminal will be opened and the command executed in that terminal.', is_open=True)
config.addOption("newterm_command", "xterm",
'Command for opening a new terminal (xterm, gnome-terminal, ...')
config.addOption("newterm_exeopt", "-e",
'Option to give to a new terminal to tell it to execute a command.')
config.addOption(
"listing_command", "ls -ltr", 'Command for listing the content of a directory')
config.addOption('fallback_command', 'less',
'Default command to use if there is no association with the file type')
config.addOption('htm', 'firefox &', 'Command for viewing html files.')
config.addOption('html', 'firefox &', 'Command for viewing html files.')
config.addOption('root', 'root.exe &&', 'Command for opening ROOT files.')
config.addOption('tar', 'file-roller &', 'Command for opening tar files.')
config.addOption('tgz', 'file-roller &', 'Command for opening tar files.')
from Ganga.Utility.Config import makeConfig
# test configuration properties
test_config = makeConfig('TestConfig','testing stuff')
test_config.addOption('None_OPT', None, '')
test_config.addOption('Int_OPT', 1, '')
test_config.addOption('List_OPT', [1,2,3], '')
test_config.addOption('String_OPT' ,'dupa', '')
# there is also an Undefine_OPT which will be used in the test case
from Ganga.Utility.Config import getConfig
outputconfig = None
if outputconfig == None:
from Ganga.Utility.Config import makeConfig
outputconfig = makeConfig(
"Output", "configuration section for postprocessing the output")
def getSharedPath():
# Required to be initialized for ShareDir object
from Ganga.Utility.files import expandfilename
import os.path
Conf_config = getConfig('Configuration')
root_default = os.path.join(expandfilename(Conf_config['gangadir']), 'shared', Conf_config['user'])
return root_default
def Configure():
outputconfig.addOption('AutoRemoveFilesWithJob', False,
'if True, each outputfile of type in list AutoRemoveFileTypes will be removed when the job is')
outputconfig.addOption('AutoRemoveFileTypes', [
'DiracFile'], 'List of outputfile types that will be auto removed when job is removed if AutoRemoveFilesWithJob is True')
outputconfig.addOption('PostProcessLocationsFileName', '__postprocesslocations__',
'name of the file that will contain the locations of the uploaded from the WN files')
outputconfig.addOption('FailJobIfNoOutputMatched', True,
'if True, a job will be marked failed if output is asked for but not found.')
"afs": "/afs/cern.ch/sw/lcg/releases/LCG_79",
"cvmfs": "/cvmfs/lhcb.cern.ch/lib/lcg/releases/LCG_79",
}
## CAUTION This could be sensitive to mixed AFS/CVMFS running but I doubt this setup is common or likely
myCurrentPath = os.path.abspath(inspect.getfile(inspect.currentframe()))
if myCurrentPath[:4].upper() == "/AFS":
return lcg_release_areas["afs"]
elif myCurrentPath[:6].upper() == "/CVMFS":
return lcg_release_areas["cvmfs"]
else:
return ""
config = makeConfig("ROOT", "Options for Root backend")
## Not needed when we can't do option substitution internally but support it at the .gangarc level!!!!! 27-09-2015 rcurrie
# config.addOption('lcgpath', getLCGRootPath(), 'Path of the LCG release that the ROOT project and it\'s externals are taken from')
config.addOption("arch", "x86_64-slc6-gcc48-opt", "Architecture of ROOT")
## Auto-Interporatation doesn't appear to work when setting the default value
# config.addOption('location', '${lcgpath}/ROOT/${version}/${arch}/', 'Location of ROOT')
config.addOption("location", "%s/ROOT/6.04.02/x86_64-slc6-gcc48-opt" % getLCGRootPath(), "Location of ROOT")
config.addOption("path", "", "Set to a specific ROOT version. Will override other options.")
## Doesn't appear to work see above ^^^
# config.addOption('pythonhome', '${lcgpath}/Python/${pythonversion}/${arch}/','Location of the python used for execution of PyROOT script')
config.addOption(
"pythonhome",
"%s/Python/2.7.9.p1/x86_64-slc6-gcc48-opt" % getLCGRootPath(),
"Location of the python used for execution of PyROOT script",
)
config.addOption("pythonversion", "2.7.9.p1", "Version number of python used for execution python ROOT script")
from Ganga.Utility.Config import makeConfig
from Ganga.Utility.logging import getLogger
from Ganga.Utility.GridShell import getShell
from Ganga.GPIDev.Credentials.ICredential import ICredential
logger = getLogger()
logger.critical('LCG Grid Simulator ENABLED')
##########################################################################
# GRID SIMULATOR
##########################################################################
config = makeConfig('GridSimulator', 'Grid Simulator configuration parameters')
config.addOption('submit_time', 'random.uniform(1,10)',
'python expression which returns the time it takes (in seconds) to complete the Grid.submit() command (also for subjob in bulk emulation)')
config.addOption(
'submit_failure_rate', 0.0, 'probability that the Grid.submit() method fails')
config.addOption('cancel_time', 'random.uniform(1,5)',
'python expression which returns the time it takes (in seconds) to complete the Grid.cancel() command (also for subjob in bulk emulation)')
config.addOption(
'cancel_failure_rate', 0.0, 'probability that the Grid.cancel() method fails')
config.addOption('status_time', 'random.uniform(1,5)',
'python expression which returns the time it takes (in seconds) to complete the status command (also for subjob in bulk emulation)')
config.addOption('get_output_time', 'random.uniform(1,5)',
else:
preposition = "on"
if jobDict[id].backend.status:
backendStatus = "".join([" (", jobDict[id].backend.status, ") "])
else:
backendStatus = ""
logger.info(
colour + "Job %s %s%s %s %s - %s" + fx.normal,
jobDict[id].fqid,
status,
backendStatus,
preposition,
hostInfo,
time.strftime("%c"),
)
return None
updateMonitoringInformation = staticmethod(updateMonitoringInformation)
# _________________________________________________________________________
from Ganga.Utility.Config import makeConfig
config = makeConfig("Condor", "Settings for Condor Batch system")
config.addOption("query_global_queues", True, "Query global condor queues, i.e. use '-global' flag")
from Ganga.Utility.Config import makeConfig
# test configuration properties
test_config = makeConfig('TestConfig', 'testing stuff')
test_config.addOption('None_OPT', None, '')
test_config.addOption('Int_OPT', 1, '')
test_config.addOption('List_OPT', [1, 2, 3], '')
test_config.addOption('String_OPT', 'dupa', '')
# there is also an Undefine_OPT which will be used in the test case
import re
import inspect
import GangaPlotHelper
## require matplotlib to produce statistic plots
from pylab import *
import sys
from functools import reduce
from Ganga.Utility.Config import makeConfig
from Ganga.Utility.logging import getLogger
from Ganga.Utility.util import isStringLike, canLoopOver
logger = getLogger()
config = makeConfig('GangaPlotter', 'GangaPlotter configuration parameters')
NOT_APPLICABLE = '*N/A*'
class GetJobAttrException(Exception):
'''exception for parser error'''
def __init__(self, msg):
self.msg = msg
def __str__(self):
return self.msg
## this function could be put in Ganga.Utility.util
def isNumeric(myValue):
## Config options
from Ganga.Utility.Config import makeConfig
from Ganga.Utility.logging import getLogger
logger = getLogger()
# -------------------------------------------------
# Panda Options
config = makeConfig('Panda','Panda backend configuration parameters')
config.addOption( 'prodSourceLabelBuild', 'panda', 'prodSourceLabelBuild')
config.addOption( 'prodSourceLabelRun', 'user', 'prodSourceLabelRun')
config.addOption( 'assignedPriorityBuild', 2000, 'assignedPriorityBuild' )
config.addOption( 'assignedPriorityRun', 1000, 'assignedPriorityRun' )
config.addOption( 'processingType', 'ganga', 'processingType' )
config.addOption( 'specialHandling', '', 'specialHandling - Expert only.' )
config.addOption( 'enableDownloadLogs', False , 'enableDownloadLogs' )
config.addOption( 'trustIS', True , 'Trust the Information System' )
config.addOption( 'serverMaxJobs', 5000 , 'Maximum number of subjobs to send to the Panda server' )
config.addOption( 'chirpconfig', '' , 'Configuration string for chirp data output, e.g. "chirp^etpgrid01.garching.physik.uni-muenchen.de^/tanyasandoval^-d chirp" ' )
config.addOption( 'chirpserver', '' , 'Configuration string for the chirp server, e.g. "voatlas92.cern.ch". If this variable is set config.Panda.chirpconfig is filled and chirp output will be enabled.' )
config.addOption( 'siteType', 'analysis' , 'Expert only.' )
config.addOption( 'baseURL', '' , 'Expert only.' )
config.addOption( 'baseURLSSL', '' , 'Expert only.' )
config.addOption( 'AllowDirectSubmission', False, 'Deprecated. Please use the Jedi backend' )
config.addOption('AGISJSONFile', '/tmp/agis_pandaresources.json', 'Local AGIS JSON file for queue -> site mapping')
# -------------------------------------------------
# Jedi Options
config = makeConfig('Jedi','Jedi backend configuration parameters')
config.addOption( 'prodSourceLabelBuild', 'panda', 'prodSourceLabelBuild')
config.addOption( 'prodSourceLabelRun', 'user', 'prodSourceLabelRun')
config.addOption( 'assignedPriorityBuild', 2000, 'assignedPriorityBuild' )
import os
#from multiprocessing import cpu_count
from Ganga.Utility.Config import makeConfig, getConfig
from Ganga.Utility.logging import getLogger
from Ganga.Utility.Config.Config import _after_bootstrap
from Ganga.GPIDev.Credentials.CredentialStore import credential_store
logger = getLogger()
if not _after_bootstrap:
configDirac = makeConfig('DIRAC', 'Parameters for DIRAC')
config = getConfig('Configuration')
# Set default values for the Dirac section.
# configDirac.addOption('ShowDIRACstdout', False,
# 'Display DIRAC API stdout to the screen in Ganga?')
configDirac.addOption('Timeout', 1000,
'Default timeout (seconds) for Dirac commands')
configDirac.addOption(
'splitFilesChunks', 5000,
'when splitting datasets, pre split into chunks of this int')
diracenv = ""
if "GANGADIRACENVIRONMENT" in os.environ:
diracenv = os.environ["GANGADIRACENVIRONMENT"]
configDirac.addOption(
'DiracEnvFile', diracenv,
# File: GangaND280/__init__.py
from Ganga.Utility.Config import makeConfig
from Ganga.Utility.Config.Config import _after_bootstrap
if not _after_bootstrap:
configND280 = makeConfig(
'ND280', 'Configuration parameters for the ND280 Ganga plugin')
configND280.addOption(
'ND280DCacheDatasetCommandStr', {
'TRIUMF':
'curl --compressed -sk --header "Accept: text/plain" https://user:[email protected]/full/path/to/https/nd280data/%s'
}, 'How to get the list of files on the DCache server')
configND280.addOption(
'ND280DCacheDatasetFileBasePath',
{'TRIUMF': 'dcap://t2ksrm.nd280.org/full/path/to/dcap/nd280data/'},
'How to access files on the DCache sever')
def loadPlugins(config={}):
import ND280Dataset
import ND280Splitter
import ND280Control
import ND280Executable
import ND280RecoValidation
import ND280Skimmer
import Highland
import Tasks
import ND280TPCGasInteractions
import ND280Checkers
# for cmsRun
# Andres Osorio
###########################################################################
"""
File: CMSSWscript.py
Purpose: Write a script setting up the cmssw release area and environment
for cmsRun
"""
#__author__ = 'Andres Osorio'
#__date__ = 'February 2009'
#__revision__ = 0.1
#................
#... just for testing -- boostrap.py provides the following at runtime:
from Ganga.Utility.Config import Config, makeConfig
shellconfig = makeConfig( "Shell", "configuration parameters for internal Shell utility.")
comment = 'list of env variables not inherited in Shell environment'
shellconfig.addOption('IgnoredVars',['_','SHVL','PWD'], comment)
#................
from Ganga.GPI import *
from Ganga.Utility.Shell import Shell
import Ganga.Utility.logging
import os
import sys
import time
import types
import warnings
import tempfile
import shutil
#################################################################################
# File: associations.py
# Author: K. Harrison
# Created: 061123
#
"""
Module defining default associations
between file types and file-viewing commands.
Values are stored in a configuration object,
and modifications/additions can be made
in the [File_Associations] section of a
configuration file.
"""
__author__ = "K.Harrison <*****@*****.**>"
__date__ = "23 November 2006"
__version__ = "1.0"
from Ganga.Utility.Config import makeConfig
config = makeConfig( "File_Associations",'Default associations between file types and file-viewing commands. The name identifies the extension and the value the commans. New extensions can be added. A single & after the command indicates that the process will be started in the background. A && after the command indicates that a new terminal will be opened and the command executed in that terminal.' , is_open=True)
config.addOption("newterm_command","xterm",'Command for opening a new terminal (xterm, gnome-terminal, ...')
config.addOption("newterm_exeopt","-e",'Option to give to a new terminal to tell it to execute a command.')
config.addOption("listing_command","ls -ltr",'Command for listing the content of a directory')
config.addOption('fallback_command','less','Default command to use if there is no association with the file type')
config.addOption('htm','firefox &','Command for viewing html files.')
config.addOption('html','firefox &','Command for viewing html files.')
config.addOption('root','root.exe &&','Command for opening ROOT files.')
config.addOption('tar','file-roller &','Command for opening tar files.')
config.addOption('tgz','file-roller &','Command for opening tar files.')
# File: GangaAtlas/__init__.py
## Config options
from Ganga.Utility.Config import makeConfig, ConfigError, getConfig
import os
from Ganga.Utility.Config.Config import _after_bootstrap
from Ganga.Utility.logging import getLogger
logger = getLogger()
if not _after_bootstrap:
# -------------------------------------------------
# Athena Options
config = makeConfig('Athena', 'Athena configuration parameters')
config.addOption(
'LCGOutputLocation',
'srm://srm-atlas.cern.ch/castor/cern.ch/grid/atlas/scratch/%s/ganga' %
os.environ['USER'], 'FIXME')
config.addOption(
'LocalOutputLocation',
'/castor/cern.ch/atlas/scratch/%s/ganga' % os.environ['USER'], 'FIXME')
config.addOption(
'IndividualSubjobDirsForLocalOutput', False,
'When copying local output, should dir structure be jid.sid (False) or jid/sid (True)'
)
config.addOption(
'SingleDirForLocalOutput', False,
'When copying local output, only a single dirs used for output and output filenames are changed with jid.sid'
)
config.addOption(
import pytest
try:
import unittest.mock as mock
except ImportError:
import mock
from Ganga.Utility.Config import getConfig, makeConfig
makeConfig('defaults_DiracProxy', '')
getConfig('defaults_DiracProxy').addOption('group', 'some_group', '')
from GangaDirac.Lib.Credentials.DiracProxy import DiracProxy, DiracProxyInfo
getConfig('defaults_DiracProxy').setSessionValue('group', 'some_group')
class FakeShell(object):
"""
A mock version of Shell which allows to customise the return values
Examples:
>>> s = FakeShell()
>>> assert s.cmd1.call_count == 0
>>> assert s.cmd1('foo') == (0, '', '')
>>> assert s.cmd1.call_count == 1
>>> assert s.cmd1('something -vo') == (0, 'some_group', '')
>>> assert s.cmd1.call_count == 2
"""
vo = 'some_group'
timeleft = 100
def __init__(self):
self.env = {}
self.cmd1 = mock.Mock(wraps=self._cmd1)
else:
colour = fg.magenta
if "submitted" == status:
preposition = "to"
else:
preposition = "on"
if jobDict[id].backend.status:
backendStatus = "".join\
([" (", jobDict[id].backend.status, ") "])
else:
backendStatus = ""
logger.info(colour + 'Job %s %s%s %s %s - %s' + fx.normal,
jobDict[
id].fqid, status, backendStatus, preposition, hostInfo,
time.strftime('%c'))
return None
updateMonitoringInformation = \
staticmethod(updateMonitoringInformation)
#_________________________________________________________________________
from Ganga.Utility.Config import makeConfig
config = makeConfig('Condor', 'Settings for Condor Batch system')
config.addOption('query_global_queues', True,
"Query global condor queues, i.e. use '-global' flag")
from Ganga.GPIDev.Base import GangaObject
from Ganga.GPIDev.Adapters.IApplication import IApplication
from Ganga.Core.GangaRepository import getRegistry
from Ganga.GPIDev.Base.Proxy import GPIProxyObjectFactory, isType
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList
from Ganga.GPIDev.Lib.File import *
import os, shutil
from Ganga.GPIDev.Schema import *
import Ganga.Utility.logging
logger = Ganga.Utility.logging.getLogger()
from Ganga.Utility.Config import makeConfig, ConfigError, getConfig
from Ganga.Utility.files import expandfilename
config = makeConfig('Preparable', 'Parameters for preparable applications')
config.addOption('unprepare_on_copy', False, 'Unprepare a prepared application when it is copied')
shared_path = os.path.join(expandfilename(getConfig('Configuration')['gangadir']),'shared',getConfig('Configuration')['user'])
class IPrepareApp(IApplication):
"""
Base class for all applications which can be placed into a prepared\
state.
"""
_schema = Schema(Version(0,0), { 'hash': SimpleItem(defvalue=None, typelist=['type(None)', 'str'],hidden=1)} )
_category='applications'
_name = 'PrepareApp'
_hidden = 1
# def _readonly(self):
# """An application is read-only once it has been prepared."""
import re
import inspect
import GangaPlotHelper
## require matplotlib to produce statistic plots
from pylab import *
import sys
from functools import reduce
from Ganga.Utility.Config import makeConfig
from Ganga.Utility.logging import getLogger
from Ganga.Utility.util import isStringLike, canLoopOver
logger = getLogger()
config = makeConfig('GangaPlotter','GangaPlotter configuration parameters')
NOT_APPLICABLE = '*N/A*'
class GetJobAttrException(Exception):
'''exception for parser error'''
def __init__(self,msg):
self.msg = msg
def __str__(self):
return self.msg
## this function could be put in Ganga.Utility.util
def isNumeric(myValue):
'''checks via type() if myVar is numeric or not'''
# File: GangaAtlas/__init__.py
## Config options
from Ganga.Utility.Config import makeConfig, getConfig
import os
from Ganga.Utility.logging import getLogger
logger = getLogger()
# -------------------------------------------------
# Athena Options
config = makeConfig('Athena','Athena configuration parameters')
config.addOption('LCGOutputLocation', 'srm://srm-atlas.cern.ch/castor/cern.ch/grid/atlas/scratch/%s/ganga' % os.environ['USER'], 'FIXME')
config.addOption('LocalOutputLocation', '/castor/cern.ch/atlas/scratch/%s/ganga' % os.environ['USER'], 'FIXME')
config.addOption('IndividualSubjobDirsForLocalOutput', False, 'When copying local output, should dir structure be jid.sid (False) or jid/sid (True)' )
config.addOption('SingleDirForLocalOutput', False, 'When copying local output, only a single dirs used for output and output filenames are changed with jid.sid' )
config.addOption('NoSubDirsAtAllForLocalOutput', False, 'When copying local output, all output is copied to the given output location with no subdirs created' )
config.addOption('ATLAS_SOFTWARE', '/cvmfs/atlas.cern.ch/repo/sw/software/', 'FIXME')
config.addOption('PRODUCTION_ARCHIVE_BASEURL', 'http://atlas-computing.web.cern.ch/atlas-computing/links/kitsDirectory/Production/kits/', 'FIXME')
config.addOption('ExcludedSites', '' , 'FIXME')
config.addOption('CMTHOME', os.path.join(os.environ['HOME'],'cmthome') , 'The path in which the cmtsetup magic function will look up the setup.sh for CMT environment setup')
config.addOption('CMTCONFIG', 'i686-slc5-gcc43-opt', 'Default value to be used as CMTCONFIG environment setup value (LCG/Batch backend)')
config.addOption('CMTCONFIG_LIST', [ 'i686-slc4-gcc34-opt', 'i686-slc5-gcc43-opt' ], 'Allowed values for CMTCONFIG environment setup value (LCG/Batch backend)')
config.addOption('MaxJobsAthenaSplitterJobLCG', 1000 , 'Number of maximum jobs allowed for job splitting with the AthenaSplitterJob and the LCG backend')
config.addOption('DCACHE_RA_BUFFER', 32768 , 'Size of the dCache read ahead buffer used for dcap input file reading')
config.addOption('ENABLE_DQ2COPY', False , 'Enable DQ2_COPY input workflow on LCG backend')
config.addOption('ENABLE_SGE_DQ2JOBSPLITTER', False , 'Enable DQ2JobSplitter for SGE backend')
config.addOption('ENABLE_SGE_FILESTAGER', False , 'Enable FILE_STAGER input access mode for SGE backend')
config.addOption('EXE_MAXFILESIZE', 1024*1024 , 'Athena.exetype=EXE jobs: Maximum size of files to be sent to WNs (default 1024*1024B)')
config.addOption('dereferenceSymLinks', False , 'Set to True to dereference symlinks in the sources area. E.g. if src is a symlink, the target will be copied into the sources archive.')
config.addOption('MaxJobsDQ2JobSplitter', 5000, 'Maximum number of allowed subjobs of DQ2JobSplitter')
config.addOption('MaxFilesPandaDQ2JobSplitter', 5000, 'Maximum number of allowed subjobs of DQ2JobSplitter')
## Config options
from Ganga.Utility.Config import makeConfig
from Ganga.Utility.logging import getLogger
logger = getLogger()
# -------------------------------------------------
# Panda Options
config = makeConfig('Panda', 'Panda backend configuration parameters')
config.addOption('prodSourceLabelBuild', 'panda', 'prodSourceLabelBuild')
config.addOption('prodSourceLabelRun', 'user', 'prodSourceLabelRun')
config.addOption('assignedPriorityBuild', 2000, 'assignedPriorityBuild')
config.addOption('assignedPriorityRun', 1000, 'assignedPriorityRun')
config.addOption('processingType', 'ganga', 'processingType')
config.addOption('specialHandling', '', 'specialHandling - Expert only.')
config.addOption('enableDownloadLogs', False, 'enableDownloadLogs')
config.addOption('trustIS', True, 'Trust the Information System')
config.addOption('serverMaxJobs', 5000,
'Maximum number of subjobs to send to the Panda server')
config.addOption(
'chirpconfig', '',
'Configuration string for chirp data output, e.g. "chirp^etpgrid01.garching.physik.uni-muenchen.de^/tanyasandoval^-d chirp" '
)
config.addOption(
'chirpserver', '',
'Configuration string for the chirp server, e.g. "voatlas92.cern.ch". If this variable is set config.Panda.chirpconfig is filled and chirp output will be enabled.'
)
config.addOption('siteType', 'analysis', 'Expert only.')
config.addOption('baseURL', '', 'Expert only.')
config.addOption('baseURLSSL', '', 'Expert only.')
config.addOption('AllowDirectSubmission', False,
'Deprecated. Please use the Jedi backend')
# retrieve and cache monitoring classes by name
_mon_classes = {}
from Ganga.Core.exceptions import GangaException
from Ganga.GPIDev.Adapters.IMonitoringService import IMonitoringService
from Ganga.Lib.MonitoringServices.Composite import CompositeMonitoringService
from Ganga.Utility.Config import makeConfig
c = makeConfig('MonitoringServices', """External monitoring systems are used
to follow the submission and execution of jobs. Each entry in this section
defines a monitoring plugin used for a particular combination of application
and backend. Asterisks may be used to specify any application or any
backend. The configuration entry syntax:
ApplicationName/BackendName = dot.path.to.monitoring.plugin.class.
Example: DummyMS plugin will be used to track executables run on all backends:
Executable/* = Ganga.Lib.MonitoringServices.DummyMS.DummyMS
""",is_open=True)
class MonitoringServiceError(GangaException):
pass
def getMonitoringClass(mclassname):
"""
Return the class object based on the class name string provided as input
If the class object is already available in cache the saved value is returned
import pytest
try:
import unittest.mock as mock
except ImportError:
import mock
from Ganga.Utility.Config import getConfig, makeConfig
makeConfig('defaults_DiracProxy', '')
getConfig('defaults_DiracProxy').addOption('group', 'some_group', '')
from GangaDirac.Lib.Credentials.DiracProxy import DiracProxy, DiracProxyInfo
getConfig('defaults_DiracProxy').setSessionValue('group', 'some_group')
class FakeShell(object):
"""
A mock version of Shell which allows to customise the return values
Examples:
>>> s = FakeShell()
>>> assert s.cmd1.call_count == 0
>>> assert s.cmd1('foo') == (0, '', '')
>>> assert s.cmd1.call_count == 1
>>> assert s.cmd1('something -vo') == (0, 'some_group', '')
>>> assert s.cmd1.call_count == 2
"""
vo = 'some_group'
timeleft = 100
def __init__(self):
self.env = {}
self.cmd1 = mock.Mock(wraps=self._cmd1)
self.system = mock.Mock(wraps=self._system)
from Ganga import GPI
from Ganga.GPIDev.Base import GangaObject
from Ganga.GPIDev.Base.Proxy import addProxy, stripProxy
from Ganga.GPIDev.Schema import *
from Ganga.Utility.Config import makeConfig, ConfigError, getConfig
from Ganga.Utility.logging import getLogger
config = makeConfig('Tasks', 'Tasks configuration options')
logger = getLogger()
# Helper function for singular/plural
def say(number, unit):
if number == 1:
return "one %s" % (unit)
else:
return "%s %ss" % (number, unit)
# Text colouring
# http://everything2.com/e2node/ANSI%2520color%2520codes
from Ganga.Utility.ColourText import ANSIMarkup, Effects
markup = ANSIMarkup()
fx = Effects()
cols = {"black":(0,0),"red":(0,1),"green":(0,2),"orange":(0,3),"blue":(0,4),"magenta":(0,5),"cyan":(0,6),"lgray":(0,7),
"dgray":(6,0),"lred":(6,1),"lgreen":(6,2),"yellow":(6,3),"lblue":(6,4),"pink":(6,5),"lcyan":(6,6),"white":(6,7)}
def col(f,b):
return '\033[%i%i;%i%im' % (4+cols[b][0],cols[b][1],3+cols[f][0],cols[f][1])
def fgcol(f):
return '\033[%i%im' % (3+cols[f][0],cols[f][1])
# Status and overview colours
status_colours = {
__version__ = "1.0"
__author__="Adrian.Muraru[at]cern[dot]ch"
import os
import sys
## Configuration
from Ganga.Utility.Config import getConfig, makeConfig
from Ganga.Utility.files import previous_dir
from Ganga.Utility.logging import getLogger
myFullPath = os.path.abspath(os.path.dirname(__file__))
gangaReleaseTopDir = previous_dir(myFullPath,3)
# testing framework configuration properties
myConfig = makeConfig('TestingFramework','Configuration section for internal testing framework')
#Release testing mode:
# - xml/html reports are generated
# - coverage analysis tool enabled
myConfig.addOption('ReleaseTesting', False, '')
#enable/disable test-runner
myConfig.addOption('EnableTestRunner', True, 'enable/disable test-runner')
#enable/disable html reporter
myConfig.addOption('EnableHTMLReporter', False, 'enable/disable html reporter')
#enable/disable xml differencer
myConfig.addOption('EnableXMLDifferencer','False', 'enable/disable xml differencer')
#search for local tests lookup
myConfig.addOption('SearchLocalTests', True, 'search for local tests lookup')
#search for tests packaged in release (PACKAGE/test dir) lookup
import os
#from multiprocessing import cpu_count
from Ganga.Utility.Config import makeConfig, getConfig
from Ganga.Utility.logging import getLogger
from Ganga.Utility.Config.Config import _after_bootstrap
from Ganga.GPIDev.Credentials.CredentialStore import credential_store
logger = getLogger()
if not _after_bootstrap:
configDirac = makeConfig('DIRAC', 'Parameters for DIRAC')
config = getConfig('Configuration')
# Set default values for the Dirac section.
# configDirac.addOption('ShowDIRACstdout', False,
# 'Display DIRAC API stdout to the screen in Ganga?')
configDirac.addOption('Timeout', 1000,
'Default timeout (seconds) for Dirac commands')
configDirac.addOption('splitFilesChunks', 5000,
'when splitting datasets, pre split into chunks of this int')
diracenv = ""
if "GANGADIRACENVIRONMENT" in os.environ:
diracenv = os.environ["GANGADIRACENVIRONMENT"]
configDirac.addOption('DiracEnvFile', diracenv, 'DEPRECATED. Ganga environment file for DIRAC environment (do not change unless you are sure you know what you are doing).')
configDirac.addOption('DiracEnvSource', None, 'File to be sourced to provide the DIRAC environment. E.g. /cvmfs/ganga.cern.ch/dirac_ui/bashrc')
# File: GangaAtlas/__init__.py
## Config options
from Ganga.Utility.Config import makeConfig, ConfigError, getConfig
import os
# -------------------------------------------------
# Athena Options
config = makeConfig("Athena", "Athena configuration parameters")
config.addOption(
"LCGOutputLocation",
"srm://srm-atlas.cern.ch/castor/cern.ch/grid/atlas/scratch/%s/ganga" % os.environ["USER"],
"FIXME",
)
config.addOption("LocalOutputLocation", "/castor/cern.ch/atlas/scratch/%s/ganga" % os.environ["USER"], "FIXME")
config.addOption(
"IndividualSubjobDirsForLocalOutput",
False,
"When copying local output, should dir structure be jid.sid (False) or jid/sid (True)",
)
config.addOption(
"SingleDirForLocalOutput",
False,
"When copying local output, only a single dirs used for output and output filenames are changed with jid.sid",
)
config.addOption(
"NoSubDirsAtAllForLocalOutput",
False,
"When copying local output, all output is copied to the given output location with no subdirs created",
)
config.addOption("ATLAS_SOFTWARE", "/afs/cern.ch/project/gd/apps/atlas/slc3/software", "FIXME")
from Ganga.Utility.util import hostname
hostname = hostname()
except Exception as x: # fixme: use OSError instead?
hostname = 'localhost'
# ------------------------------------------------
# Setup all configs for this module
from Ganga.Utility.Config import makeConfig, getConfig, expandvars
# ------------------------------------------------
# Logging
log_config = makeConfig("Logging", """control the messages printed by Ganga
The settings are applied hierarchically to the loggers. Ganga is the name of the top-level logger which
applies by default to all Ganga.* packages unless overriden in sub-packages.
You may define new loggers in this section.
The log level may be one of: CRITICAL ERROR WARNING INFO DEBUG
""", is_open=True)
# FIXME: Ganga WARNING should be turned into INFO level when the messages
# are reviewed in all the code
log_config.addOption('Ganga', "INFO", "top-level logger")
log_config.addOption('Ganga.Runtime.bootstrap', "INFO", 'FIXME')
log_config.addOption('Ganga.GPIDev', "INFO", "logger of Ganga.GPIDev.* packages")
log_config.addOption('Ganga.Utility.logging', "WARNING", "logger of the Ganga logging package itself (use with care!)")
log_config.addOption('_format', "NORMAL", "format of logging messages: TERSE,NORMAL,VERBOSE,DEBUG")
log_config.addOption('_colour', True, "enable ASCII colour formatting of messages e.g. errors in red")
log_config.addOption('_logfile', "~/.ganga.log", "location of the logfile")
log_config.addOption('_logfile_size', 100000,
"the size of the logfile (in bytes), the rotating log will never exceed this file size") # 100 K
log_config.addOption('_interactive_cache', True,
from Ganga.GPIDev.Lib.File import File, ShareDir
from Ganga.GPIDev.Base.Proxy import GPIProxyObjectFactory
from Ganga.Core.GangaRepository import getRegistry
from Ganga.GPIDev.Base.Proxy import isType
from Ganga.Core import ApplicationConfigurationError
from Ganga.Utility.Config import makeConfig, getConfig, ConfigError
from Ganga.Utility.root import getrootsys,getpythonhome
import Ganga.Utility.logging
logger = Ganga.Utility.logging.getLogger()
#config = getConfig('Root_Properties')
import sys, shutil
config = makeConfig('ROOT',"Options for Root backend")
config.addOption('arch','slc4_ia32_gcc34','Architecture of ROOT')
config.addOption('location','/afs/cern.ch/sw/lcg/external/root','Location of ROOT')
config.addOption('path','','Set to a specific ROOT version. Will override other options.')
config.addOption('pythonhome','${location}/../Python/${pythonversion}/${arch}/','Location of the python used for execution of PyROOT script')
config.addOption('pythonversion','',"Version number of python used for execution python ROOT script")
config.addOption('version','5.18.00','Version of ROOT')
import os
from Ganga.Utility.files import expandfilename
shared_path = os.path.join(expandfilename(getConfig('Configuration')['gangadir']),'shared',getConfig('Configuration')['user'])
class Root(IPrepareApp):
"""
Root application -- running ROOT
import os
# from multiprocessing import cpu_count
from Ganga.Utility.Config import makeConfig, getConfig
from Ganga.Utility.logging import getLogger
configDirac = makeConfig("DIRAC", "Parameters for DIRAC")
config = getConfig("Configuration")
logger = getLogger()
# Set default values for the Dirac section.
# configDirac.addOption('ShowDIRACstdout', False,
# 'Display DIRAC API stdout to the screen in Ganga?')
configDirac.addOption("Timeout", 1000, "Default timeout (seconds) for Dirac commands")
configDirac.addOption("splitFilesChunks", 5000, "when splitting datasets, pre split into chunks of this int")
diracenv = ""
if "GANGADIRACENVIRONMENT" in os.environ:
diracenv = os.environ["GANGADIRACENVIRONMENT"]
configDirac.addOption(
"DiracEnvFile",
diracenv,
"Ganga environment file for DIRAC environment (do not change unless you are sure you know what you are doing).",
)
configDirac.addOption(
"DiracCommandFiles",
[os.path.join(os.path.dirname(__file__), "Lib/Server/DiracCommands.py")],
"The file containing the python commands that the local DIRAC server can execute. The default DiracCommands.py is added automatically",
from os import stat
import time
import new
from Ganga.Utility.logging import getLogger
from Ganga.Utility.Config import getConfig, makeConfig, ConfigError
try:
from threading import Lock
except ImportError:
from dummy_threading import Lock
NOT_INITIALIZED = object()
log = getLogger()
config = makeConfig('Caching','Caching for DQ2 dataset')
config.addOption('CacheLifeTime', 150, 'Cache refresh time in seconds')
config.addOption('CacheMaxEntry', 10, 'For CacheMaxEntry == 0, the cache is unbounded, otherwise the Least Recently Used (LRU) entry is discarded.')
class Entry(object):
""" A cache entry, mostly an internal object. """
def __init__(self, key):
object.__init__(self)
self._key=key
self._timestamp = time.time()
self._value=NOT_INITIALIZED
self._lock=Lock()
class Cache(object):
""" An abstract, multi-threaded cache object. """
def relpath(path, start=curdir):
"""Return a relative version of a path"""
if not path:
raise ValueError("no path specified")
start_list = abspath(start).split(sep)
path_list = abspath(path).split(sep)
# Work out how much of the filepath is shared by start and path.
i = len(commonprefix([start_list, path_list]))
rel_list = [pardir] * (len(start_list) - i) + path_list[i:]
if not rel_list:
return curdir
return join(*rel_list)
# set the mergers config up
config = makeConfig('Mergers', 'parameters for mergers')
config.addOption('associate', {'log': 'TextMerger', 'root': 'RootMerger',
'text': 'TextMerger', 'txt': 'TextMerger'}, 'Dictionary of file associations')
gangadir = getConfig('Configuration')['gangadir']
config.addOption('merge_output_dir', gangadir +
'/merge_results', "location of the merger's outputdir")
config.addOption('std_merge', 'TextMerger', 'Standard (default) merger')
def getDefaultMergeDir():
"""Gets the default location of the mergers outputdir from the config"""
try:
config = getConfig('Mergers')
outputdir = config['merge_output_dir']
except ConfigError:
__version__ = "1.0"
__author__="Adrian.Muraru[at]cern[dot]ch"
import os
import sys
## Configuration
from Ganga.Utility.Config import getConfig, makeConfig
from Ganga.Utility.files import previous_dir
from Ganga.Utility.logging import getLogger
myFullPath = os.path.abspath(os.path.dirname(__file__))
gangaReleaseTopDir = previous_dir(myFullPath,3)
# testing framework configuration properties
myConfig = makeConfig('TestingFramework','Configuration section for internal testing framework')
#Release testing mode:
# - xml/html reports are generated
# - coverage analysis tool enabled
myConfig.addOption('ReleaseTesting', False, '')
#enable/disable test-runner
myConfig.addOption('EnableTestRunner', True, 'enable/disable test-runner')
#enable/disable html reporter
myConfig.addOption('EnableHTMLReporter', False, 'enable/disable html reporter')
#enable/disable xml differencer
myConfig.addOption('EnableXMLDifferencer','False', 'enable/disable xml differencer')
#search for local tests lookup
myConfig.addOption('SearchLocalTests', True, 'search for local tests lookup')
#search for tests packaged in release (PACKAGE/old_test dir) lookup
from Ganga.Utility.util import hostname
hostname = hostname()
except Exception as x: # fixme: use OSError instead?
hostname = 'localhost'
# ------------------------------------------------
# Setup all configs for this module
from Ganga.Utility.Config import makeConfig, getConfig, expandvars
# ------------------------------------------------
# Logging
log_config = makeConfig("Logging", """control the messages printed by Ganga
The settings are applied hierarchically to the loggers. Ganga is the name of the top-level logger which
applies by default to all Ganga.* packages unless overriden in sub-packages.
You may define new loggers in this section.
The log level may be one of: CRITICAL ERROR WARNING INFO DEBUG
""", is_open=True)
# FIXME: Ganga WARNING should be turned into INFO level when the messages
# are reviewed in all the code
log_config.addOption('Ganga', "INFO", "top-level logger")
log_config.addOption('Ganga.Runtime.bootstrap', "INFO", 'FIXME')
log_config.addOption('Ganga.GPIDev', "INFO", "logger of Ganga.GPIDev.* packages")
log_config.addOption('Ganga.Utility.logging', "WARNING", "logger of the Ganga logging package itself (use with care!)")
log_config.addOption('_format', "NORMAL", "format of logging messages: TERSE,NORMAL,VERBOSE,DEBUG")
log_config.addOption('_colour', True, "enable ASCII colour formatting of messages e.g. errors in red")
log_config.addOption('_logfile', "~/.ganga.log", "location of the logfile")
log_config.addOption('_logfile_size', 100000,
"the size of the logfile (in bytes), the rotating log will never exceed this file size") # 100 K
log_config.addOption('_interactive_cache', True,
# item is None if the filter is called outside of the attribute assignment context
#
# If conversion takes place filter MUST return an object which is an
# instance (derived) of GangaObject.
# Void filter does nothing. This is the default filter if no other default
# has been defined.
from Ganga.GPIDev.Base.Proxy import isType
from Ganga.Utility.Config import makeConfig
from Ganga.Utility.Config.Config import ConfigError
# test configuration properties
config = makeConfig('GPIComponentFilters', """Customization of GPI component object assignment
for each category there may be multiple filters registered, the one used being defined
in the configuration file in [GPIComponentFilters]
e.g: {'datasets':{'lhcbdatasets':lhcbFilter, 'testdatasets':testFilter}...}
""", is_open=False)
def void_filter(val, item):
return None
class _ComponentFilterManager(object):
def __init__(self):
# for each category there may be multiple filters registered, the one used being defined
# in the configuration file in [GPIComponentFilters]
# e.g: {'datasets':{'lhcbdatasets':lhcbFilter,
# 'testdatasets':testFilter}...}
__version__ = "1.0"
__author__ = "Adrian.Muraru[at]cern[dot]ch"
import os
import sys
## Configuration
from Ganga.Utility.Config import getConfig, makeConfig
from Ganga.Utility.files import previous_dir
from Ganga.Utility.logging import getLogger
myFullPath = os.path.abspath(os.path.dirname(__file__))
gangaReleaseTopDir = previous_dir(myFullPath, 3)
# testing framework configuration properties
myConfig = makeConfig("TestingFramework", "Configuration section for internal testing framework")
# Release testing mode:
# - xml/html reports are generated
# - coverage analysis tool enabled
myConfig.addOption("ReleaseTesting", False, "")
# enable/disable test-runner
myConfig.addOption("EnableTestRunner", True, "enable/disable test-runner")
# enable/disable html reporter
myConfig.addOption("EnableHTMLReporter", False, "enable/disable html reporter")
# enable/disable xml differencer
myConfig.addOption("EnableXMLDifferencer", "False", "enable/disable xml differencer")
# search for local tests lookup
myConfig.addOption("SearchLocalTests", True, "search for local tests lookup")
# search for tests packaged in release (PACKAGE/old_test dir) lookup
from Ganga.Utility.Config import makeConfig, getConfig, ConfigError
from Ganga.Utility.root import getrootsys,getpythonhome
import Ganga.Utility.logging
import sys,os
scratch = os.environ.get("HOME") + '/scratch0'
vocmssw = os.environ.get("VO_CMS_SW_DIR")
if vocmssw == None:
vocmssw = '/afs/cern.ch/cms/sw'
dbsPath = '/afs/cern.ch/cms/sw/slc4_ia32_gcc345/cms/dbs-client/DBS_2_0_6/lib/DBSAPI'
cmsconfig = makeConfig('CMSSW',"Options for CMSSW application")
cmsconfig.addOption('arch','slc5_amd64_gcc434','Architecture of CMSSW')
cmsconfig.addOption('cmsswdir',vocmssw,'Default location of CMSSW at LXPLUS')
cmsconfig.addOption('version','CMSSW_3_8_1','Version of CMSSW')
cmsconfig.addOption('dataOutput',os.environ['HOME']+'/scratch0','The place where OutputData should go')
cmsconfig.addOption('dataTwiki','','Repository of data sets')
cmsconfig.addOption('workArea',os.environ['HOME'],'The top directory where user creates its CMSSW working area')
cmsconfig.addOption('dbsPath',dbsPath,'Path to dbsCommandLine.py script')
cmsconfig.addOption('dbsCommand','dbsCommandLine.py','The dbs-client command')
cafconfig = makeConfig('CMSCAF',"Options for CMSCAF batch system")
cafconfig.addOption('copyCmd','rfcp','The copy command to be used in the CAF batch system')
cafconfig.addOption('mkdirCmd','rfmkdir','The make dir command to be used in the CAF batch system')
crabconfig = makeConfig('CRAB', "parameters of the simple Crab backend")
crabconfig.addOption('CrabEnable', False, 'Enable Crab backend or not')
# File: GangaAtlas/__init__.py
## Config options
from Ganga.Utility.Config import makeConfig, ConfigError, getConfig
import os
# -------------------------------------------------
# Athena Options
config = makeConfig('Athena','Athena configuration parameters')
config.addOption('LCGOutputLocation', 'srm://srm-atlas.cern.ch/castor/cern.ch/grid/atlas/scratch/%s/ganga' % os.environ['USER'], 'FIXME')
config.addOption('LocalOutputLocation', '/castor/cern.ch/atlas/scratch/%s/ganga' % os.environ['USER'], 'FIXME')
config.addOption('IndividualSubjobDirsForLocalOutput', False, 'When copying local output, should dir structure be jid.sid (False) or jid/sid (True)' )
config.addOption('SingleDirForLocalOutput', False, 'When copying local output, only a single dirs used for output and output filenames are changed with jid.sid' )
config.addOption('NoSubDirsAtAllForLocalOutput', False, 'When copying local output, all output is copied to the given output location with no subdirs created' )
config.addOption('ATLAS_SOFTWARE', '/afs/cern.ch/project/gd/apps/atlas/slc3/software', 'FIXME')
config.addOption('PRODUCTION_ARCHIVE_BASEURL', 'http://atlas-computing.web.cern.ch/atlas-computing/links/kitsDirectory/Production/kits/', 'FIXME')
config.addOption('ExcludedSites', '' , 'FIXME')
config.addOption('CMTHOME', os.path.join(os.environ['HOME'],'cmthome') , 'The path in which the cmtsetup magic function will look up the setup.sh for CMT environment setup')
config.addOption('CMTCONFIG', 'i686-slc5-gcc43-opt', 'Default value to be used as CMTCONFIG environment setup value (LCG/Batch backend)')
config.addOption('CMTCONFIG_LIST', [ 'i686-slc4-gcc34-opt', 'i686-slc5-gcc43-opt' ], 'Allowed values for CMTCONFIG environment setup value (LCG/Batch backend)')
config.addOption('MaxJobsAthenaSplitterJobLCG', 1000 , 'Number of maximum jobs allowed for job splitting with the AthenaSplitterJob and the LCG backend')
config.addOption('DCACHE_RA_BUFFER', 32768 , 'Size of the dCache read ahead buffer used for dcap input file reading')
config.addOption('ENABLE_DQ2COPY', False , 'Enable DQ2_COPY input workflow on LCG backend')
config.addOption('ENABLE_SGE_DQ2JOBSPLITTER', False , 'Enable DQ2JobSplitter for SGE backend')
config.addOption('ENABLE_SGE_FILESTAGER', False , 'Enable FILE_STAGER input access mode for SGE backend')
config.addOption('EXE_MAXFILESIZE', 1024*1024 , 'Athena.exetype=EXE jobs: Maximum size of files to be sent to WNs (default 1024*1024B)')
config.addOption('dereferenceSymLinks', False , 'Set to True to dereference symlinks in the sources area. E.g. if src is a symlink, the target will be copied into the sources archive.')
config.addOption('MaxJobsDQ2JobSplitter', 5000, 'Maximum number of allowed subjobs of DQ2JobSplitter')
config.addOption('MaxFilesPandaDQ2JobSplitter', 5000, 'Maximum number of allowed subjobs of DQ2JobSplitter')
config.addOption('MaxJobsDQ2JobSplitterLCGCompile', 500, 'Maximum number of allowed subjobs of DQ2JobSplitter on LCG/Cream with compile option switched on')
config.addOption('MaxFileSizeNGDQ2JobSplitter', 14336, 'Maximum total sum of filesizes per subjob of DQ2JobSplitter at the NG backend (in MB)')
import sys
import Ganga.Utility.logging
from Ganga.Core import GangaException
from Ganga.Core.GangaRepository.Registry import RegistryKeyError, RegistryIndexError, RegistryAccessError
import fnmatch
import collections
from Ganga.Utility.external.OrderedDict import OrderedDict as oDict
import Ganga.Utility.Config
from Ganga.GPIDev.Base.Proxy import isType, stripProxy, getName
from Ganga.Utility.logging import getLogger
from Ganga.Utility.Config import makeConfig
logger = getLogger()
config = makeConfig("Display", 'control the printing style of the different registries ("jobs","box","tasks"...)')
class RegistrySlice(object):
def __init__(self, name, display_prefix):
self.objects = oDict()
self.name = name
self._display_prefix = display_prefix
self._display_columns = config[self._display_prefix + "_columns"]
self._display_columns_show_empty = config[self._display_prefix + "_columns_show_empty"]
self._display_columns_width = config[self._display_prefix + "_columns_width"]
self._display_columns_functions = {}
try:
col_funcs = config[self._display_prefix + "_columns_functions"]
for this_col_func in col_funcs:
self._display_columns_functions[this_col_func] = eval(col_funcs[this_col_func])
except Exception as x: