Beispiel #1
0
    def __init__(self, configuration):
        self.config = configuration
        self.data = None
        self.source = []
        self.parent = []

        self.options = {
            'essources': [],
            'esmodules': [],
            'modules': [],
            'sequences': [],
            'services': [],
            'paths': [],
            'psets': [],
            'blocks': [],
        }

        self.labels = {}
        if self.config.fragment:
            self.labels['process'] = 'fragment'
            self.labels['dict'] = 'fragment.__dict__'
        else:
            self.labels['process'] = 'process'
            self.labels['dict'] = 'process.__dict__'

        if self.config.online:
            self.labels['connect'] = 'frontier://FrontierProd'
        else:
            self.labels['connect'] = 'frontier://FrontierProd'

        if self.config.prescale and (self.config.prescale.lower() != 'none'):
            self.labels['prescale'] = self.config.prescale

        # get the configuration from ConfdB
        from confdbOfflineConverter import OfflineConverter
        self.converter = OfflineConverter(version=self.config.menu.version,
                                          database=self.config.menu.database)
        self.buildPathList()
        self.buildOptions()
        self.getSetupConfigurationFromDB()
        self.getRawConfigurationFromDB()
        self.customize()
Beispiel #2
0
    def __init__(self, configuration):
        self.config = configuration
        self.data = None
        self.source = []
        self.parent = []

        self.options = {
            'essources': [],
            'esmodules': [],
            'modules': [],
            'sequences': [],
            'services': [],
            'paths': [],
            'psets': [],
            'blocks': [],
        }

        self.labels = {}
        if self.config.fragment:
            self.labels['process'] = 'fragment.'
            self.labels['dict'] = 'fragment.__dict__'
        else:
            self.labels['process'] = 'process.'
            self.labels['dict'] = 'process.__dict__'

        if self.config.online:
            self.labels[
                'connect'] = 'frontier://(proxyurl=http://localhost:3128)(serverurl=http://localhost:8000/FrontierOnProd)(serverurl=http://localhost:8000/FrontierOnProd)(retrieve-ziplevel=0)'
        else:
            self.labels['connect'] = 'frontier://FrontierProd'

        if self.config.prescale and (self.config.prescale.lower() != 'none'):
            self.labels['prescale'] = self.config.prescale

        # get the configuration from ConfdB
        from confdbOfflineConverter import OfflineConverter
        self.converter = OfflineConverter(database=self.config.menu.db)
        self.buildPathList()
        self.buildOptions()
        self.getRawConfigurationFromDB()
        self.customize()
Beispiel #3
0
  def __init__(self, configuration):
    self.config = configuration
    self.data   = None
    self.source = []
    self.parent = []

    self.options = {
      'essources' : [],
      'esmodules' : [],
      'modules'   : [],
      'sequences' : [],
      'services'  : [],
      'paths'     : [],
      'psets'     : [],
      'blocks'    : [],
    }

    self.labels = {}
    if self.config.fragment:
      self.labels['process'] = 'fragment'
      self.labels['dict']    = 'fragment.__dict__'
    else:
      self.labels['process'] = 'process'
      self.labels['dict']    = 'process.__dict__'

    if self.config.online:
      self.labels['connect'] = 'frontier://FrontierProd'
    else:
      self.labels['connect'] = 'frontier://FrontierProd'

    if self.config.prescale and (self.config.prescale.lower() != 'none'):
      self.labels['prescale'] = self.config.prescale

    # get the configuration from ConfdB
    from confdbOfflineConverter import OfflineConverter
    self.converter = OfflineConverter(version = self.config.menu.version, database = self.config.menu.database)
    self.buildPathList()
    self.buildOptions()
    self.getSetupConfigurationFromDB()
    self.getRawConfigurationFromDB()
    self.customize()
Beispiel #4
0
  def __init__(self, configuration):
    self.config = configuration
    self.data   = None
    self.source = []
    self.parent = []

    self.options = {
      'essources' : [],
      'esmodules' : [],
      'modules'   : [],
      'sequences' : [],
      'services'  : [],
      'paths'     : [],
      'psets'     : [],
      'blocks'    : [],
    }

    self.labels = {}
    if self.config.fragment:
      self.labels['process'] = 'fragment.'
      self.labels['dict']    = 'fragment.__dict__'
    else:
      self.labels['process'] = 'process.'
      self.labels['dict']    = 'process.__dict__'

    if self.config.online:
      self.labels['connect'] = 'frontier://(proxyurl=http://localhost:3128)(serverurl=http://localhost:8000/FrontierOnProd)(serverurl=http://localhost:8000/FrontierOnProd)(retrieve-ziplevel=0)'
    else:
      self.labels['connect'] = 'frontier://FrontierProd'

    if self.config.prescale and (self.config.prescale.lower() != 'none'):
      self.labels['prescale'] = self.config.prescale

    # get the configuration from ConfdB
    from confdbOfflineConverter import OfflineConverter
    self.converter = OfflineConverter(database = self.config.menu.db)
    self.buildPathList()
    self.buildOptions()
    self.getRawConfigurationFromDB()
    self.customize()
class HLTProcess(object):

  def __init__(self, configuration):
    self.config = configuration
    self.data   = None
    self.source = []
    self.parent = []

    self.options = {
      'essources' : [],
      'esmodules' : [],
      'modules'   : [],
      'sequences' : [],
      'services'  : [],
      'paths'     : [],
      'psets'     : [],
      'blocks'    : [],
    }

    self.labels = {}
    if self.config.fragment:
      self.labels['process'] = 'fragment'
      self.labels['dict']    = 'fragment.__dict__'
    else:
      self.labels['process'] = 'process'
      self.labels['dict']    = 'process.__dict__'

    if self.config.online:
      self.labels['connect'] = 'frontier://(proxyurl=http://localhost:3128)(serverurl=http://localhost:8000/FrontierOnProd)(serverurl=http://localhost:8000/FrontierOnProd)(retrieve-ziplevel=0)'
    else:
      self.labels['connect'] = 'frontier://FrontierProd'

    if self.config.prescale and (self.config.prescale.lower() != 'none'):
      self.labels['prescale'] = self.config.prescale

    # get the configuration from ConfdB
    from confdbOfflineConverter import OfflineConverter
    self.converter = OfflineConverter(version = self.config.menu.version, database = self.config.menu.database)
    self.buildPathList()
    self.buildOptions()
    self.getRawConfigurationFromDB()
    self.customize()


  def getRawConfigurationFromDB(self):
    if self.config.menu.run:
      args = ['--runNumber', self.config.menu.run]
    else:
      args = ['--configName', self.config.menu.name ]
    args.append('--noedsources')
    for key, vals in self.options.iteritems():
      if vals:
        args.extend(('--'+key, ','.join(vals)))

    data, err = self.converter.query( *args )
    if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
        sys.stderr.write("%s: error while retrieving the HLT menu\n\n" % os.path.basename(sys.argv[0]))
        sys.stderr.write(err + "\n\n")
        sys.exit(1)
    self.data = data


  def getPathList(self):
    if self.config.menu.run:
      args = ['--runNumber', self.config.menu.run]
    else:
      args = ['--configName', self.config.menu.name]
    args.extend( (
      '--cff',
      '--noedsources',
      '--noes',
      '--noservices',
      '--nosequences',
      '--nomodules'
    ) )

    data, err = self.converter.query( *args )
    if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
        sys.stderr.write("%s: error while retrieving the list of paths from the HLT menu\n\n" % os.path.basename(sys.argv[0]))
        sys.stderr.write(err + "\n\n")
        sys.exit(1)
    filter = re.compile(r' *= *cms.(End)?Path.*')
    paths  = [ filter.sub('', line) for line in data.splitlines() if filter.search(line) ]
    return paths


  @staticmethod
  def expandWildcards(globs, collection):
    # expand a list of unix-style wildcards matching a given collection
    # wildcards with no matches are silently discarded
    matches = []
    for glob in globs:
      negate = ''
      if glob[0] == '-':
        negate = '-'
        glob   = glob[1:]
      # translate a unix-style glob expression into a regular expression
      filter = re.compile(r'^' + glob.replace('?', '.').replace('*', '.*').replace('[!', '[^') + r'$')
      matches.extend( negate + element for element in collection if filter.match(element) )
    return matches


  @staticmethod
  def consolidateNegativeList(elements):
    # consolidate a list of path exclusions and re-inclusions
    # the result is the list of paths to be removed from the dump
    result = set()
    for element in elements:
      if element[0] == '-':
        result.add( element )
      else:
        result.discard( '-' + element )
    return sorted( element for element in result )

  @staticmethod
  def consolidatePositiveList(elements):
    # consolidate a list of path selection and re-exclusions
    # the result is the list of paths to be included in the dump
    result = set()
    for element in elements:
      if element[0] == '-':
        result.discard( element[1:] )
      else:
        result.add( element )
    return sorted( element for element in result )


  # dump the final configuration
  def dump(self):
    self.data = self.data % self.labels
    if self.config.fragment:
      self.data = re.sub( r'\bprocess\b', 'fragment', self.data )
      self.data = re.sub( r'\bProcess\b', 'ProcessFragment', self.data )
    return self.data


  # add specific customizations
  def specificCustomize(self):
    # specific customizations now live in HLTrigger.Configuration.customizeHLTforALL.customizeHLTforAll(.,.)
    if self.config.fragment:
      self.data += """
# add specific customizations
from HLTrigger.Configuration.customizeHLTforALL import customizeHLTforAll
fragment = customizeHLTforAll(fragment,"%s")
""" % (self.config.type)
    else:
      if self.config.type=="Fake":
        prefix = "run1"
      else:
        prefix = "run2"
      _gtData = "auto:"+prefix+"_hlt_"+self.config.type
      _gtMc   = "auto:"+prefix+"_mc_" +self.config.type
      self.data += """
# add specific customizations
_customInfo = {}
_customInfo['menuType'  ]= "%s"
_customInfo['globalTags']= {}
_customInfo['globalTags'][True ] = "%s"
_customInfo['globalTags'][False] = "%s"
_customInfo['inputFiles']={}
_customInfo['inputFiles'][True]  = "file:RelVal_Raw_%s_DATA.root"
_customInfo['inputFiles'][False] = "file:RelVal_Raw_%s_MC.root"
_customInfo['maxEvents' ]=  %s
_customInfo['globalTag' ]= "%s"
_customInfo['inputFile' ]=  %s
_customInfo['realData'  ]=  %s
from HLTrigger.Configuration.customizeHLTforALL import customizeHLTforAll
%%(process)s = customizeHLTforAll(%%(process)s,"%s",_customInfo)
""" % (self.config.type,_gtData,_gtMc,self.config.type,self.config.type,self.config.events,self.config.globaltag,self.source,self.config.data,self.config.type)

    self.data += """
from HLTrigger.Configuration.customizeHLTforCMSSW import customizeHLTforCMSSW
%%(process)s = customizeHLTforCMSSW(%%(process)s,"%s")
""" % (self.config.type)

  # customize the configuration according to the options
  def customize(self):

    # adapt the source to the current scenario
    if not self.config.fragment:
      self.build_source()

    # manual override some parameters
    if self.config.type in ('HIon', ):
      if self.config.data:
        if not self.config.fragment:
          self._fix_parameter( type = 'InputTag', value = 'rawDataCollector',  replace = 'rawDataRepacker')

    # if requested, remove the HLT prescales
    self.fixPrescales()

    # if requested, override all ED/HLTfilters to always pass ("open" mode)
    self.instrumentOpenMode()

    # if requested, change all HLTTriggerTypeFilter EDFilters to accept only error events (SelectedTriggerType = 0)
    self.instrumentErrorEventType()

    # if requested, instrument the self with the modules and EndPath needed for timing studies
    self.instrumentTiming()

    # if requested, override the L1 self from the GlobalTag (Xml)
    self.overrideL1MenuXml()

    # if requested, run the L1 emulator
    self.runL1Emulator()

    if self.config.fragment:
      self.data += """
# dummyfy hltGetConditions in cff's
if 'hltGetConditions' in %(dict)s and 'HLTriggerFirstPath' in %(dict)s :
    %(process)s.hltDummyConditions = cms.EDFilter( "HLTBool",
        result = cms.bool( True )
    )
    %(process)s.HLTriggerFirstPath.replace(%(process)s.hltGetConditions,%(process)s.hltDummyConditions)
"""

    else:

      # override the process name and adapt the relevant filters
      self.overrideProcessName()

      # override the output modules to output root files
      self.overrideOutput()

      # add global options
      self.addGlobalOptions()

      # if requested or necessary, override the GlobalTag and connection strings (incl. L1!)
      self.overrideGlobalTag()

      # request summary informations from the MessageLogger
      self.updateMessageLogger()

      # replace DQMStore and DQMRootOutputModule with a configuration suitable for running offline
      self.instrumentDQM()

      # load 5.2.x JECs, until they are in the GlobalTag
#      self.loadAdditionalConditions('load 5.2.x JECs',
#        {
#          'record'  : 'JetCorrectionsRecord',
#          'tag'     : 'JetCorrectorParametersCollection_AK5Calo_2012_V8_hlt_mc',
#          'label'   : 'AK5CaloHLT',
#          'connect' : '%(connect)s/CMS_CONDITIONS'
#        }, {
#          'record'  : 'JetCorrectionsRecord',
#          'tag'     : 'JetCorrectorParametersCollection_AK5PF_2012_V8_hlt_mc',
#          'label'   : 'AK5PFHLT',
#          'connect' : '%(connect)s/CMS_CONDITIONS'
#        }, {
#          'record'  : 'JetCorrectionsRecord',
#          'tag'     : 'JetCorrectorParametersCollection_AK5PFchs_2012_V8_hlt_mc',
#          'label'   : 'AK5PFchsHLT',
#          'connect' : '%(connect)s/CMS_CONDITIONS'
#        }
#      )

    # add specific customisations
    self.specificCustomize()


  def addGlobalOptions(self):
    # add global options
    self.data += """
# limit the number of events to be processed
%%(process)s.maxEvents = cms.untracked.PSet(
    input = cms.untracked.int32( %d )
)
""" % self.config.events

    if not self.config.profiling:
      self.data += """
# enable TrigReport, TimeReport and MultiThreading
%(process)s.options = cms.untracked.PSet(
    wantSummary = cms.untracked.bool( True ),
    numberOfThreads = cms.untracked.uint32( 4 ),
    numberOfStreams = cms.untracked.uint32( 0 ),
    sizeOfStackForThreadsInKB = cms.untracked.uint32( 10*1024 )
)
"""

  def _fix_parameter(self, **args):
    """arguments:
        name:     parameter name (optional)
        type:     parameter type (look for tracked and untracked variants)
        value:    original value
        replace:  replacement value
    """
    if 'name' in args:
      self.data = re.sub(
          r'%(name)s = cms(?P<tracked>(?:\.untracked)?)\.%(type)s\( (?P<quote>["\']?)%(value)s(?P=quote)' % args,
          r'%(name)s = cms\g<tracked>.%(type)s( \g<quote>%(replace)s\g<quote>' % args,
          self.data)
    else:
      self.data = re.sub(
          r'cms(?P<tracked>(?:\.untracked)?)\.%(type)s\( (?P<quote>["\']?)%(value)s(?P=quote)' % args,
          r'cms\g<tracked>.%(type)s( \g<quote>%(replace)s\g<quote>' % args,
          self.data)


  def fixPrescales(self):
    # update the PrescaleService to match the new list of paths
    if self.options['paths']:
      if self.options['paths'][0][0] == '-':
        # drop requested paths
        for minuspath in self.options['paths']:
          path = minuspath[1:]
          self.data = re.sub(r'      cms.PSet\(  pathName = cms.string\( "%s" \),\n        prescales = cms.vuint32\( .* \)\n      \),?\n' % path, '', self.data)
      else:
        # keep requested paths
        for path in self.all_paths:
          if path not in self.options['paths']:
            self.data = re.sub(r'      cms.PSet\(  pathName = cms.string\( "%s" \),\n        prescales = cms.vuint32\( .* \)\n      \),?\n' % path, '', self.data)

    if self.config.prescale and (self.config.prescale.lower() != 'none'):
      # TO DO: check that the requested prescale column is valid
      self.data += """
# force the use of a specific HLT prescale column
if 'PrescaleService' in %(dict)s:
    %(process)s.PrescaleService.forceDefault     = True
    %(process)s.PrescaleService.lvl1DefaultLabel = '%(prescale)s'
"""


  def instrumentOpenMode(self):
    if self.config.open:
      # find all EDfilters
      filters = [ match[1] for match in re.findall(r'(process\.)?\b(\w+) = cms.EDFilter', self.data) ]
      re_sequence = re.compile( r'cms\.(Path|Sequence)\((.*)\)' )
      # remove existing 'cms.ignore' and '~' modifiers
      self.data = re_sequence.sub( lambda line: re.sub( r'cms\.ignore *\( *((process\.)?\b(\w+)) *\)', r'\1', line.group(0) ), self.data )
      self.data = re_sequence.sub( lambda line: re.sub( r'~', '', line.group(0) ), self.data )
      # wrap all EDfilters with "cms.ignore( ... )", 1000 at a time (python 2.6 complains for too-big regular expressions)
      for some in splitter(filters, 1000):
        re_filters  = re.compile( r'\b((process\.)?(' + r'|'.join(some) + r'))\b' )
        self.data = re_sequence.sub( lambda line: re_filters.sub( r'cms.ignore( \1 )', line.group(0) ), self.data )


  def instrumentErrorEventType(self):
    if self.config.errortype:
      # change all HLTTriggerTypeFilter EDFilters to accept only error events (SelectedTriggerType = 0)
      self._fix_parameter(name = 'SelectedTriggerType', type ='int32', value = '1', replace = '0')
      self._fix_parameter(name = 'SelectedTriggerType', type ='int32', value = '2', replace = '0')
      self._fix_parameter(name = 'SelectedTriggerType', type ='int32', value = '3', replace = '0')


  def overrideGlobalTag(self):
    # overwrite GlobalTag
    # the logic is:
    #   - always set the correct connection string and pfnPrefix
    #   - if a GlobalTag is specified on the command line:
    #      - override the global tag
    #      - if the GT is "auto:...", insert the code to read it from Configuration.AlCa.autoCond
    #   - if a GlobalTag is NOT  specified on the command line:
    #      - when running on data, do nothing, and keep the global tag in the menu
    #      - when running on mc, take the GT from the configuration.type

    # override the GlobalTag connection string and pfnPrefix
    text = """
# override the GlobalTag, connection string and pfnPrefix
if 'GlobalTag' in %(dict)s:
"""

    # when running on MC, override the global tag even if not specified on the command line
    if not self.config.data and not self.config.globaltag:
      if self.config.type in globalTag:
        self.config.globaltag = globalTag[self.config.type]
      else:
        self.config.globaltag = globalTag['GRun']

    # if requested, override the L1 menu from the GlobalTag (using the same connect as the GlobalTag itself)
    if self.config.l1.override:
      self.config.l1.tag    = self.config.l1.override
      self.config.l1.record = 'L1TUtmTriggerMenuRcd'
      self.config.l1.connect = '%(connect)s/CMS_CONDITIONS'
      self.config.l1.label  = ''
      if not self.config.l1.snapshotTime:
        self.config.l1.snapshotTime = '9999-12-31 23:59:59.000'
      self.config.l1cond = '%(tag)s,%(record)s,%(connect)s,%(label)s,%(snapshotTime)s' % self.config.l1.__dict__
    else:
      self.config.l1cond = None

    if self.config.globaltag or self.config.l1cond:
      text += "    from Configuration.AlCa.GlobalTag import GlobalTag as customiseGlobalTag\n"
      text += "    %(process)s.GlobalTag = customiseGlobalTag(%(process)s.GlobalTag"
      if self.config.globaltag:
        text += ", globaltag = %s"  % repr(self.config.globaltag)
      if self.config.l1cond:
        text += ", conditions = %s" % repr(self.config.l1cond)
      text += ")\n"

    text += """    %(process)s.GlobalTag.connect   = '%(connect)s/CMS_CONDITIONS'
    %(process)s.GlobalTag.pfnPrefix = cms.untracked.string('%(connect)s/')
    for pset in %(process)s.GlobalTag.toGet.value():
        pset.connect = pset.connect.value().replace('frontier://FrontierProd/', '%(connect)s/')
    # fix for multi-run processing
    %(process)s.GlobalTag.RefreshEachRun = cms.untracked.bool( False )
    %(process)s.GlobalTag.ReconnectEachRun = cms.untracked.bool( False )
"""
    self.data += text

  def overrideL1MenuXml(self):
    # if requested, override the GlobalTag's L1T menu from an Xml file
    if self.config.l1Xml.XmlFile:
      text = """
# override the GlobalTag's L1T menu from an Xml file
from HLTrigger.Configuration.CustomConfigs import L1XML
%%(process)s = L1XML(%%(process)s,"%s")
""" % (self.config.l1Xml.XmlFile) 
      self.data += text

  def runL1Emulator(self):
    # if requested, run the Full L1T emulator, then repack the data into a new RAW collection, to be used by the HLT
    if self.config.emulator:
      text = """
# run the Full L1T emulator, then repack the data into a new RAW collection, to be used by the HLT
from HLTrigger.Configuration.CustomConfigs import L1REPACK
%%(process)s = L1REPACK(%%(process)s,"%s")
""" % (self.config.emulator)
      self.data += text

  def overrideOutput(self):
    # override the "online" ShmStreamConsumer output modules with "offline" PoolOutputModule's
    self.data = re.sub(
      r'\b(process\.)?hltOutput(\w+) *= *cms\.OutputModule\( *"ShmStreamConsumer" *,',
      r'%(process)s.hltOutput\2 = cms.OutputModule( "PoolOutputModule",\n    fileName = cms.untracked.string( "output\2.root" ),\n    fastCloning = cms.untracked.bool( False ),\n    dataset = cms.untracked.PSet(\n        filterName = cms.untracked.string( "" ),\n        dataTier = cms.untracked.string( "RAW" )\n    ),',
      self.data
    )

    if not self.config.fragment and self.config.output == 'full':
      # add a single "keep *" output
      self.data += """
# add a single "keep *" output
%(process)s.hltOutputFULL = cms.OutputModule( "PoolOutputModule",
    fileName = cms.untracked.string( "outputFULL.root" ),
    fastCloning = cms.untracked.bool( False ),
    dataset = cms.untracked.PSet(
        dataTier = cms.untracked.string( 'RECO' ),
        filterName = cms.untracked.string( '' )
    ),
    outputCommands = cms.untracked.vstring( 'keep *' )
)
%(process)s.FULLOutput = cms.EndPath( %(process)s.hltOutputFULL )
"""


  # override the process name and adapt the relevant filters
  def overrideProcessName(self):
    if self.config.name is None:
      return

    # sanitise process name
    self.config.name = self.config.name.replace("_","")
    # override the process name
    quote = '[\'\"]'
    self.data = re.compile(r'^(process\s*=\s*cms\.Process\(\s*' + quote + r')\w+(' + quote + r'\s*\).*)$', re.MULTILINE).sub(r'\1%s\2' % self.config.name, self.data, 1)

    # the following was stolen and adapted from HLTrigger.Configuration.customL1THLT_Options
    self.data += """
# adapt HLT modules to the correct process name
if 'hltTrigReport' in %%(dict)s:
    %%(process)s.hltTrigReport.HLTriggerResults                    = cms.InputTag( 'TriggerResults', '', '%(name)s' )

if 'hltPreExpressCosmicsOutputSmart' in %%(dict)s:
    %%(process)s.hltPreExpressCosmicsOutputSmart.hltResults = cms.InputTag( 'TriggerResults', '', '%(name)s' )

if 'hltPreExpressOutputSmart' in %%(dict)s:
    %%(process)s.hltPreExpressOutputSmart.hltResults        = cms.InputTag( 'TriggerResults', '', '%(name)s' )

if 'hltPreDQMForHIOutputSmart' in %%(dict)s:
    %%(process)s.hltPreDQMForHIOutputSmart.hltResults       = cms.InputTag( 'TriggerResults', '', '%(name)s' )

if 'hltPreDQMForPPOutputSmart' in %%(dict)s:
    %%(process)s.hltPreDQMForPPOutputSmart.hltResults       = cms.InputTag( 'TriggerResults', '', '%(name)s' )

if 'hltPreHLTDQMResultsOutputSmart' in %%(dict)s:
    %%(process)s.hltPreHLTDQMResultsOutputSmart.hltResults  = cms.InputTag( 'TriggerResults', '', '%(name)s' )

if 'hltPreHLTDQMOutputSmart' in %%(dict)s:
    %%(process)s.hltPreHLTDQMOutputSmart.hltResults         = cms.InputTag( 'TriggerResults', '', '%(name)s' )

if 'hltPreHLTMONOutputSmart' in %%(dict)s:
    %%(process)s.hltPreHLTMONOutputSmart.hltResults         = cms.InputTag( 'TriggerResults', '', '%(name)s' )

if 'hltDQMHLTScalers' in %%(dict)s:
    %%(process)s.hltDQMHLTScalers.triggerResults                   = cms.InputTag( 'TriggerResults', '', '%(name)s' )
    %%(process)s.hltDQMHLTScalers.processname                      = '%(name)s'

if 'hltDQML1SeedLogicScalers' in %%(dict)s:
    %%(process)s.hltDQML1SeedLogicScalers.processname              = '%(name)s'
""" % self.config.__dict__


  def updateMessageLogger(self):
    # request summary informations from the MessageLogger
    self.data += """
if 'MessageLogger' in %(dict)s:
    %(process)s.MessageLogger.categories.append('TriggerSummaryProducerAOD')
    %(process)s.MessageLogger.categories.append('L1GtTrigReport')
    %(process)s.MessageLogger.categories.append('L1TGlobalSummary')
    %(process)s.MessageLogger.categories.append('HLTrigReport')
    %(process)s.MessageLogger.categories.append('FastReport')
"""


  def loadAdditionalConditions(self, comment, *conditions):
    # load additional conditions
    self.data += """
# %s
if 'GlobalTag' in %%(dict)s:
""" % comment
    for condition in conditions:
      self.data += """    %%(process)s.GlobalTag.toGet.append(
        cms.PSet(
            record  = cms.string( '%(record)s' ),
            tag     = cms.string( '%(tag)s' ),
            label   = cms.untracked.string( '%(label)s' ),
            connect = cms.untracked.string( '%(connect)s' )
        )
    )
""" % condition


  def loadCffCommand(self, module):
    # load a cfi or cff module
    if self.config.fragment:
      return 'from %s import *\n' % module
    else:
      return 'process.load( "%s" )\n' % module

  def loadCff(self, module):
    self.data += self.loadCffCommand(module)


  def overrideParameters(self, module, parameters):
    # override a module's parameter if the module is present in the configuration
    self.data += "if '%s' in %%(dict)s:\n" % module
    for (parameter, value) in parameters:
      self.data += "    %%(process)s.%s.%s = %s\n" % (module, parameter, value)
    self.data += "\n"


  def instrumentTiming(self):
    if self.config.profiling:
      # instrument the menu for profiling: remove the HLTAnalyzerEndpath, add/override the HLTriggerFirstPath, with hltGetRaw and hltGetConditions
      text = ''

      if not 'hltGetRaw' in self.data:
        # add hltGetRaw
        text += """
%(process)s.hltGetRaw = cms.EDAnalyzer( "HLTGetRaw",
    RawDataCollection = cms.InputTag( "rawDataCollector" )
)
"""

      if not 'hltGetConditions' in self.data:
        # add hltGetConditions
        text += """
%(process)s.hltGetConditions = cms.EDAnalyzer( 'EventSetupRecordDataGetter',
    verbose = cms.untracked.bool( False ),
    toGet = cms.VPSet( )
)
"""

      if not 'hltBoolFalse' in self.data:
        # add hltBoolFalse
        text += """
%(process)s.hltBoolFalse = cms.EDFilter( "HLTBool",
    result = cms.bool( False )
)
"""

      # add the definition of HLTriggerFirstPath
      # FIXME in a cff, should also update the HLTSchedule
      text += """
%(process)s.HLTriggerFirstPath = cms.Path( %(process)s.hltGetRaw + %(process)s.hltGetConditions + %(process)s.hltBoolFalse )
"""
      self.data = re.sub(r'.*cms\.(End)?Path.*', text + r'\g<0>', self.data, 1)


    # instrument the menu with the Service, EDProducer and EndPath needed for timing studies
    # FIXME in a cff, should also update the HLTSchedule
    if self.config.timing:
      self.data += """
# instrument the menu with the modules and EndPath needed for timing studies
"""

      if not 'FastTimerService' in self.data:
        self.data += '\n# configure the FastTimerService\n'
        self.loadCff('HLTrigger.Timer.FastTimerService_cfi')
      else:
        self.data += '\n# configure the FastTimerService\n'

      self.data += """# this is currently ignored in CMSSW 7.x, always using the real time clock
%(process)s.FastTimerService.useRealTimeClock          = True
# enable specific features
%(process)s.FastTimerService.enableTimingPaths         = True
%(process)s.FastTimerService.enableTimingModules       = True
%(process)s.FastTimerService.enableTimingExclusive     = True
# print a text summary at the end of the job
%(process)s.FastTimerService.enableTimingSummary       = True
# skip the first path (disregard the time spent loading event and conditions data)
%(process)s.FastTimerService.skipFirstPath             = True
# enable DQM plots
%(process)s.FastTimerService.enableDQM                 = True
# enable most per-path DQM plots
%(process)s.FastTimerService.enableDQMbyPathActive     = True
%(process)s.FastTimerService.enableDQMbyPathTotal      = True
%(process)s.FastTimerService.enableDQMbyPathOverhead   = False
%(process)s.FastTimerService.enableDQMbyPathDetails    = True
%(process)s.FastTimerService.enableDQMbyPathCounters   = True
%(process)s.FastTimerService.enableDQMbyPathExclusive  = True
# disable per-module DQM plots
%(process)s.FastTimerService.enableDQMbyModule         = False
%(process)s.FastTimerService.enableDQMbyModuleType     = False
# enable per-event DQM sumary plots
%(process)s.FastTimerService.enableDQMSummary          = True
# enable per-event DQM plots by lumisection
%(process)s.FastTimerService.enableDQMbyLumiSection    = True
%(process)s.FastTimerService.dqmLumiSectionsRange      = 2500
# set the time resolution of the DQM plots
%(process)s.FastTimerService.dqmTimeRange              = 1000.
%(process)s.FastTimerService.dqmTimeResolution         =    5.
%(process)s.FastTimerService.dqmPathTimeRange          =  100.
%(process)s.FastTimerService.dqmPathTimeResolution     =    0.5
%(process)s.FastTimerService.dqmModuleTimeRange        =   40.
%(process)s.FastTimerService.dqmModuleTimeResolution   =    0.2
# set the base DQM folder for the plots
%(process)s.FastTimerService.dqmPath                   = 'HLT/TimerService'
%(process)s.FastTimerService.enableDQMbyProcesses      = True
"""


  def instrumentDQM(self):
    if not self.config.hilton:
      # remove any reference to the hltDQMFileSaver
      if 'hltDQMFileSaver' in self.data:
        self.data = re.sub(r'\b(process\.)?hltDQMFileSaver \+ ', '', self.data)
        self.data = re.sub(r' \+ \b(process\.)?hltDQMFileSaver', '', self.data)
        self.data = re.sub(r'\b(process\.)?hltDQMFileSaver',     '', self.data)

      # instrument the HLT menu with DQMStore and DQMRootOutputModule suitable for running offline
      dqmstore  = "\n# load the DQMStore and DQMRootOutputModule\n"
      dqmstore += self.loadCffCommand('DQMServices.Core.DQMStore_cfi')
      dqmstore += "%(process)s.DQMStore.enableMultiThread = True\n"
      dqmstore += """
%(process)s.dqmOutput = cms.OutputModule("DQMRootOutputModule",
    fileName = cms.untracked.string("DQMIO.root")
)
"""

      empty_path = re.compile(r'.*\b(process\.)?DQMOutput = cms\.EndPath\( *\).*')
      other_path = re.compile(r'(.*\b(process\.)?DQMOutput = cms\.EndPath\()(.*)')
      if empty_path.search(self.data):
        # replace an empty DQMOutput path
        self.data = empty_path.sub(dqmstore + '\n%(process)s.DQMOutput = cms.EndPath( %(process)s.dqmOutput )\n', self.data)
      elif other_path.search(self.data):
        # prepend the dqmOutput to the DQMOutput path
        self.data = other_path.sub(dqmstore + r'\g<1> %(process)s.dqmOutput +\g<3>', self.data)
      else:
        # ceate a new DQMOutput path with the dqmOutput module
        self.data += dqmstore
        self.data += '\n%(process)s.DQMOutput = cms.EndPath( %(process)s.dqmOutput )\n'


  @staticmethod
  def dumppaths(paths):
    sys.stderr.write('Path selection:\n')
    for path in paths:
      sys.stderr.write('\t%s\n' % path)
    sys.stderr.write('\n\n')

  def buildPathList(self):
    self.all_paths = self.getPathList()

    if self.config.paths:
      # no path list was requested, dump the full table, minus unsupported / unwanted paths
      paths = self.config.paths.split(',')
    else:
      # dump only the requested paths, plus the eventual output endpaths
      paths = []

    if self.config.fragment or self.config.output in ('none', 'full'):
      # 'full' removes all outputs (same as 'none') and then adds a single "keep *" output (see the overrideOutput method)
      if self.config.paths:
        # paths are removed by default
        pass
      else:
        # drop all output endpaths
        paths.append( "-*Output" )
        paths.append( "-RatesMonitoring")
        paths.append( "-DQMHistograms")
    elif self.config.output == 'minimal':
      # drop all output endpaths but HLTDQMResultsOutput
      if self.config.paths:
        paths.append( "HLTDQMResultsOutput" )
      else:
        paths.append( "-*Output" )
        paths.append( "-RatesMonitoring")
        paths.append( "-DQMHistograms")
        paths.append( "HLTDQMResultsOutput" )
    else:
      # keep / add back all output endpaths
      if self.config.paths:
        paths.append( "*Output" )
      else:
        pass    # paths are kepy by default

    # drop unwanted paths for profiling (and timing studies)
    if self.config.profiling:
      paths.append( "-HLTriggerFirstPath" )
      paths.append( "-HLTAnalyzerEndpath" )

    # this should never be in any dump (nor online menu)
    paths.append( "-OfflineOutput" )

    # expand all wildcards
    paths = self.expandWildcards(paths, self.all_paths)

    if self.config.paths:
      # do an "additive" consolidation
      self.options['paths'] = self.consolidatePositiveList(paths)
      if not self.options['paths']:
        raise RuntimeError('Error: option "--paths %s" does not select any valid paths' % self.config.paths)
    else:
      # do a "subtractive" consolidation
      self.options['paths'] = self.consolidateNegativeList(paths)


  def buildOptions(self):
    # common configuration for all scenarios
    self.options['services'].append( "-DQM" )
    self.options['services'].append( "-FUShmDQMOutputService" )
    self.options['services'].append( "-MicroStateService" )
    self.options['services'].append( "-ModuleWebRegistry" )
    self.options['services'].append( "-TimeProfilerService" )

    # remove the DAQ modules and the online definition of the DQMStore and DQMFileSaver
    # unless a hilton-like configuration has been requested
    if not self.config.hilton:
      self.options['services'].append( "-EvFDaqDirector" )
      self.options['services'].append( "-FastMonitoringService" )
      self.options['services'].append( "-DQMStore" )
      self.options['modules'].append( "-hltDQMFileSaver" )

    if self.config.fragment:
      # extract a configuration file fragment
      self.options['essources'].append( "-GlobalTag" )
      self.options['essources'].append( "-HepPDTESSource" )
      self.options['essources'].append( "-XMLIdealGeometryESSource" )
      self.options['essources'].append( "-eegeom" )
      self.options['essources'].append( "-es_hardcode" )
      self.options['essources'].append( "-magfield" )

      self.options['esmodules'].append( "-AutoMagneticFieldESProducer" )
      self.options['esmodules'].append( "-SlaveField0" )
      self.options['esmodules'].append( "-SlaveField20" )
      self.options['esmodules'].append( "-SlaveField30" )
      self.options['esmodules'].append( "-SlaveField35" )
      self.options['esmodules'].append( "-SlaveField38" )
      self.options['esmodules'].append( "-SlaveField40" )
      self.options['esmodules'].append( "-VBF0" )
      self.options['esmodules'].append( "-VBF20" )
      self.options['esmodules'].append( "-VBF30" )
      self.options['esmodules'].append( "-VBF35" )
      self.options['esmodules'].append( "-VBF38" )
      self.options['esmodules'].append( "-VBF40" )
      self.options['esmodules'].append( "-CSCGeometryESModule" )
      self.options['esmodules'].append( "-CaloGeometryBuilder" )
      self.options['esmodules'].append( "-CaloTowerHardcodeGeometryEP" )
      self.options['esmodules'].append( "-CastorHardcodeGeometryEP" )
      self.options['esmodules'].append( "-DTGeometryESModule" )
      self.options['esmodules'].append( "-EcalBarrelGeometryEP" )
      self.options['esmodules'].append( "-EcalElectronicsMappingBuilder" )
      self.options['esmodules'].append( "-EcalEndcapGeometryEP" )
      self.options['esmodules'].append( "-EcalLaserCorrectionService" )
      self.options['esmodules'].append( "-EcalPreshowerGeometryEP" )
      self.options['esmodules'].append( "-HcalHardcodeGeometryEP" )
      self.options['esmodules'].append( "-HcalTopologyIdealEP" )
      self.options['esmodules'].append( "-MuonNumberingInitialization" )
      self.options['esmodules'].append( "-ParametrizedMagneticFieldProducer" )
      self.options['esmodules'].append( "-RPCGeometryESModule" )
      self.options['esmodules'].append( "-SiStripGainESProducer" )
      self.options['esmodules'].append( "-SiStripRecHitMatcherESProducer" )
      self.options['esmodules'].append( "-SiStripQualityESProducer" )
      self.options['esmodules'].append( "-StripCPEfromTrackAngleESProducer" )
      self.options['esmodules'].append( "-TrackerDigiGeometryESModule" )
      self.options['esmodules'].append( "-TrackerGeometricDetESModule" )
      self.options['esmodules'].append( "-VolumeBasedMagneticFieldESProducer" )
      self.options['esmodules'].append( "-ZdcHardcodeGeometryEP" )
      self.options['esmodules'].append( "-hcal_db_producer" )
      self.options['esmodules'].append( "-L1GtTriggerMaskAlgoTrigTrivialProducer" )
      self.options['esmodules'].append( "-L1GtTriggerMaskTechTrigTrivialProducer" )
      self.options['esmodules'].append( "-hltESPEcalTrigTowerConstituentsMapBuilder" )
      self.options['esmodules'].append( "-hltESPGlobalTrackingGeometryESProducer" )
      self.options['esmodules'].append( "-hltESPMuonDetLayerGeometryESProducer" )
      self.options['esmodules'].append( "-hltESPTrackerRecoGeometryESProducer" )
      self.options['esmodules'].append( "-trackerTopology" )

      self.options['esmodules'].append( "-CaloTowerGeometryFromDBEP" )
      self.options['esmodules'].append( "-CastorGeometryFromDBEP" )
      self.options['esmodules'].append( "-EcalBarrelGeometryFromDBEP" )
      self.options['esmodules'].append( "-EcalEndcapGeometryFromDBEP" )
      self.options['esmodules'].append( "-EcalPreshowerGeometryFromDBEP" )
      self.options['esmodules'].append( "-HcalGeometryFromDBEP" )
      self.options['esmodules'].append( "-ZdcGeometryFromDBEP" )
      self.options['esmodules'].append( "-XMLFromDBSource" )
      self.options['esmodules'].append( "-sistripconn" )

      self.options['services'].append( "-MessageLogger" )

      self.options['psets'].append( "-maxEvents" )
      self.options['psets'].append( "-options" )

    if self.config.fragment or (self.config.prescale and (self.config.prescale.lower() == 'none')):
      self.options['services'].append( "-PrescaleService" )

    if self.config.fragment or self.config.timing:
      self.options['services'].append( "-FastTimerService" )


  def append_filenames(self, name, filenames):
    if len(filenames) > 255:
      token_open  = "( *("
      token_close = ") )"
    else:
      token_open  = "("
      token_close = ")"

    self.data += "    %s = cms.untracked.vstring%s\n" % (name, token_open)
    for line in filenames:
      self.data += "        '%s',\n" % line
    self.data += "    %s,\n" % (token_close)


  def expand_filenames(self, input):
    # check if the input is a dataset or a list of files
    if input[0:8] == 'dataset:':
      from dasFileQuery import dasFileQuery
      # extract the dataset name, and use DAS to fine the list of LFNs
      dataset = input[8:]
      files = dasFileQuery(dataset)
    else:
      # assume a comma-separated list of input files
      files = self.config.input.split(',')
    return files

  def build_source(self):
    if self.config.input:
      # if a dataset or a list of input files was given, use it
      self.source = self.expand_filenames(self.config.input)
    elif self.config.online:
      # online we always run on data
      self.source = [ "file:/tmp/InputCollection.root" ]
    elif self.config.data:
      # offline we can run on data...
      self.source = [ "file:RelVal_Raw_%s_DATA.root" % self.config.type ]
    else:
      # ...or on mc
      self.source = [ "file:RelVal_Raw_%s_MC.root" % self.config.type ]

    if self.config.parent:
      # if a dataset or a list of input files was given for the parent data, use it
      self.parent = self.expand_filenames(self.config.parent)

    self.data += """
%(process)s.source = cms.Source( "PoolSource",
"""
    self.append_filenames("fileNames", self.source)
    if (self.parent):
      self.append_filenames("secondaryFileNames", self.parent)
    self.data += """\
Beispiel #6
0
class HLTProcess(object):

  def __init__(self, configuration):
    self.config = configuration
    self.data   = None
    self.source = []
    self.parent = []

    self.options = {
      'essources' : [],
      'esmodules' : [],
      'modules'   : [],
      'sequences' : [],
      'services'  : [],
      'paths'     : [],
      'psets'     : [],
      'blocks'    : [],
    }

    self.labels = {}
    if self.config.fragment:
      self.labels['process'] = 'fragment'
      self.labels['dict']    = 'fragment.__dict__'
    else:
      self.labels['process'] = 'process'
      self.labels['dict']    = 'process.__dict__'

    if self.config.online:
      self.labels['connect'] = 'frontier://FrontierProd'
    else:
      self.labels['connect'] = 'frontier://FrontierProd'

    if self.config.prescale and (self.config.prescale.lower() != 'none'):
      self.labels['prescale'] = self.config.prescale

    # get the configuration from ConfdB
    from confdbOfflineConverter import OfflineConverter
    self.converter = OfflineConverter(version = self.config.menu.version, database = self.config.menu.database)
    self.buildPathList()
    self.buildOptions()
    self.getSetupConfigurationFromDB()
    self.getRawConfigurationFromDB()
    self.customize()

  def getSetupConfigurationFromDB(self):
    if not self.config.setup:
        return
    ## if --setup is a python file, use directly that file as setup_cff.py
    if ".py" in self.config.setup:
        self.config.setupFile = self.config.setup.split(".py")[0]
        return
    args = ['--configName', self.config.setup ]
    args.append('--noedsources')
    args.append('--nopaths')
    for key, vals in self.options.iteritems():
      if vals:
        args.extend(('--'+key, ','.join(vals)))
    args.append('--cff')
    data, err = self.converter.query( *args )
    if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
        sys.stderr.write("%s: error while retrieving the HLT setup menu\n\n" % os.path.basename(sys.argv[0]))
        sys.stderr.write(err + "\n\n")
        sys.exit(1)
    self.config.setupFile = "setup_"+self.config.setup[1:].replace("/","_")+"_cff"
    file(self.config.setupFile+".py","w+").write("# This file is automatically generated by hltGetConfiguration.\n" + data)

  def getRawConfigurationFromDB(self):
    if self.config.menu.run:
      args = ['--runNumber', self.config.menu.run]
    else:
      args = ['--configName', self.config.menu.name ]
    args.append('--noedsources')
    for key, vals in self.options.iteritems():
      if vals:
        args.extend(('--'+key, ','.join(vals)))
    
    data, err = self.converter.query( *args )
    if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
        sys.stderr.write("%s: error while retrieving the HLT menu\n\n" % os.path.basename(sys.argv[0]))
        sys.stderr.write(err + "\n\n")
        sys.exit(1)
    self.data = data

  def getPathList(self):
    if self.config.menu.run:
      args = ['--runNumber', self.config.menu.run]
    else:
      args = ['--configName', self.config.menu.name]
    args.extend( (
      '--cff',
      '--noedsources',
      '--noes',
      '--noservices',
      '--nosequences',
      '--nomodules'
    ) )

    data, err = self.converter.query( *args )
    if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
        sys.stderr.write("%s: error while retrieving the list of paths from the HLT menu\n\n" % os.path.basename(sys.argv[0]))
        sys.stderr.write(err + "\n\n")
        sys.exit(1)
    filter = re.compile(r' *= *cms.(End)?Path.*')
    paths  = [ filter.sub('', line) for line in data.splitlines() if filter.search(line) ]
    return paths


  @staticmethod
  def expandWildcards(globs, collection):
    # expand a list of unix-style wildcards matching a given collection
    # wildcards with no matches are silently discarded
    matches = []
    for glob in globs:
      negate = ''
      if glob[0] == '-':
        negate = '-'
        glob   = glob[1:]
      # translate a unix-style glob expression into a regular expression
      filter = re.compile(r'^' + glob.replace('?', '.').replace('*', '.*').replace('[!', '[^') + r'$')
      matches.extend( negate + element for element in collection if filter.match(element) )
    return matches


  @staticmethod
  def consolidateNegativeList(elements):
    # consolidate a list of path exclusions and re-inclusions
    # the result is the list of paths to be removed from the dump
    result = set()
    for element in elements:
      if element[0] == '-':
        result.add( element )
      else:
        result.discard( '-' + element )
    return sorted( element for element in result )

  @staticmethod
  def consolidatePositiveList(elements):
    # consolidate a list of path selection and re-exclusions
    # the result is the list of paths to be included in the dump
    result = set()
    for element in elements:
      if element[0] == '-':
        result.discard( element[1:] )
      else:
        result.add( element )
    return sorted( element for element in result )


  # dump the final configuration
  def dump(self):
    self.data = self.data % self.labels
    if self.config.fragment:
      self.data = re.sub( r'\bprocess\b', 'fragment', self.data )
      self.data = re.sub( r'\bProcess\b', 'ProcessFragment', self.data )
    return self.data


  # add specific customizations
  def specificCustomize(self):
    # specific customizations now live in HLTrigger.Configuration.customizeHLTforALL.customizeHLTforAll(.,.)
    if self.config.fragment:
      self.data += """
# add specific customizations
from HLTrigger.Configuration.customizeHLTforALL import customizeHLTforAll
fragment = customizeHLTforAll(fragment,"%s")
""" % (self.config.type)
    else:
      if self.config.type=="Fake":
        prefix = "run1"
      else:
        prefix = "run2"
      _gtData = "auto:"+prefix+"_hlt_"+self.config.type
      _gtMc   = "auto:"+prefix+"_mc_" +self.config.type
      self.data += """
# add specific customizations
_customInfo = {}
_customInfo['menuType'  ]= "%s"
_customInfo['globalTags']= {}
_customInfo['globalTags'][True ] = "%s"
_customInfo['globalTags'][False] = "%s"
_customInfo['inputFiles']={}
_customInfo['inputFiles'][True]  = "file:RelVal_Raw_%s_DATA.root"
_customInfo['inputFiles'][False] = "file:RelVal_Raw_%s_MC.root"
_customInfo['maxEvents' ]=  %s
_customInfo['globalTag' ]= "%s"
_customInfo['inputFile' ]=  %s
_customInfo['realData'  ]=  %s
from HLTrigger.Configuration.customizeHLTforALL import customizeHLTforAll
%%(process)s = customizeHLTforAll(%%(process)s,"%s",_customInfo)
""" % (self.config.type,_gtData,_gtMc,self.config.type,self.config.type,self.config.events,self.config.globaltag,self.source,self.config.data,self.config.type)

    self.data += """
from HLTrigger.Configuration.customizeHLTforCMSSW import customizeHLTforCMSSW
%%(process)s = customizeHLTforCMSSW(%%(process)s,"%s")
""" % (self.config.type)

    # Eras-based customisations
    self.data += """
# Eras-based customisations
from HLTrigger.Configuration.Eras import modifyHLTforEras
modifyHLTforEras(%(process)s)
"""
    # add the user-defined customization functions, if any
    if self.config.customise:
        self.data += "\n"
        self.data += "#User-defined customization functions\n"
        for customise in self.config.customise.split(","):
            customiseValues = customise.split(".")
            if len(customiseValues)>=3: raise Exception("--customise option cannot contain more than one dot.")
            if len(customiseValues)==1:
                 customiseValues.append("customise")
            customiseValues[0] = customiseValues[0].replace("/",".")
            self.data += "from "+customiseValues[0]+" import "+customiseValues[1]+"\n"
            self.data += "process = "+customiseValues[1]+"(process)\n"

  # customize the configuration according to the options
  def customize(self):

    # adapt the source to the current scenario
    if not self.config.fragment:
      self.build_source()

    # manual override some parameters
    if self.config.type in ('HIon', ):
      if self.config.data:
        if not self.config.fragment:
          self._fix_parameter( type = 'InputTag', value = 'rawDataCollector',  replace = 'rawDataRepacker')

    # if requested, remove the HLT prescales
    self.fixPrescales()

    # if requested, override all ED/HLTfilters to always pass ("open" mode)
    self.instrumentOpenMode()

    # if requested, change all HLTTriggerTypeFilter EDFilters to accept only error events (SelectedTriggerType = 0)
    self.instrumentErrorEventType()

    # if requested, instrument the self with the modules and EndPath needed for timing studies
    self.instrumentTiming()

    # if requested, override the L1 self from the GlobalTag (Xml)
    self.overrideL1MenuXml()

    # if requested, run the L1 emulator
    self.runL1Emulator()

    # add process.load("setup_cff")
    self.loadSetupCff()

    if self.config.fragment:
      self.data += """
# dummyfy hltGetConditions in cff's
if 'hltGetConditions' in %(dict)s and 'HLTriggerFirstPath' in %(dict)s :
    %(process)s.hltDummyConditions = cms.EDFilter( "HLTBool",
        result = cms.bool( True )
    )
    %(process)s.HLTriggerFirstPath.replace(%(process)s.hltGetConditions,%(process)s.hltDummyConditions)
"""

    else:

      # override the process name and adapt the relevant filters
      self.overrideProcessName()

      # select specific Eras 
      self.addEras()

      # override the output modules to output root files
      self.overrideOutput()

      # add global options
      self.addGlobalOptions()

      # if requested or necessary, override the GlobalTag and connection strings (incl. L1!)
      self.overrideGlobalTag()

      # request summary informations from the MessageLogger
      self.updateMessageLogger()

      # replace DQMStore and DQMRootOutputModule with a configuration suitable for running offline
      self.instrumentDQM()

      # load 5.2.x JECs, until they are in the GlobalTag
#      self.loadAdditionalConditions('load 5.2.x JECs',
#        {
#          'record'  : 'JetCorrectionsRecord',
#          'tag'     : 'JetCorrectorParametersCollection_AK5Calo_2012_V8_hlt_mc',
#          'label'   : 'AK5CaloHLT',
#          'connect' : '%(connect)s/CMS_CONDITIONS'
#        }, {
#          'record'  : 'JetCorrectionsRecord',
#          'tag'     : 'JetCorrectorParametersCollection_AK5PF_2012_V8_hlt_mc',
#          'label'   : 'AK5PFHLT',
#          'connect' : '%(connect)s/CMS_CONDITIONS'
#        }, {
#          'record'  : 'JetCorrectionsRecord',
#          'tag'     : 'JetCorrectorParametersCollection_AK5PFchs_2012_V8_hlt_mc',
#          'label'   : 'AK5PFchsHLT',
#          'connect' : '%(connect)s/CMS_CONDITIONS'
#        }
#      )

    # add specific customisations
    self.specificCustomize()


  def addGlobalOptions(self):
    # add global options
    self.data += """
# limit the number of events to be processed
%%(process)s.maxEvents = cms.untracked.PSet(
    input = cms.untracked.int32( %d )
)
""" % self.config.events

    if not self.config.profiling:
      self.data += """
# enable TrigReport, TimeReport and MultiThreading
%(process)s.options = cms.untracked.PSet(
    wantSummary = cms.untracked.bool( True ),
    numberOfThreads = cms.untracked.uint32( 4 ),
    numberOfStreams = cms.untracked.uint32( 0 ),
    sizeOfStackForThreadsInKB = cms.untracked.uint32( 10*1024 )
)
"""

  def _fix_parameter(self, **args):
    """arguments:
        name:     parameter name (optional)
        type:     parameter type (look for tracked and untracked variants)
        value:    original value
        replace:  replacement value
    """
    if 'name' in args:
      self.data = re.sub(
          r'%(name)s = cms(?P<tracked>(?:\.untracked)?)\.%(type)s\( (?P<quote>["\']?)%(value)s(?P=quote)' % args,
          r'%(name)s = cms\g<tracked>.%(type)s( \g<quote>%(replace)s\g<quote>' % args,
          self.data)
    else:
      self.data = re.sub(
          r'cms(?P<tracked>(?:\.untracked)?)\.%(type)s\( (?P<quote>["\']?)%(value)s(?P=quote)' % args,
          r'cms\g<tracked>.%(type)s( \g<quote>%(replace)s\g<quote>' % args,
          self.data)


  def fixPrescales(self):
    # update the PrescaleService to match the new list of paths
    if self.options['paths']:
      if self.options['paths'][0][0] == '-':
        # drop requested paths
        for minuspath in self.options['paths']:
          path = minuspath[1:]
          self.data = re.sub(r'      cms.PSet\(  pathName = cms.string\( "%s" \),\n        prescales = cms.vuint32\( .* \)\n      \),?\n' % path, '', self.data)
      else:
        # keep requested paths
        for path in self.all_paths:
          if path not in self.options['paths']:
            self.data = re.sub(r'      cms.PSet\(  pathName = cms.string\( "%s" \),\n        prescales = cms.vuint32\( .* \)\n      \),?\n' % path, '', self.data)

    if self.config.prescale and (self.config.prescale.lower() != 'none'):
      # TO DO: check that the requested prescale column is valid
      self.data += """
# force the use of a specific HLT prescale column
if 'PrescaleService' in %(dict)s:
    %(process)s.PrescaleService.forceDefault     = True
    %(process)s.PrescaleService.lvl1DefaultLabel = '%(prescale)s'
"""


  def instrumentOpenMode(self):
    if self.config.open:
      # find all EDfilters
      filters = [ match[1] for match in re.findall(r'(process\.)?\b(\w+) = cms.EDFilter', self.data) ]
      re_sequence = re.compile( r'cms\.(Path|Sequence)\((.*)\)' )
      # remove existing 'cms.ignore' and '~' modifiers
      self.data = re_sequence.sub( lambda line: re.sub( r'cms\.ignore *\( *((process\.)?\b(\w+)) *\)', r'\1', line.group(0) ), self.data )
      self.data = re_sequence.sub( lambda line: re.sub( r'~', '', line.group(0) ), self.data )
      # wrap all EDfilters with "cms.ignore( ... )", 1000 at a time (python 2.6 complains for too-big regular expressions)
      for some in splitter(filters, 1000):
        re_filters  = re.compile( r'\b((process\.)?(' + r'|'.join(some) + r'))\b' )
        self.data = re_sequence.sub( lambda line: re_filters.sub( r'cms.ignore( \1 )', line.group(0) ), self.data )


  def instrumentErrorEventType(self):
    if self.config.errortype:
      # change all HLTTriggerTypeFilter EDFilters to accept only error events (SelectedTriggerType = 0)
      self._fix_parameter(name = 'SelectedTriggerType', type ='int32', value = '1', replace = '0')
      self._fix_parameter(name = 'SelectedTriggerType', type ='int32', value = '2', replace = '0')
      self._fix_parameter(name = 'SelectedTriggerType', type ='int32', value = '3', replace = '0')


  def overrideGlobalTag(self):
    # overwrite GlobalTag
    # the logic is:
    #   - always set the correct connection string and pfnPrefix
    #   - if a GlobalTag is specified on the command line:
    #      - override the global tag
    #      - if the GT is "auto:...", insert the code to read it from Configuration.AlCa.autoCond
    #   - if a GlobalTag is NOT  specified on the command line:
    #      - when running on data, do nothing, and keep the global tag in the menu
    #      - when running on mc, take the GT from the configuration.type

    # override the GlobalTag connection string and pfnPrefix
    text = """
# override the GlobalTag, connection string and pfnPrefix
if 'GlobalTag' in %(dict)s:
"""

    # when running on MC, override the global tag even if not specified on the command line
    if not self.config.data and not self.config.globaltag:
      if self.config.type in globalTag:
        self.config.globaltag = globalTag[self.config.type]
      else:
        self.config.globaltag = globalTag['GRun']

    # if requested, override the L1 menu from the GlobalTag (using the same connect as the GlobalTag itself)
    if self.config.l1.override:
      self.config.l1.tag    = self.config.l1.override
      self.config.l1.record = 'L1TUtmTriggerMenuRcd'
      self.config.l1.connect = '%(connect)s/CMS_CONDITIONS'
      self.config.l1.label  = ''
      if not self.config.l1.snapshotTime:
        self.config.l1.snapshotTime = '9999-12-31 23:59:59.000'
      self.config.l1cond = '%(tag)s,%(record)s,%(connect)s,%(label)s,%(snapshotTime)s' % self.config.l1.__dict__
    else:
      self.config.l1cond = None

    if self.config.globaltag or self.config.l1cond:
      text += "    from Configuration.AlCa.GlobalTag import GlobalTag as customiseGlobalTag\n"
      text += "    %(process)s.GlobalTag = customiseGlobalTag(%(process)s.GlobalTag"
      if self.config.globaltag:
        text += ", globaltag = %s"  % repr(self.config.globaltag)
      if self.config.l1cond:
        text += ", conditions = %s" % repr(self.config.l1cond)
      text += ")\n"

    text += """    %(process)s.GlobalTag.connect   = '%(connect)s/CMS_CONDITIONS'
"""
#    %(process)s.GlobalTag.pfnPrefix = cms.untracked.string('%(connect)s/')
#    for pset in %(process)s.GlobalTag.toGet.value():
#        pset.connect = pset.connect.value().replace('frontier://FrontierProd/', '%(connect)s/')
#    # fix for multi-run processing
#    %(process)s.GlobalTag.RefreshEachRun = cms.untracked.bool( False )
#    %(process)s.GlobalTag.ReconnectEachRun = cms.untracked.bool( False )
    self.data += text

  def overrideL1MenuXml(self):
    # if requested, override the GlobalTag's L1T menu from an Xml file
    if self.config.l1Xml.XmlFile:
      text = """
# override the GlobalTag's L1T menu from an Xml file
from HLTrigger.Configuration.CustomConfigs import L1XML
%%(process)s = L1XML(%%(process)s,"%s")
""" % (self.config.l1Xml.XmlFile) 
      self.data += text

  def runL1Emulator(self):
    # if requested, run the Full L1T emulator, then repack the data into a new RAW collection, to be used by the HLT
    if self.config.emulator:
      text = """
# run the Full L1T emulator, then repack the data into a new RAW collection, to be used by the HLT
from HLTrigger.Configuration.CustomConfigs import L1REPACK
%%(process)s = L1REPACK(%%(process)s,"%s")
""" % (self.config.emulator)
      self.data += text

  def overrideOutput(self):
    # override the "online" ShmStreamConsumer output modules with "offline" PoolOutputModule's
    self.data = re.sub(
      r'\b(process\.)?hltOutput(\w+) *= *cms\.OutputModule\( *"ShmStreamConsumer" *,',
      r'%(process)s.hltOutput\2 = cms.OutputModule( "PoolOutputModule",\n    fileName = cms.untracked.string( "output\2.root" ),\n    fastCloning = cms.untracked.bool( False ),\n    dataset = cms.untracked.PSet(\n        filterName = cms.untracked.string( "" ),\n        dataTier = cms.untracked.string( "RAW" )\n    ),',
      self.data
    )

    if not self.config.fragment and self.config.output == 'full':
      # add a single "keep *" output
      self.data += """
# add a single "keep *" output
%(process)s.hltOutputFULL = cms.OutputModule( "PoolOutputModule",
    fileName = cms.untracked.string( "outputFULL.root" ),
    fastCloning = cms.untracked.bool( False ),
    dataset = cms.untracked.PSet(
        dataTier = cms.untracked.string( 'RECO' ),
        filterName = cms.untracked.string( '' )
    ),
    outputCommands = cms.untracked.vstring( 'keep *' )
)
%(process)s.FULLOutput = cms.EndPath( %(process)s.hltOutputFULL )
"""
  # select specific Eras
  def addEras(self):
    if self.config.eras is None:
      return
    self.data = re.sub(r'process = cms.Process\( *"\w+"', 'from Configuration.StandardSequences.Eras import eras\n\g<0>, '+', '.join('eras.' + era for era in self.config.eras.split(',')), self.data)

  # select specific Eras
  def loadSetupCff(self):
    if self.config.setup is None:
      return
    processLine = self.data.find("\n",self.data.find("cms.Process"))
    self.data = self.data[:processLine]+'\nprocess.load("%s")'%self.config.setupFile+self.data[processLine:]
    
  # override the process name and adapt the relevant filters
  def overrideProcessName(self):
    if self.config.name is None:
      return

    # sanitise process name
    self.config.name = self.config.name.replace("_","")
    # override the process name
    quote = '[\'\"]'
    self.data = re.compile(r'^(process\s*=\s*cms\.Process\(\s*' + quote + r')\w+(' + quote + r'\s*\).*)$', re.MULTILINE).sub(r'\1%s\2' % self.config.name, self.data, 1)

    # the following was stolen and adapted from HLTrigger.Configuration.customL1THLT_Options
    self.data += """
# adapt HLT modules to the correct process name
if 'hltTrigReport' in %%(dict)s:
    %%(process)s.hltTrigReport.HLTriggerResults                    = cms.InputTag( 'TriggerResults', '', '%(name)s' )

if 'hltPreExpressCosmicsOutputSmart' in %%(dict)s:
    %%(process)s.hltPreExpressCosmicsOutputSmart.hltResults = cms.InputTag( 'TriggerResults', '', '%(name)s' )

if 'hltPreExpressOutputSmart' in %%(dict)s:
    %%(process)s.hltPreExpressOutputSmart.hltResults        = cms.InputTag( 'TriggerResults', '', '%(name)s' )

if 'hltPreDQMForHIOutputSmart' in %%(dict)s:
    %%(process)s.hltPreDQMForHIOutputSmart.hltResults       = cms.InputTag( 'TriggerResults', '', '%(name)s' )

if 'hltPreDQMForPPOutputSmart' in %%(dict)s:
    %%(process)s.hltPreDQMForPPOutputSmart.hltResults       = cms.InputTag( 'TriggerResults', '', '%(name)s' )

if 'hltPreHLTDQMResultsOutputSmart' in %%(dict)s:
    %%(process)s.hltPreHLTDQMResultsOutputSmart.hltResults  = cms.InputTag( 'TriggerResults', '', '%(name)s' )

if 'hltPreHLTDQMOutputSmart' in %%(dict)s:
    %%(process)s.hltPreHLTDQMOutputSmart.hltResults         = cms.InputTag( 'TriggerResults', '', '%(name)s' )

if 'hltPreHLTMONOutputSmart' in %%(dict)s:
    %%(process)s.hltPreHLTMONOutputSmart.hltResults         = cms.InputTag( 'TriggerResults', '', '%(name)s' )

if 'hltDQMHLTScalers' in %%(dict)s:
    %%(process)s.hltDQMHLTScalers.triggerResults                   = cms.InputTag( 'TriggerResults', '', '%(name)s' )
    %%(process)s.hltDQMHLTScalers.processname                      = '%(name)s'

if 'hltDQML1SeedLogicScalers' in %%(dict)s:
    %%(process)s.hltDQML1SeedLogicScalers.processname              = '%(name)s'
""" % self.config.__dict__


  def updateMessageLogger(self):
    # request summary informations from the MessageLogger
    self.data += """
if 'MessageLogger' in %(dict)s:
    %(process)s.MessageLogger.categories.append('TriggerSummaryProducerAOD')
    %(process)s.MessageLogger.categories.append('L1GtTrigReport')
    %(process)s.MessageLogger.categories.append('L1TGlobalSummary')
    %(process)s.MessageLogger.categories.append('HLTrigReport')
    %(process)s.MessageLogger.categories.append('FastReport')
"""


  def loadAdditionalConditions(self, comment, *conditions):
    # load additional conditions
    self.data += """
# %s
if 'GlobalTag' in %%(dict)s:
""" % comment
    for condition in conditions:
      self.data += """    %%(process)s.GlobalTag.toGet.append(
        cms.PSet(
            record  = cms.string( '%(record)s' ),
            tag     = cms.string( '%(tag)s' ),
            label   = cms.untracked.string( '%(label)s' ),
            connect = cms.untracked.string( '%(connect)s' )
        )
    )
""" % condition


  def loadCffCommand(self, module):
    # load a cfi or cff module
    if self.config.fragment:
      return 'from %s import *\n' % module
    else:
      return 'process.load( "%s" )\n' % module

  def loadCff(self, module):
    self.data += self.loadCffCommand(module)


  def overrideParameters(self, module, parameters):
    # override a module's parameter if the module is present in the configuration
    self.data += "if '%s' in %%(dict)s:\n" % module
    for (parameter, value) in parameters:
      self.data += "    %%(process)s.%s.%s = %s\n" % (module, parameter, value)
    self.data += "\n"


  def instrumentTiming(self):
    if self.config.profiling:
      # instrument the menu for profiling: remove the HLTAnalyzerEndpath, add/override the HLTriggerFirstPath, with hltGetRaw and hltGetConditions
      text = ''

      if not 'hltGetRaw' in self.data:
        # add hltGetRaw
        text += """
%(process)s.hltGetRaw = cms.EDAnalyzer( "HLTGetRaw",
    RawDataCollection = cms.InputTag( "rawDataCollector" )
)
"""

      if not 'hltGetConditions' in self.data:
        # add hltGetConditions
        text += """
%(process)s.hltGetConditions = cms.EDAnalyzer( 'EventSetupRecordDataGetter',
    verbose = cms.untracked.bool( False ),
    toGet = cms.VPSet( )
)
"""

      if not 'hltBoolFalse' in self.data:
        # add hltBoolFalse
        text += """
%(process)s.hltBoolFalse = cms.EDFilter( "HLTBool",
    result = cms.bool( False )
)
"""

      # add the definition of HLTriggerFirstPath
      # FIXME in a cff, should also update the HLTSchedule
      text += """
%(process)s.HLTriggerFirstPath = cms.Path( %(process)s.hltGetRaw + %(process)s.hltGetConditions + %(process)s.hltBoolFalse )
"""
      self.data = re.sub(r'.*cms\.(End)?Path.*', text + r'\g<0>', self.data, 1)


    # instrument the menu with the Service, EDProducer and EndPath needed for timing studies
    # FIXME in a cff, should also update the HLTSchedule
    if self.config.timing:
      self.data += """
# instrument the menu with the modules and EndPath needed for timing studies
"""

      self.data += '\n# configure the FastTimerService\n'
      if not 'FastTimerService' in self.data:
        self.loadCff('HLTrigger.Timer.FastTimerService_cfi')

      self.data += """# print a text summary at the end of the job
%(process)s.FastTimerService.printJobSummary           = True

# enable DQM plots
%(process)s.FastTimerService.enableDQM                 = True

# disable per-module DQM plots
%(process)s.FastTimerService.enableDQMbyModule         = False

# enable per-event DQM plots by lumisection
%(process)s.FastTimerService.enableDQMbyLumiSection    = True
%(process)s.FastTimerService.dqmLumiSectionsRange      = 2500

# set the time resolution of the DQM plots
%(process)s.FastTimerService.dqmTimeRange              = 1000.
%(process)s.FastTimerService.dqmTimeResolution         =    5.
%(process)s.FastTimerService.dqmPathTimeRange          =  100.
%(process)s.FastTimerService.dqmPathTimeResolution     =    0.5
%(process)s.FastTimerService.dqmModuleTimeRange        =   40.
%(process)s.FastTimerService.dqmModuleTimeResolution   =    0.2

# set the base DQM folder for the plots
%(process)s.FastTimerService.dqmPath                   = 'HLT/TimerService'
%(process)s.FastTimerService.enableDQMbyProcesses      = True
"""


  def instrumentDQM(self):
    if not self.config.hilton:
      # remove any reference to the hltDQMFileSaver
      if 'hltDQMFileSaver' in self.data:
        self.data = re.sub(r'\b(process\.)?hltDQMFileSaver \+ ', '', self.data)
        self.data = re.sub(r' \+ \b(process\.)?hltDQMFileSaver', '', self.data)
        self.data = re.sub(r'\b(process\.)?hltDQMFileSaver',     '', self.data)

      # instrument the HLT menu with DQMStore and DQMRootOutputModule suitable for running offline
      dqmstore  = "\n# load the DQMStore and DQMRootOutputModule\n"
      dqmstore += self.loadCffCommand('DQMServices.Core.DQMStore_cfi')
      dqmstore += "%(process)s.DQMStore.enableMultiThread = True\n"
      dqmstore += """
%(process)s.dqmOutput = cms.OutputModule("DQMRootOutputModule",
    fileName = cms.untracked.string("DQMIO.root")
)
"""

      empty_path = re.compile(r'.*\b(process\.)?DQMOutput = cms\.EndPath\( *\).*')
      other_path = re.compile(r'(.*\b(process\.)?DQMOutput = cms\.EndPath\()(.*)')
      if empty_path.search(self.data):
        # replace an empty DQMOutput path
        self.data = empty_path.sub(dqmstore + '\n%(process)s.DQMOutput = cms.EndPath( %(process)s.dqmOutput )\n', self.data)
      elif other_path.search(self.data):
        # prepend the dqmOutput to the DQMOutput path
        self.data = other_path.sub(dqmstore + r'\g<1> %(process)s.dqmOutput +\g<3>', self.data)
      else:
        # ceate a new DQMOutput path with the dqmOutput module
        self.data += dqmstore
        self.data += '\n%(process)s.DQMOutput = cms.EndPath( %(process)s.dqmOutput )\n'


  @staticmethod
  def dumppaths(paths):
    sys.stderr.write('Path selection:\n')
    for path in paths:
      sys.stderr.write('\t%s\n' % path)
    sys.stderr.write('\n\n')

  def buildPathList(self):
    self.all_paths = self.getPathList()

    if self.config.paths:
      # no path list was requested, dump the full table, minus unsupported / unwanted paths
      paths = self.config.paths.split(',')
    else:
      # dump only the requested paths, plus the eventual output endpaths
      paths = []

    if self.config.fragment or self.config.output in ('none', 'full'):
      # 'full' removes all outputs (same as 'none') and then adds a single "keep *" output (see the overrideOutput method)
      if self.config.paths:
        # paths are removed by default
        pass
      else:
        # drop all output endpaths
        paths.append( "-*Output" )
        paths.append( "-RatesMonitoring")
        paths.append( "-DQMHistograms")
    elif self.config.output == 'minimal':
      # drop all output endpaths but HLTDQMResultsOutput
      if self.config.paths:
        paths.append( "HLTDQMResultsOutput" )
      else:
        paths.append( "-*Output" )
        paths.append( "-RatesMonitoring")
        paths.append( "-DQMHistograms")
        paths.append( "HLTDQMResultsOutput" )
    else:
      # keep / add back all output endpaths
      if self.config.paths:
        paths.append( "*Output" )
      else:
        pass    # paths are kepy by default

    # drop unwanted paths for profiling (and timing studies)
    if self.config.profiling:
      paths.append( "-HLTriggerFirstPath" )
      paths.append( "-HLTAnalyzerEndpath" )

    # this should never be in any dump (nor online menu)
    paths.append( "-OfflineOutput" )

    # expand all wildcards
    paths = self.expandWildcards(paths, self.all_paths)

    if self.config.paths:
      # do an "additive" consolidation
      self.options['paths'] = self.consolidatePositiveList(paths)
      if not self.options['paths']:
        raise RuntimeError('Error: option "--paths %s" does not select any valid paths' % self.config.paths)
    else:
      # do a "subtractive" consolidation
      self.options['paths'] = self.consolidateNegativeList(paths)


  def buildOptions(self):
    # common configuration for all scenarios
    self.options['services'].append( "-DQM" )
    self.options['services'].append( "-FUShmDQMOutputService" )
    self.options['services'].append( "-MicroStateService" )
    self.options['services'].append( "-ModuleWebRegistry" )
    self.options['services'].append( "-TimeProfilerService" )

    # remove the DAQ modules and the online definition of the DQMStore and DQMFileSaver
    # unless a hilton-like configuration has been requested
    if not self.config.hilton:
      self.options['services'].append( "-EvFDaqDirector" )
      self.options['services'].append( "-FastMonitoringService" )
      self.options['services'].append( "-DQMStore" )
      self.options['modules'].append( "-hltDQMFileSaver" )

    if self.config.fragment:
      # extract a configuration file fragment
      self.options['essources'].append( "-GlobalTag" )
      self.options['essources'].append( "-HepPDTESSource" )
      self.options['essources'].append( "-XMLIdealGeometryESSource" )
      self.options['essources'].append( "-eegeom" )
      self.options['essources'].append( "-es_hardcode" )
      self.options['essources'].append( "-magfield" )

      self.options['esmodules'].append( "-AutoMagneticFieldESProducer" )
      self.options['esmodules'].append( "-SlaveField0" )
      self.options['esmodules'].append( "-SlaveField20" )
      self.options['esmodules'].append( "-SlaveField30" )
      self.options['esmodules'].append( "-SlaveField35" )
      self.options['esmodules'].append( "-SlaveField38" )
      self.options['esmodules'].append( "-SlaveField40" )
      self.options['esmodules'].append( "-VBF0" )
      self.options['esmodules'].append( "-VBF20" )
      self.options['esmodules'].append( "-VBF30" )
      self.options['esmodules'].append( "-VBF35" )
      self.options['esmodules'].append( "-VBF38" )
      self.options['esmodules'].append( "-VBF40" )
      self.options['esmodules'].append( "-CSCGeometryESModule" )
      self.options['esmodules'].append( "-CaloGeometryBuilder" )
      self.options['esmodules'].append( "-CaloTowerHardcodeGeometryEP" )
      self.options['esmodules'].append( "-CastorHardcodeGeometryEP" )
      self.options['esmodules'].append( "-DTGeometryESModule" )
      self.options['esmodules'].append( "-EcalBarrelGeometryEP" )
      self.options['esmodules'].append( "-EcalElectronicsMappingBuilder" )
      self.options['esmodules'].append( "-EcalEndcapGeometryEP" )
      self.options['esmodules'].append( "-EcalLaserCorrectionService" )
      self.options['esmodules'].append( "-EcalPreshowerGeometryEP" )
      self.options['esmodules'].append( "-HcalHardcodeGeometryEP" )
      self.options['esmodules'].append( "-HcalTopologyIdealEP" )
      self.options['esmodules'].append( "-MuonNumberingInitialization" )
      self.options['esmodules'].append( "-ParametrizedMagneticFieldProducer" )
      self.options['esmodules'].append( "-RPCGeometryESModule" )
      self.options['esmodules'].append( "-SiStripGainESProducer" )
      self.options['esmodules'].append( "-SiStripRecHitMatcherESProducer" )
      self.options['esmodules'].append( "-SiStripQualityESProducer" )
      self.options['esmodules'].append( "-StripCPEfromTrackAngleESProducer" )
      self.options['esmodules'].append( "-TrackerDigiGeometryESModule" )
      self.options['esmodules'].append( "-TrackerGeometricDetESModule" )
      self.options['esmodules'].append( "-VolumeBasedMagneticFieldESProducer" )
      self.options['esmodules'].append( "-ZdcHardcodeGeometryEP" )
      self.options['esmodules'].append( "-hcal_db_producer" )
      self.options['esmodules'].append( "-L1GtTriggerMaskAlgoTrigTrivialProducer" )
      self.options['esmodules'].append( "-L1GtTriggerMaskTechTrigTrivialProducer" )
      self.options['esmodules'].append( "-hltESPEcalTrigTowerConstituentsMapBuilder" )
      self.options['esmodules'].append( "-hltESPGlobalTrackingGeometryESProducer" )
      self.options['esmodules'].append( "-hltESPMuonDetLayerGeometryESProducer" )
      self.options['esmodules'].append( "-hltESPTrackerRecoGeometryESProducer" )
      self.options['esmodules'].append( "-trackerTopology" )

      self.options['esmodules'].append( "-CaloTowerGeometryFromDBEP" )
      self.options['esmodules'].append( "-CastorGeometryFromDBEP" )
      self.options['esmodules'].append( "-EcalBarrelGeometryFromDBEP" )
      self.options['esmodules'].append( "-EcalEndcapGeometryFromDBEP" )
      self.options['esmodules'].append( "-EcalPreshowerGeometryFromDBEP" )
      self.options['esmodules'].append( "-HcalGeometryFromDBEP" )
      self.options['esmodules'].append( "-ZdcGeometryFromDBEP" )
      self.options['esmodules'].append( "-XMLFromDBSource" )
      self.options['esmodules'].append( "-sistripconn" )

      self.options['services'].append( "-MessageLogger" )

      self.options['psets'].append( "-maxEvents" )
      self.options['psets'].append( "-options" )

    if self.config.fragment or (self.config.prescale and (self.config.prescale.lower() == 'none')):
      self.options['services'].append( "-PrescaleService" )

    if self.config.fragment or self.config.timing:
      self.options['services'].append( "-FastTimerService" )


  def append_filenames(self, name, filenames):
    if len(filenames) > 255:
      token_open  = "( *("
      token_close = ") )"
    else:
      token_open  = "("
      token_close = ")"

    self.data += "    %s = cms.untracked.vstring%s\n" % (name, token_open)
    for line in filenames:
      self.data += "        '%s',\n" % line
    self.data += "    %s,\n" % (token_close)


  def expand_filenames(self, input):
    # check if the input is a dataset or a list of files
    if input[0:8] == 'dataset:':
      from dasFileQuery import dasFileQuery
      # extract the dataset name, and use DAS to fine the list of LFNs
      dataset = input[8:]
      files = dasFileQuery(dataset)
    else:
      # assume a comma-separated list of input files
      files = self.config.input.split(',')
    return files

  def build_source(self):
    if self.config.input:
      # if a dataset or a list of input files was given, use it
      self.source = self.expand_filenames(self.config.input)
    elif self.config.online:
      # online we always run on data
      self.source = [ "file:/tmp/InputCollection.root" ]
    elif self.config.data:
      # offline we can run on data...
      self.source = [ "file:RelVal_Raw_%s_DATA.root" % self.config.type ]
    else:
      # ...or on mc
      self.source = [ "file:RelVal_Raw_%s_MC.root" % self.config.type ]

    if self.config.parent:
      # if a dataset or a list of input files was given for the parent data, use it
      self.parent = self.expand_filenames(self.config.parent)

    self.data += """
%(process)s.source = cms.Source( "PoolSource",
"""
    self.append_filenames("fileNames", self.source)
    if (self.parent):
      self.append_filenames("secondaryFileNames", self.parent)
    self.data += """\
Beispiel #7
0
class HLTProcess(object):
    def __init__(self, configuration):
        self.config = configuration
        self.data = None
        self.source = []
        self.parent = []

        self.options = {
            'essources': [],
            'esmodules': [],
            'modules': [],
            'sequences': [],
            'services': [],
            'paths': [],
            'psets': [],
            'blocks': [],
        }

        self.labels = {}
        if self.config.fragment:
            self.labels['process'] = 'fragment.'
            self.labels['dict'] = 'fragment.__dict__'
        else:
            self.labels['process'] = 'process.'
            self.labels['dict'] = 'process.__dict__'

        if self.config.online:
            self.labels[
                'connect'] = 'frontier://(proxyurl=http://localhost:3128)(serverurl=http://localhost:8000/FrontierOnProd)(serverurl=http://localhost:8000/FrontierOnProd)(retrieve-ziplevel=0)'
        else:
            self.labels['connect'] = 'frontier://FrontierProd'

        if self.config.prescale and (self.config.prescale.lower() != 'none'):
            self.labels['prescale'] = self.config.prescale

        # get the configuration from ConfdB
        from confdbOfflineConverter import OfflineConverter
        self.converter = OfflineConverter(database=self.config.menu.db)
        self.buildPathList()
        self.buildOptions()
        self.getRawConfigurationFromDB()
        self.customize()

    def getRawConfigurationFromDB(self):
        if self.config.menu.run:
            args = ['--runNumber', self.config.menu.run]
        else:
            args = ['--configName', self.config.menu.name]
        args.append('--noedsources')
        for key, vals in self.options.iteritems():
            if vals:
                args.extend(('--' + key, ','.join(vals)))

        data, err = self.converter.query(*args)
        if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
            print "%s: error while retriving the HLT menu" % os.path.basename(
                sys.argv[0])
            print
            print err
            print
            sys.exit(1)
        self.data = data

    def getPathList(self):
        if self.config.menu.run:
            args = ['--runNumber', self.config.menu.run]
        else:
            args = ['--configName', self.config.menu.name]
        args.extend(('--cff', '--noedsources', '--noes', '--noservices',
                     '--nosequences', '--nomodules'))

        data, err = self.converter.query(*args)
        if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
            print "%s: error while retriving the list of paths from the HLT menu" % os.path.basename(
                sys.argv[0])
            print
            print err
            print
            sys.exit(1)
        filter = re.compile(r' *= *cms.(End)?Path.*')
        paths = [
            filter.sub('', line) for line in data.splitlines()
            if filter.search(line)
        ]
        return paths

    @staticmethod
    def expandWildcards(globs, collection):
        # expand a list of unix-style wildcards matching a given collection
        # wildcards with no matches are silently discarded
        matches = []
        for glob in globs:
            negate = ''
            if glob[0] == '-':
                negate = '-'
                glob = glob[1:]
            # translate a unix-style glob expression into a regular expression
            filter = re.compile(
                r'^' +
                glob.replace('?', '.').replace('*', '.*').replace('[!', '[^') +
                r'$')
            matches.extend(negate + element for element in collection
                           if filter.match(element))
        return matches

    @staticmethod
    def consolidateNegativeList(elements):
        # consolidate a list of path exclusions and re-inclusions
        # the result is the list of paths to be removed from the dump
        result = set()
        for element in elements:
            if element[0] == '-':
                result.add(element)
            else:
                result.discard('-' + element)
        return sorted(element for element in result)

    @staticmethod
    def consolidatePositiveList(elements):
        # consolidate a list of path selection and re-exclusions
        # the result is the list of paths to be included in the dump
        result = set()
        for element in elements:
            if element[0] == '-':
                result.discard(element[1:])
            else:
                result.add(element)
        return sorted(element for element in result)

    # dump the final configuration
    def dump(self):
        self.data = self.data % self.labels
        if self.config.fragment:
            self.data = re.sub(r'\bprocess\b', 'fragment', self.data)
            self.data = re.sub(r'\bProcess\b', 'ProcessFragment', self.data)
        return self.data

    # add specific customizations
    def specificCustomize(self):
        # specific customizations now live in HLTrigger.Configuration.customizeHLTforALL.customizeHLTforAll(.,.)
        if self.config.fragment:
            self.data += """
# add specific customizations
from HLTrigger.Configuration.customizeHLTforALL import customizeHLTforAll
fragment = customizeHLTforAll(fragment)
"""
        else:
            if self.config.type == "Fake":
                prefix = "run1"
            else:
                prefix = "run2"
            _gtData = "auto:" + prefix + "_hlt_" + self.config.type
            _gtMc = "auto:" + prefix + "_mc_" + self.config.type
            self.data += """
# add specific customizations
_customInfo = {}
_customInfo['menuType'  ]= "%s"
_customInfo['globalTags']= {}
_customInfo['globalTags'][True ] = "%s"
_customInfo['globalTags'][False] = "%s"
_customInfo['inputFiles']={}
_customInfo['inputFiles'][True]  = "file:RelVal_Raw_%s_DATA.root"
_customInfo['inputFiles'][False] = "file:RelVal_Raw_%s_MC.root"
_customInfo['maxEvents' ]=  %s
_customInfo['globalTag' ]= "%s"
_customInfo['inputFile' ]=  %s
_customInfo['realData'  ]=  %s
from HLTrigger.Configuration.customizeHLTforALL import customizeHLTforAll
process = customizeHLTforAll(process,_customInfo)
""" % (self.config.type, _gtData, _gtMc, self.config.type, self.config.type,
            self.config.events, self.config.globaltag, self.source,
            self.config.data)

    # customize the configuration according to the options
    def customize(self):

        # adapt the source to the current scenario
        if not self.config.fragment:
            self.build_source()

        # manual override some parameters
        if self.config.type in ('HIon', ):
            if self.config.data:
                if not self.config.fragment:
                    self._fix_parameter(type='InputTag',
                                        value='rawDataCollector',
                                        replace='rawDataRepacker')

#    if self.config.type in ('HIon', ):
#      self.data += """
## Disable HF Noise filters in HIon menu
#if 'hltHfreco' in %(dict)s:
#    %(process)shltHfreco.setNoiseFlags = cms.bool( False )
#"""
#    else:
#      self.data += """
## Enable HF Noise filters in non-HIon menu
#if 'hltHfreco' in %(dict)s:
#    %(process)shltHfreco.setNoiseFlags = cms.bool( True )
#"""

#    self.data += """
## untracked parameters with NO default in the code
#if 'hltHcalDataIntegrityMonitor' in %(dict)s:
#    %(process)shltHcalDataIntegrityMonitor.RawDataLabel = cms.untracked.InputTag("rawDataCollector")
#if 'hltDt4DSegments' in %(dict)s:
#    %(process)shltDt4DSegments.debug = cms.untracked.bool( False )
#"""

# if requested, remove the HLT prescales
        self.fixPrescales()

        # if requested, override all ED/HLTfilters to always pass ("open" mode)
        self.instrumentOpenMode()

        # if requested, change all HLTTriggerTypeFilter EDFilters to accept only error events (SelectedTriggerType = 0)
        self.instrumentErrorEventType()

        # if requested, instrument the self with the modules and EndPath needed for timing studies
        self.instrumentTiming()

        if self.config.fragment:
            self.data += """
# dummyfy hltGetConditions in cff's
if 'hltGetConditions' in %(dict)s and 'HLTriggerFirstPath' in %(dict)s :
    %(process)shltDummyConditions = cms.EDFilter( "HLTBool",
        result = cms.bool( True )
    )
    %(process)sHLTriggerFirstPath.replace(%(process)shltGetConditions,%(process)shltDummyConditions)
"""

        else:

            if self.config.type not in ('Fake', ):
                if '50ns' in self.config.type:
                    self.data += """
# load 2015 Run-2 L1 Menu for 50ns
from L1Trigger.Configuration.customise_overwriteL1Menu import L1Menu_Collisions2015_50ns_v1 as loadL1Menu
process = loadL1Menu(process)
"""
                elif 'HIon' in self.config.type:
                    self.data += """
# load 2015 Run-2 L1 Menu for HIon
from L1Trigger.Configuration.customise_overwriteL1Menu import L1Menu_CollisionsHeavyIons2015_v0 as loadL1Menu
process = loadL1Menu(process)
"""
                else:
                    self.data += """
# load 2015 Run-2 L1 Menu for 25ns (default for GRun, PIon)
from L1Trigger.Configuration.customise_overwriteL1Menu import L1Menu_Collisions2015_25ns_v2 as loadL1menu
process = loadL1menu(process)
"""

            # override the process name and adapt the relevant filters
            self.overrideProcessName()

            # override the output modules to output root files
            self.overrideOutput()

            # add global options
            self.addGlobalOptions()

            # if requested or necessary, override the GlobalTag and connection strings (incl. L1!)
            self.overrideGlobalTag()

            # if requested, override the L1 self from the GlobalTag (Xml)
            self.overrideL1MenuXml()

            # if requested, add snippet to run on new L1 skim
            self.switchToNewL1Skim()

            # if requested, run (part of) the L1 emulator
            self.runL1Emulator()

            # request summary informations from the MessageLogger
            self.updateMessageLogger()

            # replace DQMStore and DQMRootOutputModule with a configuration suitable for running offline
            self.instrumentDQM()

            # load 5.2.x JECs, until they are in the GlobalTag


#      self.loadAdditionalConditions('load 5.2.x JECs',
#        {
#          'record'  : 'JetCorrectionsRecord',
#          'tag'     : 'JetCorrectorParametersCollection_AK5Calo_2012_V8_hlt_mc',
#          'label'   : 'AK5CaloHLT',
#          'connect' : '%(connect)s/CMS_CONDITIONS'
#        }, {
#          'record'  : 'JetCorrectionsRecord',
#          'tag'     : 'JetCorrectorParametersCollection_AK5PF_2012_V8_hlt_mc',
#          'label'   : 'AK5PFHLT',
#          'connect' : '%(connect)s/CMS_CONDITIONS'
#        }, {
#          'record'  : 'JetCorrectionsRecord',
#          'tag'     : 'JetCorrectorParametersCollection_AK5PFchs_2012_V8_hlt_mc',
#          'label'   : 'AK5PFchsHLT',
#          'connect' : '%(connect)s/CMS_CONDITIONS'
#        }
#      )

# add specific customisations
        self.specificCustomize()

    def addGlobalOptions(self):
        # add global options
        self.data += """
# limit the number of events to be processed
%%(process)smaxEvents = cms.untracked.PSet(
    input = cms.untracked.int32( %d )
)
""" % self.config.events

        if not self.config.profiling:
            self.data += """
# enable the TrigReport and TimeReport
%(process)soptions = cms.untracked.PSet(
    wantSummary = cms.untracked.bool( True )
)
"""

    def _fix_parameter(self, **args):
        """arguments:
        name:     parameter name (optional)
        type:     parameter type (look for tracked and untracked variants)
        value:    original value
        replace:  replacement value
    """
        if 'name' in args:
            self.data = re.sub(
                r'%(name)s = cms(?P<tracked>(?:\.untracked)?)\.%(type)s\( (?P<quote>["\']?)%(value)s(?P=quote)'
                % args,
                r'%(name)s = cms\g<tracked>.%(type)s( \g<quote>%(replace)s\g<quote>'
                % args, self.data)
        else:
            self.data = re.sub(
                r'cms(?P<tracked>(?:\.untracked)?)\.%(type)s\( (?P<quote>["\']?)%(value)s(?P=quote)'
                % args,
                r'cms\g<tracked>.%(type)s( \g<quote>%(replace)s\g<quote>' %
                args, self.data)

    def fixPrescales(self):
        # update the PrescaleService to match the new list of paths
        if self.options['paths']:
            if self.options['paths'][0][0] == '-':
                # drop requested paths
                for minuspath in self.options['paths']:
                    path = minuspath[1:]
                    self.data = re.sub(
                        r'      cms.PSet\(  pathName = cms.string\( "%s" \),\n        prescales = cms.vuint32\( .* \)\n      \),?\n'
                        % path, '', self.data)
            else:
                # keep requested paths
                for path in self.all_paths:
                    if path not in self.options['paths']:
                        self.data = re.sub(
                            r'      cms.PSet\(  pathName = cms.string\( "%s" \),\n        prescales = cms.vuint32\( .* \)\n      \),?\n'
                            % path, '', self.data)

        if self.config.prescale and (self.config.prescale.lower() != 'none'):
            # TO DO: check that the requested prescale column is valid
            self.data += """
# force the use of a specific HLT prescale column
if 'PrescaleService' in %(dict)s:
    %(process)sPrescaleService.forceDefault     = True
    %(process)sPrescaleService.lvl1DefaultLabel = '%(prescale)s'
"""

    def instrumentOpenMode(self):
        if self.config.open:
            # find all EDfilters
            filters = [
                match[1] for match in re.findall(
                    r'(process\.)?\b(\w+) = cms.EDFilter', self.data)
            ]
            re_sequence = re.compile(r'cms\.(Path|Sequence)\((.*)\)')
            # remove existing 'cms.ignore' and '~' modifiers
            self.data = re_sequence.sub(
                lambda line: re.sub(
                    r'cms\.ignore *\( *((process\.)?\b(\w+)) *\)', r'\1',
                    line.group(0)), self.data)
            self.data = re_sequence.sub(
                lambda line: re.sub(r'~', '', line.group(0)), self.data)
            # wrap all EDfilters with "cms.ignore( ... )", 1000 at a time (python 2.6 complains for too-big regular expressions)
            for some in splitter(filters, 1000):
                re_filters = re.compile(r'\b((process\.)?(' + r'|'.join(some) +
                                        r'))\b')
                self.data = re_sequence.sub(
                    lambda line: re_filters.sub(r'cms.ignore( \1 )',
                                                line.group(0)), self.data)

    def instrumentErrorEventType(self):
        if self.config.errortype:
            # change all HLTTriggerTypeFilter EDFilters to accept only error events (SelectedTriggerType = 0)
            self._fix_parameter(name='SelectedTriggerType',
                                type='int32',
                                value='1',
                                replace='0')
            self._fix_parameter(name='SelectedTriggerType',
                                type='int32',
                                value='2',
                                replace='0')
            self._fix_parameter(name='SelectedTriggerType',
                                type='int32',
                                value='3',
                                replace='0')

    def overrideGlobalTag(self):
        # overwrite GlobalTag
        # the logic is:
        #   - always set the correct connection string and pfnPrefix
        #   - if a GlobalTag is specified on the command line:
        #      - override the global tag
        #      - if the GT is "auto:...", insert the code to read it from Configuration.AlCa.autoCond
        #   - if a GlobalTag is NOT  specified on the command line:
        #      - when running on data, do nothing, and keep the global tag in the menu
        #      - when running on mc, take the GT from the configuration.type

        # override the GlobalTag connection string and pfnPrefix
        text = """
# override the GlobalTag, connection string and pfnPrefix
if 'GlobalTag' in %(dict)s:
"""

        # when running on MC, override the global tag even if not specified on the command line
        if not self.config.data and not self.config.globaltag:
            if self.config.type in globalTag:
                self.config.globaltag = globalTag[self.config.type]
            else:
                self.config.globaltag = globalTag['GRun']

        # if requested, override the L1 menu from the GlobalTag (using the same connect as the GlobalTag itself)
        if self.config.l1.override:
            self.config.l1.record = 'L1GtTriggerMenuRcd'
            self.config.l1.label = ''
            self.config.l1.tag = self.config.l1.override
            if not self.config.l1.connect:
                self.config.l1.connect = '%(connect)s/CMS_CONDITIONS'
            self.config.l1cond = '%(tag)s,%(record)s,%(connect)s' % self.config.l1.__dict__
        else:
            self.config.l1cond = None

        if self.config.globaltag or self.config.l1cond:
            text += "    from Configuration.AlCa.GlobalTag_condDBv2 import GlobalTag as customiseGlobalTag\n"
            text += "    %(process)sGlobalTag = customiseGlobalTag(%(process)sGlobalTag"
            if self.config.globaltag:
                text += ", globaltag = %s" % repr(self.config.globaltag)
            if self.config.l1cond:
                text += ", conditions = %s" % repr(self.config.l1cond)
            text += ")\n"

        text += """    %(process)sGlobalTag.connect   = '%(connect)s/CMS_CONDITIONS'
    %(process)sGlobalTag.pfnPrefix = cms.untracked.string('%(connect)s/')
    for pset in process.GlobalTag.toGet.value():
        pset.connect = pset.connect.value().replace('frontier://FrontierProd/', '%(connect)s/')
    # fix for multi-run processing
    %(process)sGlobalTag.RefreshEachRun = cms.untracked.bool( False )
    %(process)sGlobalTag.ReconnectEachRun = cms.untracked.bool( False )
"""
        self.data += text

    def overrideL1MenuXml(self):
        # if requested, override the L1 menu from the GlobalTag (Xml file)
        if self.config.l1Xml.XmlFile:
            text = """
# override the L1 menu from an Xml file
%%(process)sl1GtTriggerMenuXml = cms.ESProducer("L1GtTriggerMenuXmlProducer",
  TriggerMenuLuminosity = cms.string('%(LumiDir)s'),
  DefXmlFile = cms.string('%(XmlFile)s'),
  VmeXmlFile = cms.string('')
)
%%(process)sL1GtTriggerMenuRcdSource = cms.ESSource("EmptyESSource",
  recordName = cms.string('L1GtTriggerMenuRcd'),
  iovIsRunNotTime = cms.bool(True),
  firstValid = cms.vuint32(1)
)
%%(process)ses_prefer_l1GtParameters = cms.ESPrefer('L1GtTriggerMenuXmlProducer','l1GtTriggerMenuXml')
"""
            self.data += text % self.config.l1Xml.__dict__

    def runL1EmulatorGT(self):
        # if requested, run (part of) the L1 emulator, then repack the data into a new RAW collection, to be used by the HLT
        if not self.config.emulator:
            return

        if self.config.emulator != 'gt':
            # only the GT emulator is currently supported
            return

        # run the L1 GT emulator, then repack the data into a new RAW collection, to be used by the HLT
        text = """
# run the L1 GT emulator, then repack the data into a new RAW collection, to be used by the HLT
"""
        if self.config.fragment:
            # FIXME in a cff, should also update the HLTSchedule
            text += "import Configuration.StandardSequences.SimL1EmulatorRepack_GT_cff\n"
        else:
            text += "process.load( 'Configuration.StandardSequences.SimL1EmulatorRepack_GT_cff' )\n"

        if not 'hltBoolFalse' in self.data:
            # add hltBoolFalse
            text += """
%(process)shltBoolFalse = cms.EDFilter( "HLTBool",
    result = cms.bool( False )
)
"""
        text += "process.L1Emulator = cms.Path( process.SimL1Emulator + process.hltBoolFalse )\n\n"

        self.data = re.sub(r'.*cms\.(End)?Path.*', text + r'\g<0>', self.data,
                           1)

    def runL1Emulator(self):
        # if requested, run (part of) the L1 emulator
        if self.config.emulator:
            # FIXME this fragment used "process" explicitly
            emulator = {'RawToDigi': '', 'CustomL1T': '', 'CustomHLT': ''}

            if self.config.data:
                emulator['RawToDigi'] = 'RawToDigi_Data_cff'
            else:
                emulator['RawToDigi'] = 'RawToDigi_cff'

            if self.config.emulator == 'gt':
                emulator['CustomL1T'] = 'customiseL1GtEmulatorFromRaw'
                emulator['CustomHLT'] = 'switchToSimGtDigis'
            elif self.config.emulator == 'gct,gt':
                emulator['CustomL1T'] = 'customiseL1CaloAndGtEmulatorsFromRaw'
                emulator['CustomHLT'] = 'switchToSimGctGtDigis'
            elif self.config.emulator == 'gmt,gt':
                # XXX currently unsupported
                emulator['CustomL1T'] = 'customiseL1MuonAndGtEmulatorsFromRaw'
                emulator['CustomHLT'] = 'switchToSimGmtGtDigis'
            elif self.config.emulator in ('gmt,gct,gt', 'gct,gmt,gt', 'all'):
                emulator['CustomL1T'] = 'customiseL1EmulatorFromRaw'
                emulator['CustomHLT'] = 'switchToSimGmtGctGtDigis'
            elif self.config.emulator in ('stage1,gt'):
                emulator['CustomL1T'] = 'customiseL1EmulatorFromRaw'
                emulator['CustomHLT'] = 'switchToSimStage1Digis'
            else:
                # unsupported argument, default to running the whole emulator
                emulator['CustomL1T'] = 'customiseL1EmulatorFromRaw'
                emulator['CustomHLT'] = 'switchToSimGmtGctGtDigis'

            self.data += """
# customize the L1 emulator to run %(CustomL1T)s with HLT to %(CustomHLT)s
process.load( 'Configuration.StandardSequences.%(RawToDigi)s' )
process.load( 'Configuration.StandardSequences.SimL1Emulator_cff' )
import L1Trigger.Configuration.L1Trigger_custom
#
""" % emulator

            if (self.config.emulator).find("stage1") > -1:
                self.data += """
# 2015 Run2 emulator
import L1Trigger.L1TCalorimeter.L1TCaloStage1_customForHLT
process = L1Trigger.L1TCalorimeter.L1TCaloStage1_customForHLT.%(CustomL1T)s( process )
""" % emulator
            else:
                self.data += """
# Run1 Emulator
process = L1Trigger.Configuration.L1Trigger_custom.%(CustomL1T)s( process )
""" % emulator

            self.data += """
#
process = L1Trigger.Configuration.L1Trigger_custom.customiseResetPrescalesAndMasks( process )
# customize the HLT to use the emulated results
import HLTrigger.Configuration.customizeHLTforL1Emulator
process = HLTrigger.Configuration.customizeHLTforL1Emulator.switchToL1Emulator( process )
process = HLTrigger.Configuration.customizeHLTforL1Emulator.%(CustomHLT)s( process )
""" % emulator

    def switchToNewL1Skim(self):
        # add snippet to switch to new L1 skim files
        if self.config.l1skim:
            self.data += """
# Customize the menu to use information from new L1 emulator in the L1 skim files
process.hltL2MuonSeeds.GMTReadoutCollection = cms.InputTag("simGmtDigis::L1SKIM" )
process.hltL1extraParticles.muonSource = cms.InputTag("simGmtDigis::L1SKIM" )
for module in process.__dict__.itervalues():
  if isinstance(module, cms._Module):
    for parameter in module.__dict__.itervalues():
      if isinstance(parameter, cms.InputTag):
        if parameter.moduleLabel == 'hltGtDigis':
          parameter.moduleLabel = "gtDigisFromSkim"
        elif parameter.moduleLabel == 'hltL1GtObjectMap':
          parameter.moduleLabel = "gtDigisFromSkim"
        elif parameter.moduleLabel == 'hltGctDigis':
          parameter.moduleLabel ="simCaloStage1LegacyFormatDigis"
"""

    def overrideOutput(self):
        # override the "online" ShmStreamConsumer output modules with "offline" PoolOutputModule's
        self.data = re.sub(
            r'\b(process\.)?hltOutput(\w+) *= *cms\.OutputModule\( *"ShmStreamConsumer" *,',
            r'%(process)shltOutput\2 = cms.OutputModule( "PoolOutputModule",\n    fileName = cms.untracked.string( "output\2.root" ),\n    fastCloning = cms.untracked.bool( False ),\n    dataset = cms.untracked.PSet(\n        filterName = cms.untracked.string( "" ),\n        dataTier = cms.untracked.string( "RAW" )\n    ),',
            self.data)

        if not self.config.fragment and self.config.output == 'full':
            # add a single "keep *" output
            self.data += """
# add a single "keep *" output
%(process)shltOutputFULL = cms.OutputModule( "PoolOutputModule",
    fileName = cms.untracked.string( "outputFULL.root" ),
    fastCloning = cms.untracked.bool( False ),
    dataset = cms.untracked.PSet(
        dataTier = cms.untracked.string( 'RECO' ),
        filterName = cms.untracked.string( '' )
    ),
    outputCommands = cms.untracked.vstring( 'keep *' )
)
%(process)sFULLOutput = cms.EndPath( %(process)shltOutputFULL )
"""

    # override the process name and adapt the relevant filters
    def overrideProcessName(self):
        if self.config.name is None:
            return

        # sanitise process name
        self.config.name = self.config.name.replace("_", "")
        # override the process name
        quote = '[\'\"]'
        self.data = re.compile(
            r'^(process\s*=\s*cms\.Process\(\s*' + quote + r')\w+(' + quote +
            r'\s*\).*)$', re.MULTILINE).sub(r'\1%s\2' % self.config.name,
                                            self.data, 1)

        # the following was stolen and adapted from HLTrigger.Configuration.customL1THLT_Options
        self.data += """
# adapt HLT modules to the correct process name
if 'hltTrigReport' in %%(dict)s:
    %%(process)shltTrigReport.HLTriggerResults                    = cms.InputTag( 'TriggerResults', '', '%(name)s' )

if 'hltPreExpressCosmicsOutputSmart' in %%(dict)s:
    %%(process)shltPreExpressCosmicsOutputSmart.hltResults = cms.InputTag( 'TriggerResults', '', '%(name)s' )

if 'hltPreExpressOutputSmart' in %%(dict)s:
    %%(process)shltPreExpressOutputSmart.hltResults        = cms.InputTag( 'TriggerResults', '', '%(name)s' )

if 'hltPreDQMForHIOutputSmart' in %%(dict)s:
    %%(process)shltPreDQMForHIOutputSmart.hltResults       = cms.InputTag( 'TriggerResults', '', '%(name)s' )

if 'hltPreDQMForPPOutputSmart' in %%(dict)s:
    %%(process)shltPreDQMForPPOutputSmart.hltResults       = cms.InputTag( 'TriggerResults', '', '%(name)s' )

if 'hltPreHLTDQMResultsOutputSmart' in %%(dict)s:
    %%(process)shltPreHLTDQMResultsOutputSmart.hltResults  = cms.InputTag( 'TriggerResults', '', '%(name)s' )

if 'hltPreHLTDQMOutputSmart' in %%(dict)s:
    %%(process)shltPreHLTDQMOutputSmart.hltResults         = cms.InputTag( 'TriggerResults', '', '%(name)s' )

if 'hltPreHLTMONOutputSmart' in %%(dict)s:
    %%(process)shltPreHLTMONOutputSmart.hltResults         = cms.InputTag( 'TriggerResults', '', '%(name)s' )

if 'hltDQMHLTScalers' in %%(dict)s:
    %%(process)shltDQMHLTScalers.triggerResults                   = cms.InputTag( 'TriggerResults', '', '%(name)s' )
    %%(process)shltDQMHLTScalers.processname                      = '%(name)s'

if 'hltDQML1SeedLogicScalers' in %%(dict)s:
    %%(process)shltDQML1SeedLogicScalers.processname              = '%(name)s'
""" % self.config.__dict__

    def updateMessageLogger(self):
        # request summary informations from the MessageLogger
        self.data += """
if 'MessageLogger' in %(dict)s:
    %(process)sMessageLogger.categories.append('TriggerSummaryProducerAOD')
    %(process)sMessageLogger.categories.append('L1GtTrigReport')
    %(process)sMessageLogger.categories.append('HLTrigReport')
    %(process)sMessageLogger.categories.append('FastReport')
"""

    def loadAdditionalConditions(self, comment, *conditions):
        # load additional conditions
        self.data += """
# %s
if 'GlobalTag' in %%(dict)s:
""" % comment
        for condition in conditions:
            self.data += """    %%(process)sGlobalTag.toGet.append(
        cms.PSet(
            record  = cms.string( '%(record)s' ),
            tag     = cms.string( '%(tag)s' ),
            label   = cms.untracked.string( '%(label)s' ),
            connect = cms.untracked.string( '%(connect)s' )
        )
    )
""" % condition

    def loadCffCommand(self, module):
        # load a cfi or cff module
        if self.config.fragment:
            return 'from %s import *\n' % module
        else:
            return 'process.load( "%s" )\n' % module

    def loadCff(self, module):
        self.data += self.loadCffCommand(module)

    def overrideParameters(self, module, parameters):
        # override a module's parameter if the module is present in the configuration
        self.data += "if '%s' in %%(dict)s:\n" % module
        for (parameter, value) in parameters:
            self.data += "    %%(process)s%s.%s = %s\n" % (module, parameter,
                                                           value)
        self.data += "\n"

    def instrumentTiming(self):
        if self.config.profiling:
            # instrument the menu for profiling: remove the HLTAnalyzerEndpath, add/override the HLTriggerFirstPath, with hltGetRaw and hltGetConditions
            text = ''

            if not 'hltGetRaw' in self.data:
                # add hltGetRaw
                text += """
%(process)shltGetRaw = cms.EDAnalyzer( "HLTGetRaw",
    RawDataCollection = cms.InputTag( "rawDataCollector" )
)
"""

            if not 'hltGetConditions' in self.data:
                # add hltGetConditions
                text += """
%(process)shltGetConditions = cms.EDAnalyzer( 'EventSetupRecordDataGetter',
    verbose = cms.untracked.bool( False ),
    toGet = cms.VPSet( )
)
"""

            if not 'hltBoolFalse' in self.data:
                # add hltBoolFalse
                text += """
%(process)shltBoolFalse = cms.EDFilter( "HLTBool",
    result = cms.bool( False )
)
"""

            # add the definition of HLTriggerFirstPath
            # FIXME in a cff, should also update the HLTSchedule
            text += """
%(process)sHLTriggerFirstPath = cms.Path( %(process)shltGetRaw + %(process)shltGetConditions + %(process)shltBoolFalse )
"""
            self.data = re.sub(r'.*cms\.(End)?Path.*', text + r'\g<0>',
                               self.data, 1)

        # instrument the menu with the Service, EDProducer and EndPath needed for timing studies
        # FIXME in a cff, should also update the HLTSchedule
        if self.config.timing:
            self.data += """
# instrument the menu with the modules and EndPath needed for timing studies
"""

            if not 'FastTimerService' in self.data:
                self.data += '\n# configure the FastTimerService\n'
                self.loadCff('HLTrigger.Timer.FastTimerService_cfi')
            else:
                self.data += '\n# configure the FastTimerService\n'

            self.data += """# this is currently ignored in CMSSW 7.x, always using the real time clock
%(process)sFastTimerService.useRealTimeClock          = True
# enable specific features
%(process)sFastTimerService.enableTimingPaths         = True
%(process)sFastTimerService.enableTimingModules       = True
%(process)sFastTimerService.enableTimingExclusive     = True
# print a text summary at the end of the job
%(process)sFastTimerService.enableTimingSummary       = True
# skip the first path (disregard the time spent loading event and conditions data)
%(process)sFastTimerService.skipFirstPath             = True
# enable DQM plots
%(process)sFastTimerService.enableDQM                 = True
# enable most per-path DQM plots
%(process)sFastTimerService.enableDQMbyPathActive     = True
%(process)sFastTimerService.enableDQMbyPathTotal      = True
%(process)sFastTimerService.enableDQMbyPathOverhead   = False
%(process)sFastTimerService.enableDQMbyPathDetails    = True
%(process)sFastTimerService.enableDQMbyPathCounters   = True
%(process)sFastTimerService.enableDQMbyPathExclusive  = True
# disable per-module DQM plots
%(process)sFastTimerService.enableDQMbyModule         = False
%(process)sFastTimerService.enableDQMbyModuleType     = False
# enable per-event DQM sumary plots
%(process)sFastTimerService.enableDQMSummary          = True
# enable per-event DQM plots by lumisection
%(process)sFastTimerService.enableDQMbyLumiSection    = True
%(process)sFastTimerService.dqmLumiSectionsRange      = 2500
# set the time resolution of the DQM plots
%(process)sFastTimerService.dqmTimeRange              = 1000.
%(process)sFastTimerService.dqmTimeResolution         =    5.
%(process)sFastTimerService.dqmPathTimeRange          =  100.
%(process)sFastTimerService.dqmPathTimeResolution     =    0.5
%(process)sFastTimerService.dqmModuleTimeRange        =   40.
%(process)sFastTimerService.dqmModuleTimeResolution   =    0.2
# set the base DQM folder for the plots
%(process)sFastTimerService.dqmPath                   = 'HLT/TimerService'
%(process)sFastTimerService.enableDQMbyProcesses      = True
"""

    def instrumentDQM(self):
        if not self.config.hilton:
            # remove any reference to the hltDQMFileSaver
            if 'hltDQMFileSaver' in self.data:
                self.data = re.sub(r'\b(process\.)?hltDQMFileSaver \+ ', '',
                                   self.data)
                self.data = re.sub(r' \+ \b(process\.)?hltDQMFileSaver', '',
                                   self.data)
                self.data = re.sub(r'\b(process\.)?hltDQMFileSaver', '',
                                   self.data)

            # instrument the HLT menu with DQMStore and DQMRootOutputModule suitable for running offline
            dqmstore = "\n# load the DQMStore and DQMRootOutputModule\n"
            dqmstore += self.loadCffCommand('DQMServices.Core.DQMStore_cfi')
            dqmstore += "%(process)sDQMStore.enableMultiThread = True\n"
            dqmstore += """
%(process)sdqmOutput = cms.OutputModule("DQMRootOutputModule",
    fileName = cms.untracked.string("DQMIO.root")
)
"""

            empty_path = re.compile(
                r'.*\b(process\.)?DQMOutput = cms\.EndPath\( *\).*')
            other_path = re.compile(
                r'(.*\b(process\.)?DQMOutput = cms\.EndPath\()(.*)')
            if empty_path.search(self.data):
                # replace an empty DQMOutput path
                self.data = empty_path.sub(
                    dqmstore +
                    '\n%(process)sDQMOutput = cms.EndPath( %(process)sdqmOutput )\n',
                    self.data)
            elif other_path.search(self.data):
                # prepend the dqmOutput to the DQMOutput path
                self.data = other_path.sub(
                    dqmstore + r'\g<1> %(process)sdqmOutput +\g<3>', self.data)
            else:
                # ceate a new DQMOutput path with the dqmOutput module
                self.data += dqmstore
                self.data += '\n%(process)sDQMOutput = cms.EndPath( %(process)sdqmOutput )\n'

    @staticmethod
    def dumppaths(paths):
        sys.stderr.write('Path selection:\n')
        for path in paths:
            sys.stderr.write('\t%s\n' % path)
        sys.stderr.write('\n\n')

    def buildPathList(self):
        self.all_paths = self.getPathList()

        if self.config.paths:
            # no path list was requested, dump the full table, minus unsupported / unwanted paths
            paths = self.config.paths.split(',')
        else:
            # dump only the requested paths, plus the eventual output endpaths
            paths = []

        if self.config.fragment or self.config.output in ('none', 'full'):
            # 'full' removes all outputs (same as 'none') and then adds a single "keep *" output (see the overrideOutput method)
            if self.config.paths:
                # paths are removed by default
                pass
            else:
                # drop all output endpaths
                paths.append("-*Output")
        elif self.config.output == 'minimal':
            # drop all output endpaths but HLTDQMResultsOutput
            if self.config.paths:
                paths.append("HLTDQMResultsOutput")
            else:
                paths.append("-*Output")
                paths.append("HLTDQMResultsOutput")
        else:
            # keep / add back all output endpaths
            if self.config.paths:
                paths.append("*Output")
            else:
                pass  # paths are kepy by default

        # drop unwanted paths for profiling (and timing studies)
        if self.config.profiling:
            paths.append("-HLTriggerFirstPath")
            paths.append("-HLTAnalyzerEndpath")

        # this should never be in any dump (nor online menu)
        paths.append("-OfflineOutput")

        # expand all wildcards
        paths = self.expandWildcards(paths, self.all_paths)

        if self.config.paths:
            # do an "additive" consolidation
            self.options['paths'] = self.consolidatePositiveList(paths)
            if not self.options['paths']:
                raise RuntimeError(
                    'Error: option "--paths %s" does not select any valid paths'
                    % self.config.paths)
        else:
            # do a "subtractive" consolidation
            self.options['paths'] = self.consolidateNegativeList(paths)

    def buildOptions(self):
        # common configuration for all scenarios
        self.options['services'].append("-DQM")
        self.options['services'].append("-FUShmDQMOutputService")
        self.options['services'].append("-MicroStateService")
        self.options['services'].append("-ModuleWebRegistry")
        self.options['services'].append("-TimeProfilerService")

        # remove the DAQ modules and the online definition of the DQMStore and DQMFileSaver
        # unless a hilton-like configuration has been requested
        if not self.config.hilton:
            self.options['services'].append("-EvFDaqDirector")
            self.options['services'].append("-FastMonitoringService")
            self.options['services'].append("-DQMStore")
            self.options['modules'].append("-hltDQMFileSaver")

        if self.config.fragment:
            # extract a configuration file fragment
            self.options['essources'].append("-GlobalTag")
            self.options['essources'].append("-HepPDTESSource")
            self.options['essources'].append("-XMLIdealGeometryESSource")
            self.options['essources'].append("-eegeom")
            self.options['essources'].append("-es_hardcode")
            self.options['essources'].append("-magfield")

            self.options['esmodules'].append("-AutoMagneticFieldESProducer")
            self.options['esmodules'].append("-SlaveField0")
            self.options['esmodules'].append("-SlaveField20")
            self.options['esmodules'].append("-SlaveField30")
            self.options['esmodules'].append("-SlaveField35")
            self.options['esmodules'].append("-SlaveField38")
            self.options['esmodules'].append("-SlaveField40")
            self.options['esmodules'].append("-VBF0")
            self.options['esmodules'].append("-VBF20")
            self.options['esmodules'].append("-VBF30")
            self.options['esmodules'].append("-VBF35")
            self.options['esmodules'].append("-VBF38")
            self.options['esmodules'].append("-VBF40")
            self.options['esmodules'].append("-CSCGeometryESModule")
            self.options['esmodules'].append("-CaloGeometryBuilder")
            self.options['esmodules'].append("-CaloTowerHardcodeGeometryEP")
            self.options['esmodules'].append("-CastorHardcodeGeometryEP")
            self.options['esmodules'].append("-DTGeometryESModule")
            self.options['esmodules'].append("-EcalBarrelGeometryEP")
            self.options['esmodules'].append("-EcalElectronicsMappingBuilder")
            self.options['esmodules'].append("-EcalEndcapGeometryEP")
            self.options['esmodules'].append("-EcalLaserCorrectionService")
            self.options['esmodules'].append("-EcalPreshowerGeometryEP")
            self.options['esmodules'].append("-HcalHardcodeGeometryEP")
            self.options['esmodules'].append("-HcalTopologyIdealEP")
            self.options['esmodules'].append("-MuonNumberingInitialization")
            self.options['esmodules'].append(
                "-ParametrizedMagneticFieldProducer")
            self.options['esmodules'].append("-RPCGeometryESModule")
            self.options['esmodules'].append("-SiStripGainESProducer")
            self.options['esmodules'].append("-SiStripRecHitMatcherESProducer")
            self.options['esmodules'].append("-SiStripQualityESProducer")
            self.options['esmodules'].append(
                "-StripCPEfromTrackAngleESProducer")
            self.options['esmodules'].append("-TrackerDigiGeometryESModule")
            self.options['esmodules'].append("-TrackerGeometricDetESModule")
            self.options['esmodules'].append(
                "-VolumeBasedMagneticFieldESProducer")
            self.options['esmodules'].append("-ZdcHardcodeGeometryEP")
            self.options['esmodules'].append("-hcal_db_producer")
            self.options['esmodules'].append(
                "-L1GtTriggerMaskAlgoTrigTrivialProducer")
            self.options['esmodules'].append(
                "-L1GtTriggerMaskTechTrigTrivialProducer")
            self.options['esmodules'].append(
                "-hltESPEcalTrigTowerConstituentsMapBuilder")
            self.options['esmodules'].append(
                "-hltESPGlobalTrackingGeometryESProducer")
            self.options['esmodules'].append(
                "-hltESPMuonDetLayerGeometryESProducer")
            self.options['esmodules'].append(
                "-hltESPTrackerRecoGeometryESProducer")
            self.options['esmodules'].append("-trackerTopology")

            self.options['esmodules'].append("-CaloTowerGeometryFromDBEP")
            self.options['esmodules'].append("-CastorGeometryFromDBEP")
            self.options['esmodules'].append("-EcalBarrelGeometryFromDBEP")
            self.options['esmodules'].append("-EcalEndcapGeometryFromDBEP")
            self.options['esmodules'].append("-EcalPreshowerGeometryFromDBEP")
            self.options['esmodules'].append("-HcalGeometryFromDBEP")
            self.options['esmodules'].append("-ZdcGeometryFromDBEP")
            self.options['esmodules'].append("-XMLFromDBSource")
            self.options['esmodules'].append("-sistripconn")

            self.options['services'].append("-MessageLogger")

            self.options['psets'].append("-maxEvents")
            self.options['psets'].append("-options")

        if self.config.fragment or (self.config.prescale and
                                    (self.config.prescale.lower() == 'none')):
            self.options['services'].append("-PrescaleService")

        if self.config.fragment or self.config.timing:
            self.options['services'].append("-FastTimerService")

    def append_filenames(self, name, filenames):
        if len(filenames) > 255:
            token_open = "( *("
            token_close = ") )"
        else:
            token_open = "("
            token_close = ")"

        self.data += "    %s = cms.untracked.vstring%s\n" % (name, token_open)
        for line in filenames:
            self.data += "        '%s',\n" % line
        self.data += "    %s,\n" % (token_close)

    def expand_filenames(self, input):
        # check if the input is a dataset or a list of files
        if input[0:8] == 'dataset:':
            from dasFileQuery import dasFileQuery
            # extract the dataset name, and use DAS to fine the list of LFNs
            dataset = input[8:]
            files = dasFileQuery(dataset)
        else:
            # assume a comma-separated list of input files
            files = self.config.input.split(',')
        return files

    def build_source(self):
        if self.config.input:
            # if a dataset or a list of input files was given, use it
            self.source = self.expand_filenames(self.config.input)
        elif self.config.online:
            # online we always run on data
            self.source = ["file:/tmp/InputCollection.root"]
        elif self.config.data:
            # offline we can run on data...
            self.source = ["file:RelVal_Raw_%s_DATA.root" % self.config.type]
        else:
            # ...or on mc
            self.source = ["file:RelVal_Raw_%s_MC.root" % self.config.type]

        if self.config.parent:
            # if a dataset or a list of input files was given for the parent data, use it
            self.parent = self.expand_filenames(self.config.parent)

        self.data += """
%(process)ssource = cms.Source( "PoolSource",
"""
        self.append_filenames("fileNames", self.source)
        if (self.parent):
            self.append_filenames("secondaryFileNames", self.parent)
        self.data += """\