Ejemplo n.º 1
0
 def __init__(self, transientAttrs = set(), toPublicAttrs = set(), **kw):
   "Initialize streamer and declare transient variables."
   Logger.__init__(self, kw)
   self.transientAttrs = set(transientAttrs) | {'_readVersion',}
   self.toPublicAttrs = set(toPublicAttrs)
   from RingerCore.Configure import checkForUnusedVars
   checkForUnusedVars( kw, self._logger.warning )
Ejemplo n.º 2
0
 def __init__(self, transientAttrs = set(), toPublicAttrs = set(), **kw):
   "Initialize streamer and declare transient variables."
   Logger.__init__(self, kw)
   self.transientAttrs = set(transientAttrs) | {'_readVersion',}
   self.toPublicAttrs = set(toPublicAttrs)
   from RingerCore.util import checkForUnusedVars
   checkForUnusedVars( kw, self._logger.warning )
Ejemplo n.º 3
0
 def __init__(self, localCluster=None, **kw):
     Logger.__init__(self, kw)
     if localCluster in (None, NotSet):
         localCluster = clusterManagerConf()
     self.localCluster = AvailableManager.retrieve(localCluster)
     if self.localCluster is ClusterManager.LSF:
         self.prefix = LSFArgumentParser.prefix
         prog = 'bsub'
     elif self.localCluster is ClusterManager.PBS:
         self.prefix = PBSJobArgumentParser.prefix
         from socket import gethostname
         prog = 'qsub'
         # Work on environment
         if "service1" in gethostname():
             import re
             THEANO_FLAGS = os.getenv('THEANO_FLAGS')
             m = re.search('cxx=([^,]*)(,*)?', THEANO_FLAGS)
             if m:
                 CXX_THEANO_FLAGS = m.group(1)
                 os.environ['THEANO_FLAGS'] = re.sub(
                     'cxx=[^,]*', "cxx=ssh service1 " + CXX_THEANO_FLAGS,
                     THEANO_FLAGS)
                 self._debug("Set THEANO_FLAGS to: %s",
                             os.getenv('THEANO_FLAGS'))
             else:
                 self._warning("THEANO FLAGS continues equal to: %s",
                               os.getenv('THEANO_FLAGS'))
     else:
         self._fatal(
             "Not implemented LocalClusterNamespace for cluster manager %s",
             AvailableManager.retrieve(self.localCluster),
             NotImplementedError)
     JobSubmitNamespace.__init__(self, prog=prog)
Ejemplo n.º 4
0
 def __setstate__(self, d):
   """
     Add logger to object if it doesn't have one:
   """
   v = d.pop('ignoreAttrs')
   self.__dict__['ignoreAttrs'] = [re.compile(s) for s in v]
   Logger.__setstate__(self,d)
Ejemplo n.º 5
0
  def __init__(self, *args, **kw):
    """
      Set values using min:incr:max and matlabFlag as explained in class
      documentation.
    """
    from RingerCore.Configure import checkForUnusedVars
    Logger.__init__(self, kw)
    self._matlabFlag = kw.pop('matlabFlag', False )
    checkForUnusedVars( kw, self._warning )
    del kw

    if len(args) > 3: 
      raise ValueError("Input more than 3 values, format should be [min=0:[incr=1:]]max")

    if len(args) == 0 or (args[0] in ([],tuple(),)): 
      # Create empty looping bounds
      self._vec = [0,-1,1]
      return

    if isinstance(args[0], LoopingBounds ):
      self._vec = args[0]._vec
      oldFlag = self._matlabFlag
      self._matlabFlag = args[0]._matlabFlag
      if oldFlag != self._matlabFlag:
        if self._matlabFlag:
          self.turnMatlabFlagOn()
        else:
          self.turnMatlabFlagOff()
      return
    elif isinstance(args[0], list ):
      self._vec = args[0]
    else:
      self._vec = list(args)

    if len(self._vec) == 1:
      self._vec.append( self._vec[0] + 1 )
      if not self._matlabFlag:
        self._vec[1] -= 1
      self._vec[0] = 1 if self._matlabFlag else 0
    elif len(self._vec) == 2:
      if self._matlabFlag: 
        self._vec[1] += 1
    else:
      tmp = self._vec[1]
      if self._matlabFlag:
        if tmp > 0:
          self._vec[1] = self._vec[2] + 1
        else:
          self._vec[1] = self._vec[2] - 1
      else:
        self._vec[1] = self._vec[2]
      self._vec[2] = tmp

    if len(self._vec) == 3 and self._vec[2] == 0:
      raise ValueError("Attempted to create looping bounds without increment.")
Ejemplo n.º 6
0
 def __init__(self, **kw):
   Logger.__init__(self, kw)
   self.__useFortran = kw.pop( 'useFortran', False )
   self.order          = 'F' if self.__useFortran else 'C'
   self.pdim           = 0   if self.__useFortran else 1
   self.odim           = 1   if self.__useFortran else 0
   self.fp_dtype       = np.dtype( kw.pop( 'fp_dtype',       np.float64 ) )
   self.int_dtype      = np.dtype( kw.pop( 'int_dtype',      np.int64   ) )
   self.scounter_dtype = np.dtype( kw.pop( 'scounter_dtype', np.uint8   ) )
   self.flag_dtype     = np.dtype( kw.pop( 'flag_dtype',     np.int8    ) )
   checkForUnusedVars(kw)
Ejemplo n.º 7
0
 def __init__(self, **kw):
     Logger.__init__(self, kw)
     self.__useFortran = kw.pop('useFortran', False)
     self.order = 'F' if self.__useFortran else 'C'
     self.pdim = 0 if self.__useFortran else 1
     self.odim = 1 if self.__useFortran else 0
     self.fp_dtype = np.dtype(kw.pop('fp_dtype', np.float64))
     self.int_dtype = np.dtype(kw.pop('int_dtype', np.int64))
     self.scounter_dtype = np.dtype(kw.pop('scounter_dtype', np.uint8))
     self.flag_dtype = np.dtype(kw.pop('flag_dtype', np.int8))
     checkForUnusedVars(kw)
Ejemplo n.º 8
0
    def __init__(self, outputFile, **kw):
        Logger.__init__(self, kw)
        if not outputFile.endswith(".root"):
            outputFile += ".root"
        # Create TFile object to hold everything
        from ROOT import TFile

        self._file = TFile(outputFile, "recreate")
        self._currentDir = ""
        self._objects = dict()
        self._dirs = list()
Ejemplo n.º 9
0
  def __init__(self, *args, **kw):
    """
      Set values using min:incr:max and matlabFlag as explained in class
      documentation.
    """
    from RingerCore.util import checkForUnusedVars
    Logger.__init__(self, kw)
    self._matlabFlag = kw.pop('matlabFlag', False )
    checkForUnusedVars( kw, self._logger.warning )
    del kw

    if len(args) > 3: 
      raise ValueError("Input more than 3 values, format should be [min=0:[incr=1:]]max")

    if len(args) == 0: 
      raise ValueError("No input. Format should be [min=0,[incr=1,]]max")

    if isinstance(args[0], LoopingBounds ):
      self._vec = args[0]._vec
      oldFlag = self._matlabFlag
      self._matlabFlag = args[0]._matlabFlag
      if oldFlag != self._matlabFlag:
        if self._matlabFlag:
          self.turnMatlabFlagOn()
        else:
          self.turnMatlabFlagOff()
      return
    elif isinstance(args[0], list ):
      self._vec = args[0]
    else:
      self._vec = list(args)

    if len(self._vec) == 1:
      self._vec.append( self._vec[0] + 1 )
      if not self._matlabFlag:
        self._vec[1] -= 1
      self._vec[0] = 1 if self._matlabFlag else 0
    elif len(self._vec) == 2:
      if self._matlabFlag: 
        self._vec[1] += 1
    else:
      tmp = self._vec[1]
      if self._matlabFlag:
        if tmp > 0:
          self._vec[1] = self._vec[2] + 1
        else:
          self._vec[1] = self._vec[2] - 1
      else:
        self._vec[1] = self._vec[2]
      self._vec[2] = tmp

    if len(self._vec) == 3 and self._vec[2] == 0:
      raise ValueError("Attempted to create looping bounds without increment.")
Ejemplo n.º 10
0
def sourceEnvFile():
  """
    Emulate source new_env_file.sh on python environment.
  """
  try:
    from RingerCore.Logger import Logger
    logger = Logger.getModuleLogger(__name__)
    import os, sys
    global loadedEnvFile
    if not loadedEnvFile:
      with open(os.path.expandvars('$ROOTCOREBIN/../FastNetTool/cmt/new_env_file.sh'),'r') as f:
        lines = f.readlines()
        lineparser = re.compile(r'test "\$\{(?P<shellVar>[A-Z1-9]*)#\*(?P<addedPath>\S+)\}" = "\$\{(?P=shellVar)\}" && export (?P=shellVar)=\$(?P=shellVar):(?P=addedPath) || true')
        for line in lines:
          m = lineparser.match(line)
          if m:
            shellVar = m.group('shellVar')
            if shellVar != 'PYTHONPATH':
              continue
            addedPath = os.path.expandvars(m.group('addedPath'))
            if not addedPath:
              logger.warning("Couldn't retrieve added path on line \"%s\".", line)
              continue
            if not os.path.exists(addedPath):
              logger.warning("Couldn't find following path \"%s\".", addedPath)
              continue
            if not addedPath in os.environ[shellVar]:
              sys.path.append(addedPath)
              logger.info("Successfully added path: \"%s\".", line)
      loadedEnvFile=True
  except IOError:
    raise RuntimeError("Cannot find new_env_file.sh, did you forget to set environment or compile the package?")
Ejemplo n.º 11
0
 def __init__(self, ignoreAttrs = set(), toProtectedAttrs = set(), ignoreRawChildren = False, **kw ):
   """
     -> ignoreAttrs: not consider this attributes on the dictionary values.
     -> toProtectedAttrs: change public attributes to protected or private
       attributes. That is, suppose the dictionary value is 'val' and the class
       value should be _val or __val, then add toProtectedAttrs = ['_val'] or
       '__val'.
     -> ignoreRawChildren: Do not attempt to conver raw children to higher level object.
   """
   Logger.__init__(self, kw)
   ignoreAttrs = list(set(ignoreAttrs) | RawDictCnv.baseAttrs)
   import re
   self.ignoreAttrs = [re.compile(ignoreAttr) for ignoreAttr in ignoreAttrs]
   self.toProtectedAttrs = set(toProtectedAttrs)
   self.ignoreRawChildren = ignoreRawChildren
   from RingerCore.Configure import checkForUnusedVars
   checkForUnusedVars( kw, self._logger.warning )
Ejemplo n.º 12
0
 def __init__(self, ignoreAttrs = set(), toProtectedAttrs = set(), ignoreRawChildren = False, **kw ):
   """
     -> ignoreAttrs: not consider this attributes on the dictionary values.
     -> toProtectedAttrs: change public attributes to protected or private
       attributes. That is, suppose the dictionary value is 'val' and the class
       value should be _val or __val, then add toProtectedAttrs = ['_val'] or
       '__val'.
     -> ignoreRawChildren: Do not attempt to conver raw children to higher level object.
   """
   Logger.__init__(self, kw)
   ignoreAttrs = list(set(ignoreAttrs) | RawDictCnv.baseAttrs)
   import re
   self.ignoreAttrs = [re.compile(ignoreAttr) for ignoreAttr in ignoreAttrs]
   self.toProtectedAttrs = set(toProtectedAttrs)
   self.ignoreRawChildren = ignoreRawChildren
   from RingerCore.util import checkForUnusedVars
   checkForUnusedVars( kw, self._logger.warning )
Ejemplo n.º 13
0
 def __init__(self, d={}, **kw):
     if None in self._contextManager._acceptedTypes:
         self._contextManager._acceptedTypes = TexObject,
     if (not isinstance(self, TexObjectCollection)
             and not hasattr(self, '_preamble')
             and not hasattr(self, '_enclosure')
             and not hasattr(self, '_body')
             and not hasattr(self, '_footer')
             and not hasattr(self, '_appendix')):
         raise TexException(
             self, 'Class %s does not write any tex code.' %
             self.__class__.__name__)
     d.update(kw)
     Logger.__init__(self, d)
     if hasattr(self, '_body'):
         self._body = formatTex(self._body, retrieve_kw(d, 'textWidth', 80))
     self._stream = kw.pop('stream', tss)
     self._keywords = {
         key: val
         for key, val in d.iteritems() if not key.startswith('_')
     }
     self._keywords.update({
         key: val
         for key, val in self.__dict__.iteritems()
         if not key.startswith('_')
     })
     if 'star' in self._keywords and self._keywords['star']:
         self._keywords['star'] = '*'
     else:
         self._keywords['star'] = ''
     if hasattr(self, '_assertVars'):
         for key in self._assertVars:
             if not key in self._keywords:
                 raise TexException(self, "Assert var %s failed." % key)
     gcc.set(self)
     self._contextManaged = d.pop('_contextManaged', True)
     self._context = self._contextManager()
     self._isInContext = self._context is not None
     if (self._isInContext
             and isinstance(self._context, TexObjectCollection)
             and self._contextManaged):
         self._context += self
     if self._isInContext:
         self._stream = self._context._stream
Ejemplo n.º 14
0
 def __init__(self, prog=None, **kw):
     Logger.__init__(self, kw)
     if prog is not None:
         self.prog = prog
     else:
         try:
             self.prog = self.__class__.prog
         except AttributeError:
             raise AttributeError(
                 "Not specified class (%s) prog attribute!" %
                 self.__class__.__name__)
     if not hasattr(self, 'prefix'):
         try:
             self.prefix = self.__class__.ParserClass.prefix
         except AttributeError:
             raise AttributeError(
                 "Not specified class (%s) ParserClass attribute!" %
                 self.__class__.__name__)
     argparse.Namespace.__init__(self)
     self.fcn = os.system
  def __init__(self, tes, **kw):
    Logger.__init__( self, **kw )
    import logging
    self.baseDir        = kw.pop('baseDir','')
    self.bucketName     = kw.pop('bucketName','')
    self.pidName        = kw.pop('pidName','tight')
    self._level         = kw.pop('level',logging.INFO)
    self.appName        = kw.pop('appName','TrigAnalysisTool')
    checkForUnusedVars(kw)
    del kw

    self._logger.info('Start wrapper between python and c++ core.')
    
    self._trigAnalysisTool = ROOT.TrigAnalysisTool(self.appName,
                                                   self.bucketName,
                                                   self.baseDir,
                                                   self.pidName,
                                                   self._level)
    for te in tes:
      self._trigAnalysisTool.push_back( te() )

    self._logger.info('TrigAnalysisTool has been created.' )
Ejemplo n.º 16
0
def watchLock(filename):
    logger = Logger.getModuleLogger("watchLock")
    lockFileName = os.path.join(
        os.path.join(os.path.dirname(filename),
                     '.' + os.path.basename(filename) + '.lock'))
    firstMsg = True
    while os.path.exists(lockFileName):
        if firstMsg:
            logger.warning("Waiting other process to unlock file %s...",
                           lockFileName)
            firstMsg = False
        sleep(1)
    lockFile = LockFile(lockFileName)
    return lockFile
Ejemplo n.º 17
0
 def __getstate__(self):
   """
     Makes logger invisible for pickle
   """
   odict = Logger.__getstate__(self)
   #def getStr(i):
   #  if isinstance(i, re
   if 'ignoreAttrs' in odict:
     s = odict['ignoreAttrs']
     def getStr(c):
       try:
         return  c.pattern
       except AttributeError:
         return c
     odict['ignoreAttrs'] = [getStr(v) for v in s]
   return odict
Ejemplo n.º 18
0
def sourceEnvFile():
    """
    Emulate source new_env_file.sh on python environment.
  """
    try:
        from RingerCore.Logger import Logger
        logger = Logger.getModuleLogger(__name__)
        import os, sys
        global loadedEnvFile
        if not loadedEnvFile:
            with open(
                    os.path.expandvars(
                        '$ROOTCOREBIN/../FastNetTool/cmt/new_env_file.sh'),
                    'r') as f:
                lines = f.readlines()
                lineparser = re.compile(
                    r'test "\$\{(?P<shellVar>[A-Z1-9]*)#\*(?P<addedPath>\S+)\}" = "\$\{(?P=shellVar)\}" && export (?P=shellVar)=\$(?P=shellVar):(?P=addedPath) || true'
                )
                for line in lines:
                    m = lineparser.match(line)
                    if m:
                        shellVar = m.group('shellVar')
                        if shellVar != 'PYTHONPATH':
                            continue
                        addedPath = os.path.expandvars(m.group('addedPath'))
                        if not addedPath:
                            logger.warning(
                                "Couldn't retrieve added path on line \"%s\".",
                                line)
                            continue
                        if not os.path.exists(addedPath):
                            logger.warning(
                                "Couldn't find following path \"%s\".",
                                addedPath)
                            continue
                        if not addedPath in os.environ[shellVar]:
                            sys.path.append(addedPath)
                            logger.info("Successfully added path: \"%s\".",
                                        line)
            loadedEnvFile = True
    except IOError:
        raise RuntimeError(
            "Cannot find new_env_file.sh, did you forget to set environment or compile the package?"
        )
Ejemplo n.º 19
0
__all__ = ['RawDictStreamable', 'RawDictStreamer', 'RawDictCnv', 'mangle_attr',
           'LoggerStreamable', 'LoggerRawDictStreamer', 'checkAttrOrSetDefault',
           'isRawDictFormat', 'retrieveRawDict']

from RingerCore.Logger import Logger

mLogger = Logger.getModuleLogger( __name__ )

def mangle_attr(source, attr):
  """
  Simulate python private attritubutes mangling. Taken from:
  http://stackoverflow.com/a/7789483/1162884
  """
  # return public attrs unchanged
  if not attr.startswith("__") or attr.endswith("__") or '.' in attr:
    return attr
  # if source is an object, get the class
  if not hasattr(source, "__bases__"):
    source = source.__class__
  # mangle attr
  return "_%s%s" % (source.__name__.lstrip("_"), attr)

class RawDictStreamer( Logger ):
  """
  This is the default streamer class, responsible of converting python classes
  to raw dictionaries.
  """

  def __init__(self, transientAttrs = set(), toPublicAttrs = set(), **kw):
    "Initialize streamer and declare transient variables."
    Logger.__init__(self, kw)
Ejemplo n.º 20
0
 def _print_message(self, message, file=None):
     from RingerCore.Logger import Logger
     logger = Logger.getModuleLogger(self.__class__.__name__)
     if message:
         logger.info('\n' + message)
Ejemplo n.º 21
0
 def __init__(self, d = {}, **kw): 
   Logger.__init__(self, d, **kw)
Ejemplo n.º 22
0
 def __init__(self, outputFile):
     Logger.__init__(self)
     from RingerCore.FileIO import ensureExtension
     if not outputFile:
         raise TexException(self, 'Cannot stream to empty file path.')
     self.outputFile = ensureExtension(outputFile, self._outputExtension)
Ejemplo n.º 23
0
# Make outputs not usable by user
parser.add_argument('--outputs', action='store_const',
    required = False, default = '"tuningData*"', const = '"tuningData*"', 
    dest = 'grid_outputs',
    help = argparse.SUPPRESS )
# Force secondary to be reusable:
parser.add_argument('--reusableSecondary', action='store_const',
    required = False, default = 'BKG', const = 'BKG', dest = 'grid_reusableSecondary',
    help = """Allow reuse secondary dataset.""")

emptyArgumentsPrintHelp(parser)

# Retrieve parser args:
args = parser.parse_args( namespace = TuningToolGridNamespace('prun') )
from RingerCore.Logger import Logger
mainLogger = Logger.getModuleLogger( __name__, args.output_level )
# Treat special argument
if len(args.reference) > 2:
  mainLogger.fatal("--reference set to multiple values: %r" % args.reference, ValueError)
if len(args.reference) is 1:
  args.reference.append( args.reference[0] )
if args.operation != 'Offline' and not args.treePath:
  mainLogger.fatal("If operation is not set to Offline, it is needed to set the TreePath manually.")

if ( len(args.inDS_BKG) > 1 or len(args.grid_inDS) > 1):
  mainLogger.fatal("Cannot use multiple datasets in this version.", NotImplementedError)

from subprocess import check_output

# TODO For now we only support one container for signal and one for background.
# We need to change it so that we can input multiple containers. This can be
Ejemplo n.º 24
0
 def __init__(self, prog = 'prun', **kw):
   Logger.__init__( self, kw )
   LoggerNamespace.__init__( self, **kw )
   self.prog = prog
Ejemplo n.º 25
0
__all__ = ['RawDictStreamable', 'RawDictStreamer', 'RawDictCnv', 'mangle_attr',
           'LoggerStreamable', 'LoggerRawDictStreamer', 'checkAttrOrSetDefault',
           'isRawDictFormat', 'retrieveRawDict']

from RingerCore.Logger import Logger

mLogger = Logger.getModuleLogger( __name__ )

def mangle_attr(source, attr):
  """
  Simulate python private attritubutes mangling. Taken from:
  http://stackoverflow.com/a/7789483/1162884
  """
  # return public attrs unchanged
  if not attr.startswith("__") or attr.endswith("__") or '.' in attr:
    return attr
  # if source is an object, get the class
  if not hasattr(source, "__bases__"):
    source = source.__class__
  # mangle attr
  return "_%s%s" % (source.__name__.lstrip("_"), attr)

class RawDictStreamer( Logger ):
  """
  This is the default streamer class. Overload this method to deal with
  special cases.
  """

  def __init__(self, transientAttrs = set(), toPublicAttrs = set(), **kw):
    "Initialize streamer and declare transient variables."
    Logger.__init__(self, kw)
Ejemplo n.º 26
0
 def __init__(self, d = {}, **kw): 
   Logger.__init__(self, d, **kw)
Ejemplo n.º 27
0
 def __init__(self, **kw):
     self._choice = NotSet
     if not hasattr(self, 'name'):
         self.name = self.__class__.__name__.lstrip('_')
         self.name = self.name.replace('Configure', '')
     Logger.__init__(self, kw, logName=self.name)
Ejemplo n.º 28
0
    dest = 'grid_outputs',
    help = argparse.SUPPRESS )
# Force secondary to be reusable:
parser.add_argument('--reusableSecondary', action='store_const',
    required = False, default = 'BKG', const = 'BKG', dest = 'grid_reusableSecondary',
    help = """Allow reuse secondary dataset.""")

import sys
if len(sys.argv)==1:
  parser.print_help()
  sys.exit(1)

# Retrieve parser args:
args = parser.parse_args( namespace = TuningToolGridNamespace('prun') )
from RingerCore.Logger import Logger
mainLogger = Logger.getModuleLogger( __name__, args.output_level )
# Treat special argument
if len(args.reference) > 2:
  mainLogger.fatal("--reference set to multiple values: %r" % args.reference, ValueError)
if len(args.reference) is 1:
  args.reference.append( args.reference[0] )
if args.operation != 'Offline' and not args.treePath:
  mainLogger.fatal("If operation is not set to Offline, it is needed to set the TreePath manually.")

if ( len(args.inDS_BKG) > 1 or len(args.grid_inDS) > 1):
  mainLogger.fatal("Cannot use multiple datasets in this version.", NotImplementedError)

from subprocess import check_output

# TODO For now we only support one container for signal and one for background.
# We need to change it so that we can input multiple containers. This can be
Ejemplo n.º 29
0
from RingerCore.Logger import Logger, LoggingLevel
from TuningTools.TuningJob import TuningJob
from TuningTools.PreProc import *
from TuningTools.TuningJob import fixPPCol
from TuningTools.coreDef import coreConf, TuningToolCores
from TuningTools.TuningJob import ReferenceBenchmark, ReferenceBenchmarkCollection, BatchSizeMethod
from RingerCore.Configure import Development
import logging
import argparse

start = timer()
Development.set(True)
coreConf.set(TuningToolCores.keras)
#coreConf.set(TuningToolCores.FastNet)

mainLogger = Logger.getModuleLogger("job")
parser = argparse.ArgumentParser(description='', add_help=False)
parser = argparse.ArgumentParser()

parser.add_argument('-d',
                    '--data',
                    action='store',
                    dest='data',
                    required=True,
                    help="The input tuning files.")

parser.add_argument('-g',
                    '--gpu',
                    action='store',
                    dest='gpu',
                    required=False,