def __init__(self, transientAttrs = set(), toPublicAttrs = set(), **kw):
"Initialize streamer and declare transient variables."
Logger.__init__(self, kw)
self.transientAttrs = set(transientAttrs) | {'_readVersion',}
self.toPublicAttrs = set(toPublicAttrs)
from Gaugi import checkForUnusedVars
checkForUnusedVars( kw, self._logger.warning )
def __init__(self,
generator,
etbins,
etabins,
x_bin_size,
y_bin_size,
ymin,
ymax,
false_alarm_limit=0.5,
level=LoggingLevel.INFO,
xmin_percentage=1,
xmax_percentage=99,
plot_stage='Internal',
palette=kBlackBody,
xmin=None,
xmax=None):
# init base class
Logger.__init__(self, level=level)
self.__generator = generator
self.__etbins = etbins
self.__etabins = etabins
self.__ymin = ymin
self.__ymax = ymax
self.__x_bin_size = x_bin_size
self.__y_bin_size = y_bin_size
self.__false_alarm_limit = false_alarm_limit
self.__xmin_percentage = xmin_percentage
self.__xmax_percentage = xmax_percentage
self.__plot_stage = plot_stage
self.__xmin = xmin
self.__xmax = xmax
self.__palette = palette
def __init__(self, db, args=None):
Logger.__init__(self)
self.__db = db
if args:
run_parser = argparse.ArgumentParser(
description='Run pilot command lines.', add_help=False)
run_parser.add_argument(
'-n',
'--node',
action='store',
dest='node',
required=False,
default=socket.gethostname(),
help="The node name registered into the database.")
run_parser.add_argument(
'-m',
'--master',
action='store_true',
dest='master',
required=False,
help="This is a master branch. One node must be a master.")
parent = argparse.ArgumentParser(description='', add_help=False)
subparser = parent.add_subparsers(dest='option')
subparser.add_parser('run', parents=[run_parser])
args.add_parser('pilot', parents=[parent])
def __init__(self, outputFile, **kw):
Logger.__init__(self, **kw)
if not outputFile.endswith('.root'):
outputFile += '.root'
from Gaugi.utilities import retrieve_kw
# Use this property to rebuild the storegate from a root file
self._restoreStoreGate = retrieve_kw(kw, 'restoreStoreGate', False)
filterDirs = retrieve_kw(kw, 'filterDirs', None)
#Create TFile object to hold everything
from ROOT import TFile
from Gaugi import expandPath
outputFile = expandPath(outputFile)
if self._restoreStoreGate:
import os.path
if not os.path.exists(outputFile):
raise ValueError("File '%s' does not exist" % outputFile)
self._file = TFile(outputFile, "read")
else:
self._file = TFile(outputFile, "recreate")
self._currentDir = ""
self._objects = dict()
self._dirs = list()
import os
self._outputFile = os.path.abspath(outputFile)
if self._restoreStoreGate:
retrievedObjs = self.__restore(self._file, filterDirs=filterDirs)
for name, obj in retrievedObjs:
self._dirs.append(name)
self._objects[name] = obj
def __init__(self, cls, id, queue_size=1):
Logger.__init__(self)
Process.__init__(self)
self._queue = Queue(queue_size)
self._cls = cls
self._id = id
self._is_alive_event = Event()
def __init__(self , pattern_generator, crossval, **kw ):
Logger.__init__(self)
self.__pattern_generator = pattern_generator
self.crossval = crossval
self.optimizer = retrieve_kw( kw, 'optimizer' , 'adam' )
self.loss = retrieve_kw( kw, 'loss' , 'binary_crossentropy' )
self.epochs = retrieve_kw( kw, 'epochs' , 1000 )
self.batch_size = retrieve_kw( kw, 'batch_size' , 1024 )
self.callbacks = retrieve_kw( kw, 'callbacks' , [] )
self.metrics = retrieve_kw( kw, 'metrics' , [] )
self.sorts = retrieve_kw( kw, 'sorts' , range(1) )
self.inits = retrieve_kw( kw, 'inits' , 1 )
job_auto_config = retrieve_kw( kw, 'job' , None )
self.__verbose = retrieve_kw( kw, 'verbose' , True )
self.__class_weight = retrieve_kw( kw, 'class_weight' , False )
self.__save_history = retrieve_kw( kw, 'save_history' , True )
self.decorators = retrieve_kw( kw, 'decorators' , [] )
# read the job configuration from file
if job_auto_config:
if type(job_auto_config) is str:
MSG_INFO( self, 'Reading job configuration from: %s', job_auto_config )
from saphyra.core.readers import JobReader
job = JobReader().load( job_auto_config )
else:
job = job_auto_config
# retrive sort/init lists from file
self.sorts = job.getSorts()
self.inits = job.getInits()
self.__models, self.__id_models = job.getModels()
self.__jobId = job.id()
# get model and tag from model file or lists
models = retrieve_kw( kw, 'models', None )
if models:
self.__models = models
self.__id_models = [id for id in range(len(models))]
self.__jobId = 0
self.__outputfile = retrieve_kw( kw, 'outputFile' , None )
if self.__outputfile:
from saphyra.core.readers.versions import TunedData_v1
self.__tunedData = TunedData_v1()
checkForUnusedVars(kw)
from saphyra import Context
self.__context = Context()
self.__index_from_cv = None
def __init__(self, t):
Logger.__init__(self)
import collections
self._containers = collections.OrderedDict()
self._tree = NotSet
self._decorations = dict()
self._current_entry = NotSet
self._tree = t
def __init__(self, etbins, etabins, fill_colors=fill_colors, line_colors=line_colors):
Logger.__init__(self)
self.__etbins = etbins
self.__etabins = etabins
self.__these_fill_colors=fill_colors
self.__these_line_colors=line_colors
self.__hist = {}
def __init__(self, db, args=None):
Logger.__init__(self)
self.__db = db
if args:
# Upload dataset using the dataset CLI
registry_parser = argparse.ArgumentParser(
description='Dataset registry command lines.', add_help=False)
registry_parser.add_argument(
'-d',
'--dataset',
action='store',
dest='datasetname',
required=True,
help=
"The dataset name used to registry into the database. (e.g: user.jodafons...)"
)
registry_parser.add_argument('-p',
'--path',
action='store',
dest='path',
required=True,
help="The path to the dataset")
# Delete dataset using the dataset CLI
unregistry_parser = argparse.ArgumentParser(
description='Dataset unregistry command lines.',
add_help=False)
unregistry_parser.add_argument(
'-d',
'--dataset',
action='store',
dest='datasetname',
required=True,
help="The dataset name to be removed")
# Delete dataset using the dataset CLI
list_parser = argparse.ArgumentParser(
description='Dataset List command lines.', add_help=False)
list_parser.add_argument(
'-u',
'--user',
action='store',
dest='username',
required=False,
default=config['username'],
help="List all datasets for a selected user.")
parent = argparse.ArgumentParser(description='', add_help=False)
subparser = parent.add_subparsers(dest='option')
# Datasets
subparser.add_parser('registry', parents=[registry_parser])
subparser.add_parser('unregistry', parents=[unregistry_parser])
subparser.add_parser('list', parents=[list_parser])
args.add_parser('castor', parents=[parent])
def __init__(self, nthreads, **kw):
Logger.__init__(self, **kw)
self._nthreads = nthreads
self._nFilesPerJob = 20
self._skip_these_keys = [
"features", "etBins", "etaBins", "etBinIdx", "etaBinIdx"
]
import re
self._pat = re.compile(
r'.+(?P<binID>et(?P<etBinIdx>\d+).eta(?P<etaBinIdx>\d+))\..+$')
def __init__(self, command, njobs, maxJobs, output):
Logger.__init__(self)
self.process_pipe = []
self.output_to_merge = []
import random
import time
random.seed(time.time())
self._base_id = random.randrange(100000)
self._jobList = list(range(njobs))
self._maxJobs = maxJobs
self._command = command
self._output = output
def __init__(self, db, args=None):
Logger.__init__(self)
self.__db = db
if args:
create_parser = argparse.ArgumentParser(
description='User create command lines.', add_help=False)
create_parser.add_argument('-n',
'--name',
action='store',
dest='name',
required=True,
help="The name of the user.")
create_parser.add_argument('-e',
'--email',
action='store',
dest='email',
required=True,
help="The user email.")
delete_parser = argparse.ArgumentParser(
description='User remove command lines.', add_help=False)
delete_parser.add_argument('-n',
'--name',
action='store',
dest='name',
required=True,
help="The dataset name to be removed")
# Delete dataset using the dataset CLI
list_parser = argparse.ArgumentParser(
description='List all users command lines.', add_help=False)
list_parser.add_argument('-u',
'--user',
action='store',
dest='name',
required=False,
help="List all attributes for this user")
init_parser = argparse.ArgumentParser(
description='Initialize the database.', add_help=False)
parent = argparse.ArgumentParser(description='', add_help=False)
subparser = parent.add_subparsers(dest='option')
# Datasets
subparser.add_parser('create', parents=[create_parser])
subparser.add_parser('delete', parents=[delete_parser])
subparser.add_parser('list', parents=[list_parser])
subparser.add_parser('init', parents=[init_parser])
args.add_parser('user', parents=[parent])
def __init__(self, fList):
Logger.__init__(self)
from Gaugi import csvStr2List
from Gaugi import expandFolders
self.fList = csvStr2List(fList)
self.fList = expandFolders(fList)
self.process_pipe = []
self.output_stack = []
import random
import time
random.seed(time.time())
self._base_id = random.randrange(100000)
def __init__(self):
Logger.__init__(self)
self._idx = 0
self._is_hlt = False
self._decoration = dict()
self._tree = None
self._event = None
self._context = None
# this is used for metadata properties
self._useMetadataParams = False
self._metadataParams = {}
self._branches = list() # hold all branches from the body class
def __init__(self, name):
Logger.__init__(self)
self._name = name
# flags
self._wtd = StatusWTD.DISABLE
self._status = StatusTool.ENABLE
self._initialized = StatusTool.NOT_INITIALIZED
self._finalized = StatusTool.NOT_FINALIZED
# services and EDMs
self._context = NotSet
self._storegateSvc = NotSet
self._dataframe = NotSet
# property
self.__property = {}
def __init__(self,
verbose=False,
save_the_best=False,
patience=False,
**kw):
super(Callback, self).__init__()
Logger.__init__(self, **kw)
self.__verbose = verbose
self.__patience = patience
self.__ipatience = 0
self.__best_sp = 0.0
self.__save_the_best = save_the_best
self.__best_weights = NotSet
self.__best_epoch = 0
self._validation_data = NotSet
def __init__(self, ignoreAttrs = set(), toProtectedAttrs = set(), ignoreRawChildren = False, **kw ):
"""
-> ignoreAttrs: not consider this attributes on the dictionary values.
-> toProtectedAttrs: change public attributes to protected or private
attributes. That is, suppose the dictionary value is 'val' and the class
value should be _val or __val, then add toProtectedAttrs = ['_val'] or
'__val'.
-> ignoreRawChildren: Do not attempt to conver raw children to higher level object.
"""
Logger.__init__(self, kw)
ignoreAttrs = list(set(ignoreAttrs) | RawDictCnv.baseAttrs)
import re
self.ignoreAttrs = [re.compile(ignoreAttr) for ignoreAttr in ignoreAttrs]
self.toProtectedAttrs = set(toProtectedAttrs)
self.ignoreRawChildren = ignoreRawChildren
from Gaugi import checkForUnusedVars
checkForUnusedVars( kw, self._logger.warning )
def __init__(self, generator, etbins, etabins, x_bin_size, y_bin_size, ymin, ymax,
false_alarm_limit=0.5,
level=LoggingLevel.INFO,
xmin_percentage=1,
xmax_percentage=99):
# init base class
Logger.__init__(self, level=level)
self.__generator = generator
self.__etbins = etbins
self.__etabins = etabins
self.__ymin = ymin
self.__ymax = ymax
self.__x_bin_size = x_bin_size
self.__y_bin_size = y_bin_size
self.__false_alarm_limit = false_alarm_limit
self.__xmin_percentage=xmin_percentage
self.__xmax_percentage=xmax_percentage
def __init__(self, fList, reader, nFilesPerJob, nthreads):
Logger.__init__(self)
from Gaugi import csvStr2List
from Gaugi import expandFolders
fList = csvStr2List(fList)
self._fList = expandFolders(fList)
def chunks(l, n):
"""Yield successive n-sized chunks from l."""
for i in range(0, len(l), n):
yield l[i:i + n]
self._fList = [l for l in chunks(self._fList, nFilesPerJob)]
self.process_pipe = []
self._outputs = []
self._nthreads = nthreads
self._reader = reader
def __init__(self, d={}, **kw):
if None in self._contextManager._acceptedTypes:
self._contextManager._acceptedTypes = TexObject,
if (not isinstance(self, TexObjectCollection)
and not hasattr(self, '_preamble')
and not hasattr(self, '_enclosure')
and not hasattr(self, '_body')
and not hasattr(self, '_footer')
and not hasattr(self, '_appendix')):
raise TexException(
self, 'Class %s does not write any tex code.' %
self.__class__.__name__)
d.update(kw)
Logger.__init__(self, d)
if hasattr(self, '_body'):
self._body = formatTex(self._body, retrieve_kw(d, 'textWidth', 80))
self._stream = kw.pop('stream', tss)
self._keywords = {
key: val
for key, val in d.items() if not key.startswith('_')
}
self._keywords.update({
key: val
for key, val in self.__dict__.items() if not key.startswith('_')
})
if 'star' in self._keywords and self._keywords['star']:
self._keywords['star'] = '*'
else:
self._keywords['star'] = ''
if hasattr(self, '_assertVars'):
for key in self._assertVars:
if not key in self._keywords:
raise TexException(self, "Assert var %s failed." % key)
gcc.set(self)
self._contextManaged = d.pop('_contextManaged', True)
self._context = self._contextManager()
self._isInContext = self._context is not None
if (self._isInContext
and isinstance(self._context, TexObjectCollection)
and self._contextManaged):
self._context += self
if self._isInContext:
self._stream = self._context._stream
def __init__(self, **kw):
Logger.__init__(self)
self.epochs = retrieve_kw(kw, 'epochs', 1000)
self.batch_size = retrieve_kw(kw, 'batch_size', 1024)
self.lambda_disco = retrieve_kw(kw, 'lambda_disco', 300)
self.callbacks = retrieve_kw(kw, 'callbacks', [])
self.metrics = retrieve_kw(kw, 'metrics', [])
job_auto_config = retrieve_kw(kw, 'job', None)
self.sorts = retrieve_kw(kw, 'sorts', range(1))
self.inits = retrieve_kw(kw, 'inits', 1)
self.__verbose = retrieve_kw(kw, 'verbose', True)
self.__model_generator = retrieve_kw(kw, 'model_generator', None)
self.total = 100000
self.background_percent = 0.99
self.test_size = 0.3
# read the job configuration from file
if job_auto_config:
if type(job_auto_config) is str:
MSG_INFO(self, 'Reading job configuration from: %s',
job_auto_config)
from saphyra.core.readers import JobReader
job = JobReader().load(job_auto_config)
else:
job = job_auto_config
# retrive sort/init lists from file
self.sorts = job.getSorts()
self.inits = job.getInits()
self.__models, self.__id_models = job.getModels()
self.__jobId = job.id()
# get model and tag from model file or lists
models = retrieve_kw(kw, 'models', None)
if models:
self.__models = models
self.__id_models = [id for id in range(len(models))]
self.__jobId = 0
checkForUnusedVars(kw)
def __init__(self, db, args=None):
Logger.__init__(self)
self.__db = db
if args:
create_parser = argparse.ArgumentParser(
description='Node create command lines.', add_help=False)
create_parser.add_argument('-n',
'--name',
action='store',
dest='name',
required=True,
help="The name of the node.")
create_parser.add_argument('-ec',
'--enabledCPUSlots',
action='store',
dest='enabledCPUSlots',
required=True,
help="The number of CPU enabled slots.")
create_parser.add_argument(
'-mc',
'--maxNumberOfCPUSlots',
action='store',
dest='maxNumberOfCPUSlots',
required=True,
help="The total number of CPU slots for this node.")
create_parser.add_argument('-eg',
'--enabledGPUSlots',
action='store',
dest='enabledGPUSlots',
required=True,
help="The number of GPU enabled slots.")
create_parser.add_argument(
'-mg',
'--maxNumberOfGPUSlots',
action='store',
dest='maxNumberOfGPUSlots',
required=True,
help="The total number of GPU slots for this node.")
delete_parser = argparse.ArgumentParser(
description='Node remove command lines.', add_help=False)
delete_parser.add_argument('-n',
'--name',
action='store',
dest='name',
required=True,
help="The node name to be removed")
stop_parser = argparse.ArgumentParser(
description='Node stop command lines.', add_help=False)
stop_parser.add_argument('-n',
'--name',
action='store',
dest='name',
required=True,
help="The node name to be stop")
# Delete dataset using the dataset CLI
list_parser = argparse.ArgumentParser(
description='List all users command lines.', add_help=False)
parent = argparse.ArgumentParser(description='', add_help=False)
subparser = parent.add_subparsers(dest='option')
# Datasets
subparser.add_parser('create', parents=[create_parser])
subparser.add_parser('delete', parents=[delete_parser])
subparser.add_parser('list', parents=[list_parser])
subparser.add_parser('stop', parents=[stop_parser])
args.add_parser('node', parents=[parent])
def __init__(self, name):
Logger.__init__(self)
import collections
self._name = name
self._tools = collections.OrderedDict()
def __init__(self, d = {}, **kw):
#RawDictStreamable.__init__(self, d, **kw)
Logger.__init__(self, d, **kw)
def __init__(self, **kw):
Logger.__init__(self, **kw)
def __init__(self, trigger):
Logger.__init__(self)
# Compile all internal variables
self.compile(trigger)
def __init__(self, outputFile):
Logger.__init__(self)
if not outputFile:
raise TexException(self, 'Cannot stream to empty file path.')
self.outputFile = ensureExtension(outputFile, self._outputExtension)
def __init__(self): Logger.__init__(self) self.__containers = collections.OrderedDict()
def __init__(self, skip_these_keys, **kw):
Logger.__init__(self, kw)
self._skip_these_keys = skip_these_keys
def __init__(self, db, args=None):
Logger.__init__(self)
self.__db = db
if args:
# Create Task
create_parser = argparse.ArgumentParser(description='',
add_help=False)
create_parser.add_argument('-v',
'--volume',
action='store',
dest='volume',
required=True,
help="The volume")
create_parser.add_argument(
'-t',
'--task',
action='store',
dest='taskname',
required=True,
help="The task name to be append into the db.")
create_parser.add_argument(
'-c',
'--configFile',
action='store',
dest='configFile',
required=True,
help=
"The job config file that will be used to configure the job (sort and init)."
)
create_parser.add_argument(
'-d',
'--dataFile',
action='store',
dest='dataFile',
required=True,
help="The data/target file used to train the model.")
create_parser.add_argument(
'--sd',
'--secondaryDS',
action='store',
dest='secondaryDS',
required=False,
default="{}",
help=
"The secondary datasets to be append in the --exec command. This should be:"
+ "--secondaryData='{'REF':'path/to/my/extra/data',...}'")
create_parser.add_argument('--exec',
action='store',
dest='execCommand',
required=True,
help="The exec command")
create_parser.add_argument('--queue',
action='store',
dest='queue',
required=True,
default='gpu',
help="The cluste queue [gpu or cpu]")
create_parser.add_argument('--dry_run',
action='store_true',
dest='dry_run',
required=False,
default=False,
help="Use this as debugger.")
create_parser.add_argument('--bypass',
action='store_true',
dest='bypass',
required=False,
default=False,
help="Bypass the job test.")
# Create Task
repro_parser = argparse.ArgumentParser(description='',
add_help=False)
repro_parser.add_argument('-v',
'--volume',
action='store',
dest='volume',
required=True,
help="The volume")
repro_parser.add_argument(
'--new_task',
action='store',
dest='new_taskname',
required=True,
help="The new task name after the reprocessing phase")
repro_parser.add_argument(
'--old_task',
action='store',
dest='old_taskname',
required=True,
help=
"The old task name that will be used into the reprocessing phase"
)
repro_parser.add_argument(
'-d',
'--dataFile',
action='store',
dest='dataFile',
required=True,
help="The data/target file used to train the model.")
repro_parser.add_argument(
'--sd',
'--secondaryDS',
action='store',
dest='secondaryDS',
required=False,
default="{}",
help=
"The secondary datasets to be append in the --exec command. This should be:"
+ "--secondaryData='{'REF':'path/to/my/extra/data',...}'")
repro_parser.add_argument('--exec',
action='store',
dest='execCommand',
required=True,
help="The exec command")
repro_parser.add_argument('--queue',
action='store',
dest='queue',
required=True,
default='gpu',
help="The cluste queue [gpu or cpu]")
repro_parser.add_argument('--dry_run',
action='store_true',
dest='dry_run',
required=False,
default=False,
help="Use this as debugger.")
retry_parser = argparse.ArgumentParser(description='',
add_help=False)
retry_parser.add_argument('--id',
action='store',
nargs='+',
dest='id_list',
required=False,
default=None,
help="All task ids to be removed",
type=int)
retry_parser.add_argument(
'--id_min',
action='store',
dest='id_min',
required=False,
help="Down taks id limit to apply on the loop",
type=int,
default=None)
retry_parser.add_argument(
'--id_max',
action='store',
dest='id_max',
required=False,
help="Upper task id limit to apply on the loop",
type=int,
default=None)
delete_parser = argparse.ArgumentParser(description='',
add_help=False)
delete_parser.add_argument('--id',
action='store',
nargs='+',
dest='id_list',
required=False,
default=None,
help="All task ids to be removed",
type=int)
delete_parser.add_argument(
'--id_min',
action='store',
dest='id_min',
required=False,
help="Down taks id limit to apply on the loop",
type=int,
default=None)
delete_parser.add_argument(
'--id_max',
action='store',
dest='id_max',
required=False,
help="Upper task id limit to apply on the loop",
type=int,
default=None)
delete_parser.add_argument(
'--remove',
action='store_true',
dest='remove',
required=False,
help=
"Remove all files for this task into the storage. Beware when use this flag becouse you will lost your data too."
)
delete_parser.add_argument('--force',
action='store_true',
dest='force',
required=False,
help="Force delete.")
list_parser = argparse.ArgumentParser(description='',
add_help=False)
list_parser.add_argument('-u',
'--user',
action='store',
dest='username',
required=False,
default=config['username'],
help="The username.")
list_parser.add_argument('-a',
'--all',
action='store_true',
dest='all',
required=False,
help="List all tasks.")
list_parser.add_argument('-i',
'--interactive',
action='store_true',
dest='interactive',
required=False,
help="List all tasks interactive mode.")
kill_parser = argparse.ArgumentParser(description='',
add_help=False)
kill_parser.add_argument('--id',
action='store',
nargs='+',
dest='id_list',
required=False,
default=None,
help="All task ids to be removed",
type=int)
kill_parser.add_argument(
'--id_min',
action='store',
dest='id_min',
required=False,
help="Down taks id limit to apply on the loop",
type=int,
default=None)
kill_parser.add_argument(
'--id_max',
action='store',
dest='id_max',
required=False,
help="Upper task id limit to apply on the loop",
type=int,
default=None)
queue_parser = argparse.ArgumentParser(description='',
add_help=False)
queue_parser.add_argument('-n',
'--name',
action='store',
dest='name',
required=False,
help="The queue name")
parent = argparse.ArgumentParser(description='', add_help=False)
subparser = parent.add_subparsers(dest='option')
# Datasets
subparser.add_parser('create', parents=[create_parser])
subparser.add_parser('repro', parents=[repro_parser])
subparser.add_parser('retry', parents=[retry_parser])
subparser.add_parser('delete', parents=[delete_parser])
subparser.add_parser('list', parents=[list_parser])
subparser.add_parser('kill', parents=[kill_parser])
subparser.add_parser('queue', parents=[queue_parser])
args.add_parser('task', parents=[parent])
