-
Notifications
You must be signed in to change notification settings - Fork 0
/
coreDef.py
256 lines (219 loc) · 9.17 KB
/
coreDef.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
__all__ = [ 'hasExmachina', 'hasFastnet', 'hasKeras', 'TuningToolCores'
, 'AvailableTuningToolCores', 'CoreConfiguration', 'coreConf'
, 'NumpyConfiguration', 'npCurrent'
, 'DataframeConfiguration' , 'dataframeConf'
, 'TuningToolsGit']
import os, pkgutil
hasExmachina = bool( pkgutil.find_loader( 'exmachina' ) )
hasFastnet = bool( pkgutil.find_loader( 'libTuningTools' ) )
hasKeras = bool( pkgutil.find_loader( 'keras' ) )
from RingerCore import ( EnumStringification, npConstants, Configure
, EnumStringificationOptionConfigure, Holder
, NotSet, ArgumentError, GitConfiguration )
TuningToolsGit = GitConfiguration( 'TuningToolsGit', __file__, tagArgStr = '--tuning-tools-info')
class TuningToolCores( EnumStringification ):
_ignoreCase = True
FastNet = 0
ExMachina = 1
keras = 2
class AvailableTuningToolCores( EnumStringification ):
_ignoreCase = True
if hasFastnet: FastNet = 0
if hasExmachina: ExMachina = 1
if hasKeras: keras = 2
@classmethod
def retrieve(cls, val):
ret = TuningToolCores.retrieve( val )
if not cls.tostring( ret ):
raise ValueError("TuningTool core %s is not available in the current system." % TuningToolCores.tostring( ret ))
return ret
class _ConfigureCoreFramework( EnumStringificationOptionConfigure ):
"""
Singleton class for configurating the core framework used tuning data
It also specifies how the numpy data should be represented for that specified
core.
"""
_enumType = TuningToolCores
core = property( EnumStringificationOptionConfigure.get, EnumStringificationOptionConfigure.set )
def auto( self ):
self._logger.debug("Using automatic configuration for core specification.")
# Check whether we can retrieve from the parser.
from TuningTools.parsers.BaseModuleParser import coreFrameworkParser
import sys
try:
args, argv = coreFrameworkParser.parse_known_args()
if args.core_framework not in (None, NotSet):
self.core = args.core_framework
# Consume option
sys.argv = sys.argv[:1] + argv
else:
self.core = self.default()
except (ArgumentError, ValueError) as e:
self._logger.verbose("Ignored argument parsing error:\n %s", e )
self._logger.debug("Using default TuningTools core (%s).", AvailableTuningToolCores.tostring(self.default()))
# Couldn't retrieve from the parser, retrieve default:
self.core = self.default()
def default( self ):
if hasFastnet:
core = TuningToolCores.FastNet
elif hasKeras:
core = TuningToolCores.keras
elif hasExmachina:
core = TuningToolCores.ExMachina
else:
self._fatal("Couldn't define which tuning core was compiled.")
return core
def numpy_wrapper(self):
"""
Returns the api instance which is to be used to read the data
"""
import numpy as np
if self.core is TuningToolCores.ExMachina:
# Define the exmachina numpy constants
kwargs = { 'useFortran' : True, 'fp_dtype' : np.float64, 'int_dtype' : np.int64 }
elif self.core is TuningToolCores.FastNet:
kwargs = { 'useFortran' : False, 'fp_dtype' : np.float32, 'int_dtype' : np.int32 }
elif self.core is TuningToolCores.keras:
from keras.backend import backend
if backend() == "theano": # Theano copies data if input is not c-contiguous
kwargs = { 'useFortran' : False, 'fp_dtype' : np.float32, 'int_dtype' : np.int32 }
elif backend() == "tensorflow": # tensorflow copies data if input is not fortran-contiguous
kwargs = { 'useFortran' : False, 'fp_dtype' : np.float32, 'int_dtype' : np.int32 }
return npConstants( **kwargs )
def core_framework(self):
if self.core is TuningToolCores.FastNet:
from libTuningTools import TuningToolPyWrapper as RawWrapper
import sys, os
class TuningToolPyWrapper( RawWrapper, object ):
def __init__( self
, level
, useColor = not(int(os.environ.get('RCM_GRID_ENV',0)) or not(sys.stdout.isatty()))
, seed = None):
self._doMultiStop = False
if seed is None:
RawWrapper.__init__(self, level, useColor)
else:
RawWrapper.__init__(self, level, useColor, seed)
@property
def multiStop(self):
return self._doMultiStop
@multiStop.setter
def multiStop(self, value):
if value:
self._doMultiStop = True
self.useAll()
else:
self._doMultiStop = False
self.useSP()
# End of TuningToolPyWrapper
return TuningToolPyWrapper
elif self.core is TuningToolCores.ExMachina:
import exmachina
return exmachina
elif self.core is TuningToolCores.keras:
import keras
return keras
# The singleton holder
CoreConfiguration = Holder( _ConfigureCoreFramework() )
# Standard core configuration object
coreConf = CoreConfiguration()
class _ConfigureNumpyWrapper( Configure ):
"""
Wrapper for numpy module setting defaults accordingly to the core used.
"""
wrapper = property( Configure.get, Configure.set )
def auto( self ):
self._logger.debug("Using automatic configuration for numpy wrapper.")
self.wrapper = coreConf.numpy_wrapper()
self._logger.debug("Retrieved the following numpy wrapper:\n%s", self.wrapper)
def __getattr__(self, attr ):
if hasattr( self.wrapper, attr ):
return getattr( self.wrapper, attr )
else:
raise AttributeError( attr )
# The singleton holder
NumpyConfiguration = Holder( _ConfigureNumpyWrapper() )
# Standard numpy configuration object
npCurrent = NumpyConfiguration()
from TuningTools.dataframe.EnumCollection import Dataframe as DataframeEnum
class _ConfigureDataframe( EnumStringificationOptionConfigure ):
"""
Singleton class for configurating the data framework used for reading the
files and generating the tuning-data
"""
_enumType = DataframeEnum
dataframe = property( EnumStringificationOptionConfigure.get, EnumStringificationOptionConfigure.set )
def auto_retrieve_testing_sample( self, sample ):
self._sample = sample
def can_autoconfigure( self ):
" Returns whether the dataframe can autoconfigure itself"
if hasattr(self, '_sample') and isinstance(self._sample, (dict, list, basestring)):
return True
return False
def auto( self ):
self._debug("Using automatic configuration for dataframe specification.")
# Check whether we can retrieve from the parser.
from TuningTools.parsers.BaseModuleParser import dataframeParser
import sys
try:
args, argv = dataframeParser.parse_known_args()
if args.data_framework not in (None, NotSet):
self.dataframe = args.data_framework
# Consume option
sys.argv = sys.argv[:1] + argv
except (ArgumentError, ValueError) as e:
self._debug("Ignored argument parsing error:\n %s", e )
pass
from RingerCore import csvStr2List, expandFolders
if not self.configured() and not self.can_autoconfigure():
self._fatal("Cannot auto-configure which dataframe to use because no sample was specified via the auto_retrieve_sample() method.")
elif not self.configured():
if isinstance(self._sample, dict):
for key in self._sample:
if 'elCand2_' in key:
self.dataframe = DataframeEnum.SkimmedNtuple
else:
self.dataframe = DataframeEnum.PhysVal
break
elif self._sample and isinstance(self._sample, list):
if not isinstance(self._sample[0], basestring ):
self._fatal("Cannot autoconfigure dataframe using the following list: %r", self._sample )
fList = csvStr2List ( self._sample[0] )
fList = expandFolders( fList )
for inputFile in fList:
self._checkFile( inputFile )
if self.configured(): break
elif isinstance( self._sample, basestring ):
if os.path.isdir( self._sample ):
fList = expandFolders( self._sample )
for inputFile in fList:
self._checkFile( inputFile )
if self.configured(): break
else:
self._checkFile( self._sample )
if not self.configured():
self._fatal("Couldn't autoconfigure using source: %r", self._sample)
def _checkFile( self, inputFile ):
from ROOT import TFile
f = TFile.Open(inputFile, 'read')
if not f or f.IsZombie(): return
self.dataframe = DataframeEnum.PhysVal
for key in f.GetListOfKeys():
if key.GetName == "ZeeCanditate":
self.dataframe = DataframeEnum.SkimmedNtuple
break
def api(self):
"""
Returns the api instance which is to be used to read the data
"""
if self.dataframe is DataframeEnum.PhysVal:
from TuningTools.dataframe.ReadPhysVal import readData
elif self.dataframe is DataframeEnum.PhysVal_v2:
from TuningTools.dataframe.ReadPhysVal_v2 import readData
elif self.dataframe is DataframeEnum.SkimmedNtuple:
from TuningTools.dataframe.ReadSkimmedNtuple import readData
return readData
# The singleton holder
DataframeConfiguration = Holder( _ConfigureDataframe() )
# Standard dataframe configuration object
dataframeConf = DataframeConfiguration()