def __init__(self, project, config, dbConnector, fileServer, options):
super(GenericDetectron2Model,
self).__init__(project, config, dbConnector, fileServer, options)
if isinstance(options, str):
try:
options = json.loads(options)
except:
# something went wrong; discard options #TODO
options = None
# try to fill and substitute global definitions in JSON-enhanced options
if isinstance(options, dict):
try:
updatedOptions = optionsHelper.merge_options(
self.getDefaultOptions(), options.copy())
updatedOptions = optionsHelper.substitute_definitions(
updatedOptions)
self.options = updatedOptions
except:
# something went wrong; ignore
pass
# verify options
result = self.verifyOptions(self.options)
if not result['valid']:
print(
f'[{self.project}] WARNING: provided options are invalid; replacing with defaults...'
)
self.options = self.getDefaultOptions()
# prepare Detectron2 configuration
self.detectron2cfg = self._get_config()
def _load_default_options(cls, jsonFilePath, defaultOptions):
try:
# try to load defaults from JSON file first
options = json.load(open(jsonFilePath, 'r'))
except Exception as e:
# error; fall back to built-in defaults
print(
f'Error reading default options file "{jsonFilePath}" (message: "{str(e)}"), falling back to built-in options.'
)
options = defaultOptions
# expand options
options = optionsHelper.substitute_definitions(options)
return options
def getDefaultOptions():
jsonFile = 'config/ai/model/pytorch/boundingBoxes/retinanet.json'
try:
# try to load defaults from JSON file first
options = json.load(open(jsonFile, 'r'))
except Exception as e:
# error; fall back to built-in defaults
print(
f'Error reading default RetinaNet options file "{jsonFile}" (message: "{str(e)}"), falling back to built-in options.'
)
options = DEFAULT_OPTIONS
# expand options
options = optionsHelper.substitute_definitions(options)
return options
def verifyOptions(cls, options):
defaultOptions = cls.getDefaultOptions()
if options is None:
return {'valid': True, 'options': defaultOptions}
try:
if isinstance(options, str):
options = json.loads(options)
options = optionsHelper.substitute_definitions(options)
except Exception as e:
return {
'valid':
False,
'errors':
[f'Options are not in a proper format (message: {str(e)}).']
}
try:
# mandatory field: model config
modelConfig = optionsHelper.get_hierarchical_value(
options, ['options', 'model', 'config', 'value'])
if modelConfig is None:
raise Exception('missing model type field in options.')
opts, warnings, errors = optionsHelper.verify_options(
options['options'], autoCorrect=True)
options['options'] = opts
return {
'valid': not len(errors),
'warnings': warnings,
'errors': errors,
'options': options
}
except Exception as e:
return {
'valid':
False,
'errors': [
f'An error occurred trying to verify options (message: {str(e)}).'
]
}
def __init__(self, project, config, dbConnector, fileServer, options):
super(GenericPyTorchModel, self).__init__(project, config, dbConnector,
fileServer, options)
# try to fill and substitute global definitions in JSON-enhanced options
if isinstance(options, dict) and 'defs' in options:
try:
updatedOptions = optionsHelper.substitute_definitions(
options.copy())
self.options = updatedOptions
except:
# something went wrong; ignore
pass
# retrieve executables
try:
self.model_class = get_class_executable(
optionsHelper.get_hierarchical_value(
self.options, ['options', 'model', 'class']))
except:
self.model_class = None
try:
self.criterion_class = get_class_executable(
optionsHelper.get_hierarchical_value(
self.options, ['options', 'train', 'criterion', 'class']))
except:
self.criterion_class = None
try:
self.optim_class = get_class_executable(
optionsHelper.get_hierarchical_value(
self.options, ['options', 'train', 'optim']))
except:
self.optim_class = SGD
try:
self.dataset_class = get_class_executable(
optionsHelper.get_hierarchical_value(self.options,
['options', 'dataset']))
except:
self.dataset_class = None
def initializeTransforms(self, mode='train', options=None):
'''
AIDE's Detectron2-compliant models all support the same transforms and thus
can be initialized the same way. "mode" determines whether the transforms
specified for 'train' or for 'inference' are to be initialized. If
"options" contains a dict of AIDE model options, the transforms are to be
initialized from there; otherwise the current class-specific ones are
used.
'''
assert mode in ('train',
'inference'), 'Invalid transform mode specified'
if isinstance(options, dict):
opt = copy.deepcopy(options)
else:
opt = copy.deepcopy(self.options)
opt = optionsHelper.substitute_definitions(opt)
# parse transforms
transforms = []
#TODO: add a resize transform in any case
transformOpts = optionsHelper.get_hierarchical_value(
opt, ['options', mode, 'transform', 'value'])
for tr in transformOpts:
trClass = tr['id']
args = optionsHelper.filter_reserved_children(tr, True)
for a in args.keys():
args[a] = optionsHelper.get_hierarchical_value(
args[a], ['value'])
if a == 'interp':
#TODO: ugly solution to convert interpolation methods
args[a] = int(args[a])
# initialize
transform = getattr(T, trClass)(**args)
transforms.append(transform)
return transforms
def verifyOptions(options):
# get default options to compare to
defaultOptions = RetinaNet.getDefaultOptions()
# updated options with modifications made
if options is None:
updatedOptions = defaultOptions.copy()
else:
if not isinstance(options, dict):
try:
options = json.loads(options)
except Exception as e:
return {
'valid':
False,
'warnings': [],
'errors': [
f'Options are not in valid JSON format (message: "{str(e)}").'
]
}
updatedOptions = options.copy()
result = {'valid': True, 'warnings': [], 'errors': []}
if not 'defs' in updatedOptions:
# old version (without GUI formatting): convert first
updatedOptions, warnings = RetinaNet._convertOldOptions(
updatedOptions, defaultOptions)
result['warnings'].append(
'Options have been converted to new format.')
result['warnings'].extend(warnings)
# flatten and fill globals
updatedOptions = optionsHelper.substitute_definitions(updatedOptions)
# do the verification
missingClassOptions = optionsHelper.get_hierarchical_value(
updatedOptions, ['options', 'general', 'labelClasses'])
if not isinstance(missingClassOptions, dict):
updatedOptions['options']['general']['labelClasses'] = \
optionsHelper.get_hierarchical_value(defaultOptions, ['options', 'general', 'labelClasses'])
#TODO: verify rest
# verify transforms
transforms_train = updatedOptions['options']['train']['transform'][
'value']
transforms_train, w, e = RetinaNet._verify_transforms(
transforms_train, True)
result['warnings'].extend(w)
result['errors'].extend(e)
if transforms_train is None:
result['valid'] = False
else:
updatedOptions['options']['train']['transform'][
'value'] = transforms_train
transforms_inf = updatedOptions['options']['inference']['transform'][
'value']
transforms_inf, w, e = RetinaNet._verify_transforms(
transforms_inf, False)
result['warnings'].extend(w)
result['errors'].extend(e)
if transforms_inf is None:
result['valid'] = False
else:
updatedOptions['options']['inference']['transform'][
'value'] = transforms_inf
if result['valid']:
result['options'] = updatedOptions
return result