def parse_aide_config(cls, config, detectron2cfg=None):
if detectron2cfg is None:
detectron2cfg = get_cfg()
if isinstance(config, dict):
for key in config.keys():
if isinstance(key, str) and key.startswith('DETECTRON2.'):
value = optionsHelper.get_hierarchical_value(
config[key], ['value', 'id'])
if isinstance(value, list):
for v in range(len(value)):
value[v] = optionsHelper.get_hierarchical_value(
value[v], ['value', 'id'])
# copy over to Detectron2 configuration
tokens = key.split('.')
attr = detectron2cfg
for t in range(1, len(tokens)): # skip "DETECTRON2" marker
if t == len(tokens) - 1: # last element
setattr(attr, tokens[t], value)
else:
attr = getattr(attr, tokens[t])
else:
GenericDetectron2Model.parse_aide_config(
config[key], detectron2cfg)
return detectron2cfg
def initializeModel(self,
stateDict,
data,
addMissingLabelClasses=False,
removeObsoleteLabelClasses=False):
'''
Converts the provided stateDict from a bytes array to a torch-loadable
object and initializes a model from it. Also returns a 'labelClassMap',
defining the indexing between label classes and the model.
If the stateDict object is None, a new model and labelClassMap are crea-
ted from the defaults.
If "addMissingLabelClasses" is True, new output neurons are appended for
label classes that are not present in the model's current labelclass map,
and the map is updated.
Likewise, if "removeObsoleteLabelClasses" is True, existing outputs for
label classes that are not present in the set of label classes anymore are
removed, and the map is also updated.
'''
# initialize model
if stateDict is not None:
stateDict = torch.load(io.BytesIO(stateDict),
map_location=lambda storage, loc: storage)
model = self.model_class.loadFromStateDict(stateDict)
# mapping labelclass (UUID) to index in model (number)
labelclassMap = stateDict['labelclassMap']
if addMissingLabelClasses or removeObsoleteLabelClasses:
# modification of model outputs
if hasattr(model, 'updateModel'):
model.updateModel(data['labelClasses'],
addMissingLabelClasses,
removeObsoleteLabelClasses)
else:
# create new label class map
labelclassMap = {}
for idx, lcID in enumerate(data['labelClasses']):
labelclassMap[
lcID] = idx #NOTE: we do not use the labelclass' serial 'idx', since this might contain gaps
# self.options['options']['model']['labelclassMap'] = labelclassMap #TODO: never used and obsolete with new options format
# initialize a fresh model
modelKwargs = {}
modelOptions = optionsHelper.get_hierarchical_value(
self.options, ['options', 'model'], [])
for key in modelOptions.keys():
if key not in optionsHelper.RESERVED_KEYWORDS:
modelKwargs[key] = optionsHelper.get_hierarchical_value(
modelOptions[key], ['value', 'id'])
modelKwargs['labelclassMap'] = labelclassMap
model = self.model_class(**modelKwargs)
return model, labelclassMap
def _get_config(self):
cfg = get_cfg()
cfg.set_new_allowed(True)
# augment and initialize Detectron2 cfg with selected model
defaultConfig = optionsHelper.get_hierarchical_value(
self.options, ['options', 'model', 'config', 'value', 'id'])
if isinstance(defaultConfig, str):
# try to load from Detectron2's model zoo
try:
configFile = model_zoo.get_config_file(defaultConfig)
except:
# not available; try to load locally instead
configFile = os.path.join(
os.getcwd(), 'ai/models/detectron2/_functional/configs',
defaultConfig)
if not os.path.exists(configFile):
configFile = None
if configFile is not None:
cfg.merge_from_file(configFile)
try:
cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(defaultConfig)
except:
pass
return cfg
def _parse_transform(tr):
tr_kwargs = {}
if isinstance(tr, dict):
tr_class = optionsHelper.get_hierarchical_value(tr, ['id'])
for key in tr.keys():
if key not in optionsHelper.RESERVED_KEYWORDS:
tr_kwargs[key] = optionsHelper.get_hierarchical_value(
tr[key], ['value'])
elif isinstance(tr, str):
tr_class = tr
if tr_class.endswith('DefaultTransform'):
subTr = GenericPyTorchModel.parseTransforms(tr['transform'])
tr_kwargs = {'transform': subTr}
elif tr_class == 'torchvision.transforms.Normalize':
mean = optionsHelper.get_hierarchical_value(
tr_kwargs, ['mean', 'value'])
std = optionsHelper.get_hierarchical_value(
tr_kwargs, ['std', 'value'])
tr_kwargs = {
'mean':
[mean[0]['value'], mean[1]['value'], mean[2]['value']],
'std': [std[0]['value'], std[1]['value'], std[2]['value']]
}
elif tr_class == 'torchvision.transforms.ColorJitter':
brightness = optionsHelper.get_hierarchical_value(
tr_kwargs, ['brightness', 'value'])
contrast = optionsHelper.get_hierarchical_value(
tr_kwargs, ['contrast', 'value'])
saturation = optionsHelper.get_hierarchical_value(
tr_kwargs, ['saturation', 'value'])
hue = optionsHelper.get_hierarchical_value(
tr_kwargs, ['hue', 'value'])
tr_kwargs = {
'brightness': [
brightness[0]['value'],
brightness[1]['value'],
],
'contrast': [
contrast[0]['value'],
contrast[1]['value'],
],
'saturation': [
saturation[0]['value'],
saturation[1]['value'],
],
'hue': [
hue[0]['value'],
hue[1]['value'],
]
}
elif tr_class.endswith('Compose'):
sub_transforms = GenericPyTorchModel.parseTransforms(
tr['transforms'])
tr_kwargs = {'transforms': sub_transforms}
#TODO: others?
return parse_transforms({'class': tr_class, 'kwargs': tr_kwargs})
def _get_config(self):
cfg = get_cfg()
add_torchvision_classifier_config(cfg)
defaultConfig = optionsHelper.get_hierarchical_value(
self.options, ['defs', 'model'])
if isinstance(defaultConfig, dict):
defaultConfig = defaultConfig['id']
configFile = os.path.join(os.getcwd(),
'ai/models/detectron2/_functional/configs',
defaultConfig)
cfg.merge_from_file(configFile)
return cfg
def __init__(self, project, config, dbConnector, fileServer, options):
super(GenericPyTorchModel, self).__init__(project, config, dbConnector,
fileServer, options)
# try to fill and substitute global definitions in JSON-enhanced options
if isinstance(options, dict) and 'defs' in options:
try:
updatedOptions = optionsHelper.substitute_definitions(
options.copy())
self.options = updatedOptions
except:
# something went wrong; ignore
pass
# retrieve executables
try:
self.model_class = get_class_executable(
optionsHelper.get_hierarchical_value(
self.options, ['options', 'model', 'class']))
except:
self.model_class = None
try:
self.criterion_class = get_class_executable(
optionsHelper.get_hierarchical_value(
self.options, ['options', 'train', 'criterion', 'class']))
except:
self.criterion_class = None
try:
self.optim_class = get_class_executable(
optionsHelper.get_hierarchical_value(
self.options, ['options', 'train', 'optim']))
except:
self.optim_class = SGD
try:
self.dataset_class = get_class_executable(
optionsHelper.get_hierarchical_value(self.options,
['options', 'dataset']))
except:
self.dataset_class = None
def _get_config(self):
cfg = get_cfg()
add_deeplab_config(cfg)
defaultConfig = optionsHelper.get_hierarchical_value(self.options, ['options', 'model', 'config', 'value', 'id'])
configFile = os.path.join(os.getcwd(), 'ai/models/detectron2/_functional/configs', defaultConfig)
cfg.merge_from_file(configFile)
# disable SyncBatchNorm if not running on distributed system
if comm.get_world_size() <= 1:
cfg.MODEL.RESNETS.NORM = 'BN'
cfg.MODEL.SEM_SEG_HEAD.NORM = 'BN'
return cfg
def initializeTransforms(self, mode='train', options=None):
'''
AIDE's Detectron2-compliant models all support the same transforms and thus
can be initialized the same way. "mode" determines whether the transforms
specified for 'train' or for 'inference' are to be initialized. If
"options" contains a dict of AIDE model options, the transforms are to be
initialized from there; otherwise the current class-specific ones are
used.
'''
assert mode in ('train',
'inference'), 'Invalid transform mode specified'
if isinstance(options, dict):
opt = copy.deepcopy(options)
else:
opt = copy.deepcopy(self.options)
opt = optionsHelper.substitute_definitions(opt)
# parse transforms
transforms = []
#TODO: add a resize transform in any case
transformOpts = optionsHelper.get_hierarchical_value(
opt, ['options', mode, 'transform', 'value'])
for tr in transformOpts:
trClass = tr['id']
args = optionsHelper.filter_reserved_children(tr, True)
for a in args.keys():
args[a] = optionsHelper.get_hierarchical_value(
args[a], ['value'])
if a == 'interp':
#TODO: ugly solution to convert interpolation methods
args[a] = int(args[a])
# initialize
transform = getattr(T, trClass)(**args)
transforms.append(transform)
return transforms
def verifyOptions(cls, options):
defaultOptions = cls.getDefaultOptions()
if options is None:
return {'valid': True, 'options': defaultOptions}
try:
if isinstance(options, str):
options = json.loads(options)
options = optionsHelper.substitute_definitions(options)
except Exception as e:
return {
'valid':
False,
'errors':
[f'Options are not in a proper format (message: {str(e)}).']
}
try:
# mandatory field: model config
modelConfig = optionsHelper.get_hierarchical_value(
options, ['options', 'model', 'config', 'value'])
if modelConfig is None:
raise Exception('missing model type field in options.')
opts, warnings, errors = optionsHelper.verify_options(
options['options'], autoCorrect=True)
options['options'] = opts
return {
'valid': not len(errors),
'warnings': warnings,
'errors': errors,
'options': options
}
except Exception as e:
return {
'valid':
False,
'errors': [
f'An error occurred trying to verify options (message: {str(e)}).'
]
}
def train(self, stateDict, data, updateStateFun):
'''
Main training function.
'''
# initialize model
model, stateDict, _, projectToStateMap = self.initializeModel(
stateDict, data, False)
# wrap dataset for usage with Detectron2
ignoreUnsure = optionsHelper.get_hierarchical_value(
self.options, ['options', 'train', 'ignore_unsure', 'value'],
fallback=True)
transforms = self.initializeTransforms(mode='train')
indexMap = self._get_labelclass_index_map(stateDict['labelclassMap'],
False)
try:
imageFormat = self.detectron2cfg.INPUT.FORMAT
assert imageFormat.upper() in ('RGB', 'BGR')
except:
imageFormat = 'BGR'
datasetMapper = Detectron2DatasetMapper(self.project,
self.fileServer,
transforms,
True,
imageFormat,
classIndexMap=indexMap)
dataLoader = build_detection_train_loader(
dataset=getDetectron2Data(
data, stateDict['labelclassMap'], projectToStateMap,
ignoreUnsure,
self.detectron2cfg.DATALOADER.FILTER_EMPTY_ANNOTATIONS),
mapper=datasetMapper,
total_batch_size=self.detectron2cfg.SOLVER.IMS_PER_BATCH *
comm.get_world_size(), #TODO: verify
aspect_ratio_grouping=True,
num_workers=0)
numImg = len(data['images'])
# train
model.train()
optimizer = self._build_optimizer(self.detectron2cfg, model)
scheduler = self._build_lr_scheduler(self.detectron2cfg, optimizer)
imgCount = 0
start_iter = 0 #TODO
tbar = trange(numImg)
dataLoaderIter = iter(dataLoader)
with EventStorage(start_iter) as storage:
for idx in range(numImg):
batch = next(dataLoaderIter)
storage.iter = idx #TODO: start_iter
loss_dict = model(batch)
losses = sum(loss_dict.values())
assert torch.isfinite(losses).all(), \
'Model produced Inf and/or NaN values; training was aborted. Try reducing the learning rate.'
loss_dict_reduced = {
k: v.item()
for k, v in comm.reduce_dict(loss_dict).items()
}
losses_reduced = sum(loss
for loss in loss_dict_reduced.values())
if comm.is_main_process():
storage.put_scalars(total_loss=losses_reduced,
**loss_dict_reduced)
optimizer.zero_grad()
losses.backward()
optimizer.step()
scheduler.step()
# update worker state
tbar.update(1)
imgCount += len(batch)
updateStateFun(state='PROGRESS',
message='training',
done=imgCount,
total=numImg)
stats = storage.latest()
for key in stats:
if isinstance(stats[key], tuple):
stats[key] = stats[key][0]
tbar.close()
# all done; return state dict as bytes and stats
return self.exportModelState(stateDict, model), stats
def initializeModel(self, stateDict, data, revertProjectToStateMap=False):
'''
Loads Bytes object "stateDict" through torch and looks for a Detectron2
config to initialize the model structure, optionally with pre-trained
weights if available.
Returns the model instance, the state dict, inter alia augmented with a
'labelClassMap', defining the indexing between label classes and the model,
and a list of new label classes (i.e., those that are present in the "data"
dict, but not in the model definition).
If the stateDict object is None, a new model and labelClassMap are crea-
ted from the defaults.
Note that this function does NOT modify the model w.r.t. the actually
present label classes in the project. This functionality requires re-
configuring the model's prediction output and depends on the model imple-
mentation.
'''
# check if user has forced creating a new model
forceNewModel = optionsHelper.get_hierarchical_value(
self.options, ['options', 'model', 'force', 'value'],
fallback=False)
if isinstance(forceNewModel, bool) and forceNewModel:
# print warning and reset flag
print(
f'[{self.project}] User has selected to force recreating a brand-new model.'
)
optionsHelper.set_hierarchical_value(
self.options, ['options', 'model', 'force', 'value'], False)
else:
forceNewModel = False
# load state dict
if stateDict is not None and not forceNewModel:
if not isinstance(stateDict, dict):
stateDict = torch.load(
io.BytesIO(stateDict),
map_location=lambda storage, loc: storage)
else:
stateDict = {}
# retrieve Detectron2 cfg
if 'detectron2cfg' in stateDict and not forceNewModel:
# medium priority: model overrides
self.detectron2cfg.set_new_allowed(True)
self.detectron2cfg.update(stateDict['detectron2cfg'])
# top priority: AIDE config overrides
self.detectron2cfg = self.parse_aide_config(self.options,
self.detectron2cfg)
stateDict['detectron2cfg'] = self.detectron2cfg
# check if CUDA is available; set to CPU temporarily if not
if not torch.cuda.is_available():
self.detectron2cfg.MODEL.DEVICE = 'cpu'
# construct model and load state
model = detectron2.modeling.build_model(self.detectron2cfg)
checkpointer = DetectionCheckpointerInMem(model)
if 'model' in stateDict and not forceNewModel:
# trained weights available
checkpointer.loadFromObject(stateDict)
else:
# fresh model; initialize from Detectron2 weights
checkpointer.load(self.detectron2cfg.MODEL.WEIGHTS)
# load or create labelclass map
if 'labelclassMap' in stateDict and not forceNewModel:
labelclassMap = stateDict['labelclassMap']
else:
labelclassMap = {}
# add existing label classes; any non-UUID entry will be discarded during prediction
try:
pretrainedDataset = self.detectron2cfg.DATASETS.TRAIN
if isinstance(pretrainedDataset, list) or isinstance(
pretrainedDataset, tuple):
pretrainedDataset = pretrainedDataset[0]
pretrainedMeta = MetadataCatalog.get(pretrainedDataset)
for idx, cID in enumerate(pretrainedMeta.thing_classes):
labelclassMap[cID] = idx
except:
pass
# check data for new label classes
projectToStateMap = {}
newClasses = []
for lcID in data['labelClasses']:
if lcID not in labelclassMap:
# check if original label class got re-mapped
lcMap_origin_id = data['labelClasses'][lcID].get(
'labelclass_id_model', None)
if lcMap_origin_id is None or lcMap_origin_id not in labelclassMap:
# no remapping done; class really is new
newClasses.append(lcID)
else:
# class has been re-mapped
if revertProjectToStateMap:
projectToStateMap[lcMap_origin_id] = lcID
else:
projectToStateMap[lcID] = lcMap_origin_id
stateDict['labelclassMap'] = labelclassMap
# parallelize model (if architecture supports it) #TODO: try out whether/how well this works
distributed = comm.get_world_size() > 1
if distributed:
model = DistributedDataParallel(model,
device_ids=[comm.get_local_rank()],
broadcast_buffers=False)
return model, stateDict, newClasses, projectToStateMap
def get_device(self):
device = optionsHelper.get_hierarchical_value(
self.options, ['options', 'general', 'device', 'value', 'id'])
if 'cuda' in device and not torch.cuda.is_available():
device = 'cpu'
return device
def _convertOldOptions(options, defaults):
'''
Receives options in the previous JSON encoding
and converts them to the new GUI-enhanced scheme.
Returns the new, converted options accordingly.
'''
newOptions = defaults.copy()
warnings = []
# update defaults key by key
for key in OPTIONS_MAPPING.keys():
newTokens = ['options']
newTokens.extend(key.split('.'))
oldTokens = OPTIONS_MAPPING[key].split('.')
oldValue = optionsHelper.get_hierarchical_value(
options, oldTokens, None)
if oldValue is None:
warnings.append(
f'Value for options "{key}" could not be found in given options (expected location: "{OPTIONS_MAPPING[key]}").'
)
else:
optionsHelper.set_hierarchical_value(newOptions, newTokens,
oldValue)
# take special care of the optimizer: try all possible values (only the ones present will be retained)
currentOptimType = options['train']['optim']['class']
optionsHelper.set_hierarchical_value(newOptions,
('train', 'optim', 'value'),
currentOptimType)
optionsHelper.update_hierarchical_value(
options, newOptions,
('train', 'optim', 'options', currentOptimType, 'lr', 'value'),
('train', 'optim', 'kwargs', 'lr'))
optionsHelper.update_hierarchical_value(
options, newOptions, ('train', 'optim', 'options',
currentOptimType, 'weight_decay', 'value'),
('train', 'optim', 'kwargs', 'weight_decay'))
optionsHelper.update_hierarchical_value(
options, newOptions, ('train', 'optim', 'options',
currentOptimType, 'momentum', 'value'),
('train', 'optim', 'kwargs', 'momentum'))
optionsHelper.update_hierarchical_value(
options, newOptions,
('train', 'optim', 'options', currentOptimType, 'alpha', 'value'),
('train', 'optim', 'kwargs', 'alpha'))
optionsHelper.update_hierarchical_value(
options, newOptions, ('train', 'optim', 'options',
currentOptimType, 'centered', 'value'),
('train', 'optim', 'kwargs', 'centered'))
optionsHelper.update_hierarchical_value(
options, newOptions, ('train', 'optim', 'options',
currentOptimType, 'dampening', 'value'),
('train', 'optim', 'kwargs', 'dampening'))
optionsHelper.update_hierarchical_value(
options, newOptions, ('train', 'optim', 'options',
currentOptimType, 'nesterov', 'value'),
('train', 'optim', 'kwargs', 'nesterov'))
# also take special care of the transforms
def _update_transforms(currentTransforms):
newTransforms = []
for tr in currentTr_train:
# get from template definition and then replace values
trClass = tr['class']
if trClass not in newOptions['defs']['transform']:
warnings.append(
f'Transform "{trClass}" is not defined in the new scheme and will be substituted appropriately.'
)
continue
newTr = newOptions['defs']['transform'][trClass]
for kw in tr['kwargs'].keys():
if kw == 'size':
newTr['width']['value'] = tr['kwargs']['size'][0]
newTr['height']['value'] = tr['kwargs']['size'][1]
elif kw in ('brightness', 'contrast', 'saturation', 'hue'):
newTr[kw]['minV']['value'] = 0
newTr[kw]['maxV']['value'] = tr['kwargs'][kw]
warnings.append(
f'{kw} values of transforms have been set as maximums (min: 0).'
)
elif kw in ('mean', 'std'):
newTr['mean']['r'] = tr['kwargs'][kw][0]
newTr['mean']['g'] = tr['kwargs'][kw][1]
newTr['mean']['b'] = tr['kwargs'][kw][2]
elif kw in newTr:
newTr[kw]['value'] = tr['kwargs'][kw]
newTransforms.append(newTr)
return newTransforms
currentTr_train = options['train']['transform']['kwargs']['transforms']
newTr_train = _update_transforms(currentTr_train)
newOptions['options']['train']['transform']['value'] = newTr_train
currentTr_inference = options['inference']['transform']['kwargs'][
'transforms']
newTr_inference = _update_transforms(currentTr_inference)
newOptions['options']['inference']['transform'][
'value'] = newTr_inference
print('Old RetinaNet options successfully converted to new format.')
return newOptions, warnings
def inference(self, stateDict, data, updateStateFun):
# initialize model
if stateDict is None:
raise Exception(
'No trained model state found, but required for inference.')
# read state dict from bytes
model, labelclassMap = self.initializeModel(stateDict, data)
# initialize data loader, dataset, transforms
inputSize = (int(
optionsHelper.get_hierarchical_value(
self.options,
['options', 'general', 'imageSize', 'width', 'value'])),
int(
optionsHelper.get_hierarchical_value(
self.options, [
'options', 'general', 'imageSize', 'height',
'value'
])))
transform = RetinaNet._init_transform_instances(
optionsHelper.get_hierarchical_value(
self.options, ['options', 'inference', 'transform', 'value']),
inputSize)
dataset = BoundingBoxesDataset(data=data,
fileServer=self.fileServer,
labelclassMap=labelclassMap,
transform=transform)
dataEncoder = encoder.DataEncoder(
minIoU_pos=0.5, maxIoU_neg=0.4) # IoUs don't matter for inference
collator = collation.Collator(self.project, self.dbConnector, (
inputSize[1],
inputSize[0],
), dataEncoder)
dataLoader = DataLoader(dataset=dataset,
collate_fn=collator.collate_fn,
shuffle=False)
# perform inference
response = {}
device = self.get_device()
model.to(device)
imgCount = 0
for (img, _, _, fVec, imgID) in tqdm(dataLoader):
# TODO: implement feature vectors
# if img is not None:
# dataItem = img.to(device)
# isFeatureVector = False
# else:
# dataItem = fVec.to(device)
# isFeatureVector = True
dataItem = img.to(device)
with torch.no_grad():
bboxes_pred_batch, labels_pred_batch = model(
dataItem, False) #TODO: isFeatureVector
bboxes_pred_batch, labels_pred_batch, confs_pred_batch = dataEncoder.decode(
bboxes_pred_batch.squeeze(0).cpu(),
labels_pred_batch.squeeze(0).cpu(),
inputSize,
cls_thresh=optionsHelper.get_hierarchical_value(
self.options, [
'options', 'inference', 'encoding', 'cls_thresh',
'value'
],
fallback=0.1),
nms_thresh=optionsHelper.get_hierarchical_value(
self.options, [
'options', 'inference', 'encoding', 'nms_thresh',
'value'
],
fallback=0.1),
numPred_max=int(
optionsHelper.get_hierarchical_value(self.options, [
'options', 'inference', 'encoding', 'numPred_max',
'value'
],
fallback=128)),
return_conf=True)
for i in range(len(imgID)):
bboxes_pred = bboxes_pred_batch[i]
labels_pred = labels_pred_batch[i]
confs_pred = confs_pred_batch[i]
if bboxes_pred.dim() == 2:
bboxes_pred = bboxes_pred.unsqueeze(0)
labels_pred = labels_pred.unsqueeze(0)
confs_pred = confs_pred.unsqueeze(0)
# convert bounding boxes to YOLO format
predictions = []
bboxes_pred_img = bboxes_pred[0, ...]
labels_pred_img = labels_pred[0, ...]
confs_pred_img = confs_pred[0, ...]
if len(bboxes_pred_img):
bboxes_pred_img[:, 2] -= bboxes_pred_img[:, 0]
bboxes_pred_img[:, 3] -= bboxes_pred_img[:, 1]
bboxes_pred_img[:, 0] += bboxes_pred_img[:, 2] / 2
bboxes_pred_img[:, 1] += bboxes_pred_img[:, 3] / 2
bboxes_pred_img[:, 0] /= inputSize[0]
bboxes_pred_img[:, 1] /= inputSize[1]
bboxes_pred_img[:, 2] /= inputSize[0]
bboxes_pred_img[:, 3] /= inputSize[1]
# limit to image bounds
bboxes_pred_img = torch.clamp(bboxes_pred_img, 0, 1)
# append to dict
for b in range(bboxes_pred_img.size(0)):
bbox = bboxes_pred_img[b, :]
label = labels_pred_img[b]
logits = confs_pred_img[b, :]
predictions.append({
'x':
bbox[0].item(),
'y':
bbox[1].item(),
'width':
bbox[2].item(),
'height':
bbox[3].item(),
'label':
dataset.labelclassMap_inv[label.item()],
'logits':
logits.numpy().tolist(
), #TODO: for AL criterion?
'confidence':
torch.max(logits).item()
})
response[imgID[i]] = {
'predictions': predictions,
#TODO: exception if fVec is not torch tensor: 'fVec': io.BytesIO(fVec.numpy().astype(np.float32)).getvalue()
}
# update worker state
imgCount += len(imgID)
updateStateFun(state='PROGRESS',
message='predicting',
done=imgCount,
total=len(dataLoader.dataset))
model.cpu()
if 'cuda' in device:
torch.cuda.empty_cache()
return response
def train(self, stateDict, data, updateStateFun):
'''
Initializes a model based on the given stateDict and a data loader from the
provided data and trains the model, taking into account the parameters speci-
fied in the 'options' given to the class.
Returns a serializable state dict of the resulting model.
'''
# initialize model
model, labelclassMap = self.initializeModel(
stateDict, data,
optionsHelper.get_hierarchical_value(
self.options,
['options', 'general', 'labelClasses', 'add_missing', 'value'
]),
optionsHelper.get_hierarchical_value(self.options, [
'options', 'general', 'labelClasses', 'remove_obsolete',
'value'
]))
# setup transform, data loader, dataset, optimizer, criterion
inputSize = (int(
optionsHelper.get_hierarchical_value(
self.options,
['options', 'general', 'imageSize', 'width', 'value'])),
int(
optionsHelper.get_hierarchical_value(
self.options, [
'options', 'general', 'imageSize', 'height',
'value'
])))
transform = RetinaNet._init_transform_instances(
optionsHelper.get_hierarchical_value(
self.options, ['options', 'train', 'transform', 'value']),
inputSize)
dataset = BoundingBoxesDataset(
data=data,
fileServer=self.fileServer,
labelclassMap=labelclassMap,
targetFormat='xyxy',
transform=transform,
ignoreUnsure=optionsHelper.get_hierarchical_value(
self.options,
['options', 'train', 'encoding', 'ignore_unsure', 'value'],
fallback=False))
dataEncoder = encoder.DataEncoder(
minIoU_pos=optionsHelper.get_hierarchical_value(
self.options,
['options', 'train', 'encoding', 'minIoU_pos', 'value'],
fallback=0.5),
maxIoU_neg=optionsHelper.get_hierarchical_value(
self.options,
['options', 'train', 'encoding', 'maxIoU_neg', 'value'],
fallback=0.4))
collator = collation.Collator(self.project, self.dbConnector, (
inputSize[1],
inputSize[0],
), dataEncoder)
dataLoader = DataLoader(
dataset=dataset,
collate_fn=collator.collate_fn,
shuffle=optionsHelper.get_hierarchical_value(
self.options,
['options', 'train', 'dataLoader', 'shuffle', 'value'],
fallback=True))
# optimizer
optimArgs = optionsHelper.get_hierarchical_value(
self.options, ['options', 'train', 'optim', 'value'], None)
optimArgs_out = {}
optimClass = get_class_executable(optimArgs['id'])
for key in optimArgs.keys():
if key not in optionsHelper.RESERVED_KEYWORDS:
optimArgs_out[key] = optionsHelper.get_hierarchical_value(
optimArgs[key], ['value'])
optimizer = optimClass(params=model.parameters(), **optimArgs_out)
# loss criterion
critArgs = optionsHelper.get_hierarchical_value(
self.options, ['options', 'train', 'criterion'], None)
critArgs_out = {}
for key in critArgs.keys():
if key not in optionsHelper.RESERVED_KEYWORDS:
critArgs_out[key] = optionsHelper.get_hierarchical_value(
critArgs[key], ['value'])
criterion = loss.FocalLoss(**critArgs_out)
# train model
device = self.get_device()
seed = int(
optionsHelper.get_hierarchical_value(
self.options, ['options', 'general', 'seed', 'value'],
fallback=0))
torch.manual_seed(seed)
if 'cuda' in device:
torch.cuda.manual_seed(seed)
model.to(device)
imgCount = 0
for (img, bboxes_target, labels_target, fVec, _) in tqdm(dataLoader):
img, bboxes_target, labels_target = img.to(device), \
bboxes_target.to(device), \
labels_target.to(device)
optimizer.zero_grad()
bboxes_pred, labels_pred = model(img)
loss_value = criterion(bboxes_pred, bboxes_target, labels_pred,
labels_target)
loss_value.backward()
optimizer.step()
# check for Inf and NaN values and raise exception if needed
if any([
torch.any(torch.isinf(bboxes_pred)).item(),
torch.any(torch.isinf(labels_pred)).item(),
torch.any(torch.isnan(bboxes_pred)).item(),
torch.any(torch.isnan(labels_pred)).item()
]):
raise Exception(
'Model produced Inf and/or NaN values; training was aborted. Try reducing the learning rate.'
)
# update worker state
imgCount += img.size(0)
updateStateFun(state='PROGRESS',
message='training',
done=imgCount,
total=len(dataLoader.dataset))
# all done; return state dict as bytes
return self.exportModelState(model)
def verifyOptions(options):
# get default options to compare to
defaultOptions = RetinaNet.getDefaultOptions()
# updated options with modifications made
if options is None:
updatedOptions = defaultOptions.copy()
else:
if not isinstance(options, dict):
try:
options = json.loads(options)
except Exception as e:
return {
'valid':
False,
'warnings': [],
'errors': [
f'Options are not in valid JSON format (message: "{str(e)}").'
]
}
updatedOptions = options.copy()
result = {'valid': True, 'warnings': [], 'errors': []}
if not 'defs' in updatedOptions:
# old version (without GUI formatting): convert first
updatedOptions, warnings = RetinaNet._convertOldOptions(
updatedOptions, defaultOptions)
result['warnings'].append(
'Options have been converted to new format.')
result['warnings'].extend(warnings)
# flatten and fill globals
updatedOptions = optionsHelper.substitute_definitions(updatedOptions)
# do the verification
missingClassOptions = optionsHelper.get_hierarchical_value(
updatedOptions, ['options', 'general', 'labelClasses'])
if not isinstance(missingClassOptions, dict):
updatedOptions['options']['general']['labelClasses'] = \
optionsHelper.get_hierarchical_value(defaultOptions, ['options', 'general', 'labelClasses'])
#TODO: verify rest
# verify transforms
transforms_train = updatedOptions['options']['train']['transform'][
'value']
transforms_train, w, e = RetinaNet._verify_transforms(
transforms_train, True)
result['warnings'].extend(w)
result['errors'].extend(e)
if transforms_train is None:
result['valid'] = False
else:
updatedOptions['options']['train']['transform'][
'value'] = transforms_train
transforms_inf = updatedOptions['options']['inference']['transform'][
'value']
transforms_inf, w, e = RetinaNet._verify_transforms(
transforms_inf, False)
result['warnings'].extend(w)
result['errors'].extend(e)
if transforms_inf is None:
result['valid'] = False
else:
updatedOptions['options']['inference']['transform'][
'value'] = transforms_inf
if result['valid']:
result['options'] = updatedOptions
return result
