def test_add(self):
entry1 = Entry({'width': 400, 'height': 400, 'name': 'image1'})
obs1 = Observable({'xmin': 0.0, 'name': 'car', 'type': 'object'})
dataset1 = Dataset(min_entries_attributes=['width'],
min_obs_common_attributes=['xmin'],
min_obs_type_attributes={'object': ['name']})
dataset1.add_entry(entry1, observables=[obs1])
entry2 = Entry({'width': 600, 'height': 600, 'name': 'image2'})
obs2 = Observable({
'xmin': 10.0,
'ymin': 10.0,
'name': 'person',
'other': 0,
'type': 'object'
})
dataset2 = Dataset(
min_entries_attributes=['width', 'height'],
min_obs_common_attributes=['xmin', 'ymin'],
min_obs_type_attributes={'object': ['name', 'other']})
dataset2.add_entry(entry2, observables=[obs2])
dataset = dataset1 + dataset2
self.assertEqual(len(dataset), 2)
self.assertEqual(dataset.min_entries_attributes,
dataset1.min_entries_attributes)
self.assertEqual(dataset.min_obs_common_attributes,
dataset1.min_obs_common_attributes)
self.assertEqual(dataset.min_obs_type_attributes,
dataset1.min_obs_type_attributes)
def test_merge(self):
entry1 = Entry({'width': 400, 'height': 400, 'name': 'image1'})
obs1 = Observable({'xmin': 0.0, 'name': 'car', 'type': 'object'})
dataset1 = Dataset(min_entries_attributes=['width'],
min_obs_common_attributes=['xmin'],
min_obs_type_attributes={'object': ['name']})
dataset1.add_entry(entry1, observables=[obs1])
entry2 = Entry({'width': 600, 'height': 600, 'name': 'image2'})
obs2 = Observable({
'xmin': 10.0,
'ymin': 10.0,
'name': 'person',
'other': 0,
'type': 'object'
})
dataset2 = Dataset(
min_entries_attributes=['width', 'height'],
min_obs_common_attributes=['xmin', 'ymin'],
min_obs_type_attributes={'object': ['name', 'other']})
dataset2.add_entry(entry2, observables=[obs2])
dataset = Dataset.merge([dataset1, dataset2],
attributes_merging='intersection')
self.assertEqual(len(dataset), 2)
self.assertEqual(dataset.min_entries_attributes, ['width'])
self.assertEqual(dataset.min_obs_common_attributes, ['xmin'])
self.assertEqual(dataset.min_obs_type_attributes, {'object': ['name']})
with self.assertRaises(MissingAttributes):
dataset = Dataset.merge([dataset1, dataset2],
attributes_merging='union')
def test_init(self):
obs1 = Observable({
'xmin': 0.0,
'name': 'car',
'type': 'object',
'entry_id': 42
})
self.assertEqual(obs1.typ, 'object')
self.assertEqual(obs1.entry_id, 42)
self.assertEqual(obs1['xmin'], 0.0)
obs1 = Observable({
'xmin': 0.0,
'name': 'car'
},
typ='object',
entry_id=42)
self.assertEqual(obs1.typ, 'object')
self.assertEqual(obs1.entry_id, 42)
self.assertEqual(obs1['xmin'], 0.0)
def construct_basic_objects(self):
entry1 = Entry({'width': 400, 'height': 400, 'name': 'image1'})
entry2 = Entry({'width': 600, 'height': 600, 'name': 'image2'})
obs1 = Observable({
'xmin': 10.0,
'xmax': 20.0,
'name': 'car',
'type': 'box'
})
obs2 = Observable({
'xmin': 20.0,
'xmax': 40.0,
'name': 'person',
'type': 'object'
})
dataset = Dataset(min_entries_attributes=['width'],
min_obs_common_attributes=['xmin'],
min_obs_type_attributes={'object': ['name']})
return entry1, entry2, obs1, obs2, dataset
def test_add_observable(self):
entry1, _, obs1, obs2, dataset = self.construct_basic_objects()
dataset.add_entry(entry1, observables=[obs1])
dataset.add_observable('image1', obs2)
entry, observables = dataset['image1']
self.assertEqual(len(observables), 2)
self.assertEqual(entry.obs_ids,
[observables[0].idx, observables[1].idx])
obs3 = Observable({'xmax': 0.0, 'name': 'car', 'type': 'object'})
with self.assertRaises(MissingAttributes):
dataset.add_observable('image1', obs3)
def feed(self, dataset: Dataset, clear_existing_data: bool=False,
entry_format: Optional[Dict[str, Any]] = None,
obs_format: Optional[Dict[str, Any]] = None) -> None:
if clear_existing_data:
dataset.clear_data()
entry_dataset_ids = {}
# load entry format if exists
if (self.root_dir / '.pickled' / 'entry_format.pickle').exists():
with open((self.root_dir / '.pickled' / 'entry_format.pickle'), 'rb') as handle:
entry_format = pickle.load(handle)
print('Datum reader will type entries attributes as follow :\n{}'.format(entry_format))
else:
entry_format = None
# load obs format if exists
if (self.root_dir / '.pickled' / 'obs_format.pickle').exists():
with open((self.root_dir / '.pickled' / 'obs_format.pickle'), 'rb') as handle:
obs_format = pickle.load(handle)
print('Datum reader will type observables attributes as follow :\n{}'.format(obs_format))
else:
obs_format = None
with open(self.root_dir / 'entries.json', 'r') as out_file:
entries_json = json.load(out_file)
for entry_json in entries_json:
# Format attributes values loaded from json
if entry_format:
for attr, val in entry_json['data'].items():
if attr in entry_format:
entry_json['data'][attr] = entry_format[attr](val)
entry = Entry.from_dict(entry_json)
entry_dataset_ids[entry.idx] = dataset.add_entry(entry)
with open(self.root_dir / 'observables.json', 'r') as out_file:
observables_json = json.load(out_file)
for obs_json in observables_json:
if obs_format:
for attr, val in obs_json['data'].items():
if attr in obs_format:
entry_json['data'][attr] = obs_format[attr](val)
obs = Observable.from_dict(obs_json)
dataset.add_observable(entry_dataset_ids[obs.entry_id], obs)
def add_observable(self, entry_token: Union[int, str],
observable: Union[Observable, Dict[str, Any]]) -> int:
"""Adds observable to dataset
Args:
entry_token: Id/name of entry related to observable to add to dataset.
observable: observable (Observable or dict of attributes)
to add to dataset.
Returns:
Id given to observable in dataset.
"""
entry_id, _ = self.infer_entry_id_name(entry_token)
if not isinstance(observable, Observable):
observable = Observable(observable,
idx=self._next_observable_id,
entry_id=entry_id)
else:
# Forget about observable id and entry_id as they are managed by dataset
observable._update_data({
'idx': self._next_observable_id,
'entry_id': entry_id
})
# Check that observable has common/type-specific obs attributes
type_attributes = self._min_obs_type_attributes.get(observable.typ, [])
if not (observable.check_attributes(self._min_obs_common_attributes) \
and observable.check_attributes(type_attributes)) :
raise MissingAttributes(observable)
self._observables[self._next_observable_id] = observable
self._entries[entry_id]._add_obs(self._next_observable_id)
# Apply mappers from new_obs_mappers
for mapper in self.new_obs_mappers:
if mapper.obs_type != 'all' and mapper.obs_type != observable.typ:
continue
mapper_out = mapper.apply_to(observable,
entry=self._entries[entry_id],
apply_mode='force')
if mapper_out:
self.update_observable_data(observable.idx, mapper_out)
self._next_observable_id += 1
return self._next_observable_id - 1
def apply_to(self, observable: Observable,
entry: Optional[Entry] = None,
apply_mode: str = 'force') -> Dict[str, Any]:
"""Apply mapper to input observable and optional corresponding entry.
Args:
observable: input entry.
entry: optional observable corresponding entry if mapper also takes as input
entry attributes.
apply_mode: if 'optional' mapper is not applied if entry has not required attributes.
Returns:
dict containing mapper output (key, value) pairs
"""
if self.entry_in_attrs and not entry:
raise ValueError('Mapper takes as input observable corresponding entry attributes.')
if apply_mode not in ['force', 'optional']:
raise ValueError('apply_mode should be in |"force"|"optional"|')
if self.obs_type != 'all' and self.obs_type != observable.typ:
raise ValueError('Observable mapper should apply to observables with type {} (vs. {})'
.format(self.obs_type, observable.typ))
# Check that input observable has correct attributes
if not observable.check_attributes(self.obs_in_attrs):
if apply_mode == 'force':
raise MissingAttributes(observable, self.obs_in_attrs)
return {}
# Check that input entry has correct attributes
if self.entry_in_attrs:
if not entry.check_attributes(self.entry_in_attrs):
if apply_mode == 'force':
raise MissingAttributes(entry, self.entry_in_attrs)
return {}
out = self([observable[attr] for attr in self.obs_in_attrs] +
[entry[attr] for attr in self.entry_in_attrs])
if len(self.out_attrs) > 1:
return {attr: out[k] for k, attr in enumerate(self.out_attrs)}
else:
return {self.out_attrs[0]: out}