def test_merge(self):
entry1 = Entry({'width': 400, 'height': 400, 'name': 'image1'})
obs1 = Observable({'xmin': 0.0, 'name': 'car', 'type': 'object'})
dataset1 = Dataset(min_entries_attributes=['width'],
min_obs_common_attributes=['xmin'],
min_obs_type_attributes={'object': ['name']})
dataset1.add_entry(entry1, observables=[obs1])
entry2 = Entry({'width': 600, 'height': 600, 'name': 'image2'})
obs2 = Observable({
'xmin': 10.0,
'ymin': 10.0,
'name': 'person',
'other': 0,
'type': 'object'
})
dataset2 = Dataset(
min_entries_attributes=['width', 'height'],
min_obs_common_attributes=['xmin', 'ymin'],
min_obs_type_attributes={'object': ['name', 'other']})
dataset2.add_entry(entry2, observables=[obs2])
dataset = Dataset.merge([dataset1, dataset2],
attributes_merging='intersection')
self.assertEqual(len(dataset), 2)
self.assertEqual(dataset.min_entries_attributes, ['width'])
self.assertEqual(dataset.min_obs_common_attributes, ['xmin'])
self.assertEqual(dataset.min_obs_type_attributes, {'object': ['name']})
with self.assertRaises(MissingAttributes):
dataset = Dataset.merge([dataset1, dataset2],
attributes_merging='union')
def test_add_entry(self):
entry1, entry2, obs1, obs2, dataset = self.construct_basic_objects()
dataset.add_entry(entry1, observables=[obs1])
entry, obs = dataset['image1']
self.assertEqual(entry.idx, 0)
self.assertEqual(entry.name, 'image1')
self.assertEqual(entry.obs_ids, [0])
self.assertEqual(entry['width'], 400)
self.assertEqual(obs[0].idx, 0)
self.assertEqual(obs[0].entry_id, 0)
self.assertEqual(obs[0]['xmin'], 0.0)
dataset.add_entry(entry2, observables=[obs2])
entry, obs = dataset['image2']
self.assertEqual(entry.idx, 1)
self.assertEqual(entry.name, 'image2')
self.assertEqual(entry.obs_ids, [1])
self.assertEqual(entry['width'], 600)
self.assertEqual(obs[0].idx, 1)
self.assertEqual(obs[0].entry_id, 1)
self.assertEqual(obs[0]['xmin'], 10.0)
entry3 = Entry({'height': 400, 'name': 'image3'})
with self.assertRaises(MissingAttributes):
dataset.add_entry(entry3)
entry3 = Entry({'width': 400, 'height': 400, 'name': 'image1'})
with self.assertRaises(EntryAlreadyExists):
dataset.add_entry(entry3)
def test_add(self):
entry1 = Entry({'width': 400, 'height': 400, 'name': 'image1'})
obs1 = Observable({'xmin': 0.0, 'name': 'car', 'type': 'object'})
dataset1 = Dataset(min_entries_attributes=['width'],
min_obs_common_attributes=['xmin'],
min_obs_type_attributes={'object': ['name']})
dataset1.add_entry(entry1, observables=[obs1])
entry2 = Entry({'width': 600, 'height': 600, 'name': 'image2'})
obs2 = Observable({
'xmin': 10.0,
'ymin': 10.0,
'name': 'person',
'other': 0,
'type': 'object'
})
dataset2 = Dataset(
min_entries_attributes=['width', 'height'],
min_obs_common_attributes=['xmin', 'ymin'],
min_obs_type_attributes={'object': ['name', 'other']})
dataset2.add_entry(entry2, observables=[obs2])
dataset = dataset1 + dataset2
self.assertEqual(len(dataset), 2)
self.assertEqual(dataset.min_entries_attributes,
dataset1.min_entries_attributes)
self.assertEqual(dataset.min_obs_common_attributes,
dataset1.min_obs_common_attributes)
self.assertEqual(dataset.min_obs_type_attributes,
dataset1.min_obs_type_attributes)
def test_remove_obs(self):
entry1 = Entry({
'width': 400,
'height': 400,
'name': 'image1',
'idx': 42,
'obs_ids': [1, 2]
})
entry1._remove_obs(2)
self.assertEqual(entry1.obs_ids, [1])
def test_check_attributes(self):
entry1 = Entry({
'width': 400,
'height': 400,
'name': 'image1',
'idx': 42,
'obs_ids': [1, 2]
})
check = entry1.check_attributes(['width', 'name'])
self.assertEqual(check, True)
check = entry1.check_attributes(['width', 'depth'])
self.assertEqual(check, False)
def test__update_data(self):
entry1 = Entry({
'width': 400,
'height': 400,
'name': 'image1',
'idx': 42,
'obs_ids': [1, 2]
})
entry1._update_data({'width': 600, 'depth': 3, 'name': 'image1.1'})
self.assertEqual(entry1.name, 'image1.1')
self.assertEqual(entry1['width'], 600)
self.assertEqual(entry1['depth'], 3)
def add_entry(
self,
entry: Union[Entry, Dict[str, Any]],
observables: Optional[List[Union[Observable, Dict[str, Any]]]] = None
) -> int:
"""Adds an entry to dataset and observables related to it
Args:
entry: entry to add to dataset.
observables: List of observables (Observable/Dict)
to associate with entry.
Returns:
Id given to entry in dataset.
"""
if not isinstance(entry, Entry):
if 'name' not in entry:
entry = Entry(entry,
idx=self._next_entry_id,
name=str(self._next_entry_id))
else:
entry = Entry(entry, idx=self._next_entry_id)
else:
# Forget about entry id and obs_ds as they are managed by dataset
update = {'idx': self._next_entry_id, 'obs_ids': []}
if entry.name is None:
update['name'] = str(self._next_entry_id)
entry._update_data(update)
if not entry.check_attributes(self._min_entries_attributes):
raise MissingAttributes(entry)
if entry.name in self._names_ids_map:
raise EntryAlreadyExists(entry.name, 'Entry')
self._entries[self._next_entry_id] = entry
self._names_ids_map[entry.name] = self._next_entry_id
# Apply mappers from new_entry_mappers
for mapper in self.new_entry_mappers:
mapper_out = mapper.apply_to(entry, apply_mode='force')
if mapper_out:
self.update_entry_data(entry.idx, mapper_out)
if observables:
for observable in observables:
self.add_observable(entry.idx, observable)
self._next_entry_id += 1
return self._next_entry_id - 1
def test_get(self):
entry1 = Entry({
'width': 400,
'height': 400,
'name': 'image1',
'idx': 42,
'obs_ids': [1, 2]
})
with self.assertRaises(MissingAttributes):
entry1['depth']
def construct_basic_objects(self):
entry1 = Entry({'width': 400, 'height': 400, 'name': 'image1'})
entry2 = Entry({'width': 600, 'height': 600, 'name': 'image2'})
obs1 = Observable({
'xmin': 10.0,
'xmax': 20.0,
'name': 'car',
'type': 'box'
})
obs2 = Observable({
'xmin': 20.0,
'xmax': 40.0,
'name': 'person',
'type': 'object'
})
dataset = Dataset(min_entries_attributes=['width'],
min_obs_common_attributes=['xmin'],
min_obs_type_attributes={'object': ['name']})
return entry1, entry2, obs1, obs2, dataset
def test_init(self):
entry1 = Entry({
'width': 400,
'height': 400,
'name': 'image1',
'idx': 42,
'obs_ids': [1, 2]
})
self.assertEqual(entry1.name, 'image1')
self.assertEqual(entry1.idx, 42)
self.assertEqual(entry1.obs_ids, [1, 2])
self.assertEqual(entry1['width'], 400)
entry1 = Entry({
'width': 400,
'height': 400
},
name='image1',
idx=42,
obs_ids=[1, 2])
self.assertEqual(entry1.name, 'image1')
self.assertEqual(entry1.idx, 42)
self.assertEqual(entry1.obs_ids, [1, 2])
self.assertEqual(entry1['width'], 400)
def feed(self, dataset: Dataset, clear_existing_data: bool=False,
entry_format: Optional[Dict[str, Any]] = None,
obs_format: Optional[Dict[str, Any]] = None) -> None:
if clear_existing_data:
dataset.clear_data()
entry_dataset_ids = {}
# load entry format if exists
if (self.root_dir / '.pickled' / 'entry_format.pickle').exists():
with open((self.root_dir / '.pickled' / 'entry_format.pickle'), 'rb') as handle:
entry_format = pickle.load(handle)
print('Datum reader will type entries attributes as follow :\n{}'.format(entry_format))
else:
entry_format = None
# load obs format if exists
if (self.root_dir / '.pickled' / 'obs_format.pickle').exists():
with open((self.root_dir / '.pickled' / 'obs_format.pickle'), 'rb') as handle:
obs_format = pickle.load(handle)
print('Datum reader will type observables attributes as follow :\n{}'.format(obs_format))
else:
obs_format = None
with open(self.root_dir / 'entries.json', 'r') as out_file:
entries_json = json.load(out_file)
for entry_json in entries_json:
# Format attributes values loaded from json
if entry_format:
for attr, val in entry_json['data'].items():
if attr in entry_format:
entry_json['data'][attr] = entry_format[attr](val)
entry = Entry.from_dict(entry_json)
entry_dataset_ids[entry.idx] = dataset.add_entry(entry)
with open(self.root_dir / 'observables.json', 'r') as out_file:
observables_json = json.load(out_file)
for obs_json in observables_json:
if obs_format:
for attr, val in obs_json['data'].items():
if attr in obs_format:
entry_json['data'][attr] = obs_format[attr](val)
obs = Observable.from_dict(obs_json)
dataset.add_observable(entry_dataset_ids[obs.entry_id], obs)
def apply_to(self, entry: Entry, apply_mode: str = 'force') -> Dict[str, Any]:
"""Apply mapper to input entry.
Args:
entry: input entry
apply_mode: if 'optional' mapper is not applied if entry has not required attributes.
Returns:
dict containing mapper output (key, value) pairs
"""
if apply_mode not in ['force', 'optional']:
raise ValueError('apply_mode should be in |"force"|"optional"|')
if not entry.check_attributes(self.in_attrs):
if apply_mode == 'force':
raise MissingAttributes(entry, self.in_attrs)
return {}
out = self([entry[attr] for attr in self.in_attrs])
if len(self.out_attrs) > 1:
return {attr: out[k] for k, attr in enumerate(self.out_attrs)}
else:
return {self.out_attrs[0]: out}