def setUp(self):
path = './test/test_images'
source_type = 'image_dir'
params = {'path': path}
self.importer = DatasetImporter().create(source_type, params)
self.data = self.importer.import_from_source()
self.test_id_dog = 'Dogs#dog3192'
self.invalid_id = 'Dogs#dog.3206'
def setUp(self):
path = './test/test_files/panthera_csv_new.csv'
source_type = 'csv'
params = {'path': path,
'image_path_col_list': 'dir',
'capture_id_col': 'capture_id',
'attributes_col_list': ['empty', 'species', 'count', 'standing', 'resting', 'moving', 'eating', 'interacting', 'babies', 'species_panthera'],
'meta_col_list': ['survey', 'image', 'location', 'split_name']}
self.importer = DatasetImporter().create(
source_type, params)
self.data = self.importer.import_from_source()
def setUp(self):
path = './test/test_files/dataset_single_image_multi_species.csv'
source_type = 'csv'
params = {
'path': path,
'image_path_col_list': 'image',
'capture_id_col': 'capture_id',
'attributes_col_list': ['species', 'count', 'standing']
}
self.importer = DatasetImporter().create(source_type, params)
self.data = self.importer.import_from_source()
class ImportFromPantheraNew(unittest.TestCase):
""" Test Import from new Panthera CSV """
def setUp(self):
path = './test/test_files/panthera_csv_new.csv'
source_type = 'csv'
params = {'path': path,
'image_path_col_list': 'dir',
'capture_id_col': 'capture_id',
'attributes_col_list': ['empty', 'species', 'count', 'standing', 'resting', 'moving', 'eating', 'interacting', 'babies', 'species_panthera'],
'meta_col_list': ['survey', 'image', 'location', 'split_name']}
self.importer = DatasetImporter().create(
source_type, params)
self.data = self.importer.import_from_source()
def testNormalCase(self):
self.assertEqual(self.data['S1_20130521_20130704_1'],
{'labels': [{
'empty': 'species',
'species': 'otherbird',
'count': '1',
'standing': '-1',
'resting': '-1',
'moving': '-1',
'eating': '-1',
'interacting': '-1',
'babies': '-1',
'species_panthera': 'Bird'}],
'meta_data': {'survey': 'S1_20130521_20130704',
'image': 'S1__Station1__Camera1__CAM30817__2013-05-22__10-55-13.JPG',
'location': 'S1_20130521_20130704', 'split_name': 'test_S1_20130521_20130704'},
'images': ["/raw_images/S1_20130521_20130704/S1__Station1__Camera1__CAM30817__2013-05-22__10-55-13.JPG"]})
class ImportFromCSVPantheraTester(unittest.TestCase):
""" Test Import from Json """
def setUp(self):
path = './test/test_files/panthera.csv'
source_type = 'panthera_csv'
params = {'path': path}
self.importer = DatasetImporter().create(source_type, params)
self.data = self.importer.import_from_source()
def testNormalCase(self):
self.assertEqual(
self.data["record_bird_1"], {
'labels': [{
'species': 'Bird',
'count': '1'
}],
'images': ["~/a/b/record_bird_1.JPG"]
})
def testNAinCounts(self):
self.assertEqual(
self.data["record_human_NA"], {
'labels': [{
'species': 'HUMAN',
'count': 'NA'
}],
'images': ["~/a/b/record_human_NA.JPG"]
})
def testCountCats(self):
self.assertEqual(self.data["record_elephant_10"]['labels'][0]['count'],
'10')
self.assertEqual(self.data["record_elephant_11"]['labels'][0]['count'],
'11-50')
self.assertEqual(self.data["record_elephant_50"]['labels'][0]['count'],
'11-50')
self.assertEqual(self.data["record_elephant_51"]['labels'][0]['count'],
'51+')
def testDuplicateSpeciesOnlyOnceInList(self):
self.assertEqual(
self.data["record_multi_LionLion"], {
'labels': [{
'species': 'Lion',
'count': '1'
}],
'images': ["~/a/b/record_multi_LionLion.JPG"]
})
def testMultiSpeciesConsolidation(self):
all_classes = [
x['species']
for x in self.data["record_multi_ZebraGiraffe"]['labels']
]
self.assertIn('Zebra', all_classes)
self.assertIn('Giraffe', all_classes)
# self.assertIn(1, self.data["record_multi_ZebraGiraffe"]['labels']['counts'])
# self.assertIn(2, self.data["record_multi_ZebraGiraffe"]['labels']['counts'])
self.assertEqual(["~/a/b/record_multi_ZebraGiraffe.JPG"],
self.data["record_multi_ZebraGiraffe"]['images'])
class ImportFromImageDirsTester(unittest.TestCase):
""" Test Import from Image Directories """
def setUp(self):
path = './test/test_images'
source_type = 'image_dir'
params = {'path': path}
self.importer = DatasetImporter().create(source_type, params)
self.data = self.importer.import_from_source()
self.test_id_dog = 'Dogs#dog3192'
self.invalid_id = 'Dogs#dog.3206'
def testInstanceInDataSet(self):
self.assertIn(self.test_id_dog, self.data)
def testInvalidImageNameNotInResults(self):
self.assertNotIn(self.invalid_id, self.data)
def testInstanceHasCorrectClass(self):
self.assertEqual('Dogs',
self.data[self.test_id_dog]['labels'][0]['class'])
def setUp(self):
path = './test/test_files/panthera.csv'
source_type = 'panthera_csv'
params = {'path': path}
self.importer = DatasetImporter().create(source_type, params)
self.data = self.importer.import_from_source()
class ImportFromCSVSingleImageTester(unittest.TestCase):
""" Test Import from CSV """
def setUp(self):
path = './test/test_files/dataset_single_image_multi_species.csv'
source_type = 'csv'
params = {
'path': path,
'image_path_col_list': 'image',
'capture_id_col': 'capture_id',
'attributes_col_list': ['species', 'count', 'standing']
}
self.importer = DatasetImporter().create(source_type, params)
self.data = self.importer.import_from_source()
def testNormalCase(self):
self.assertEqual(
self.data["ele_1_0"], {
'labels': [{
'species': 'Elephant',
'count': '1',
'standing': '0'
}],
'images': ["/path/capture_ele.jpg"]
})
self.assertEqual(
self.data["zebra_2_1"], {
'labels': [{
'species': 'Zebra',
'count': '2',
'standing': '1'
}],
'images': ["/path/capture_zebra.jpg"]
})
def testCountCategoryImporters(self):
self.assertEqual(
self.data["wild_1050_0"], {
'labels': [{
'species': 'Wildebeest',
'count': '10-50',
'standing': '0'
}],
'images': ["/path/capture_wilde.jpg"]
})
self.assertEqual(
self.data["wild_50+_1"], {
'labels': [{
'species': 'Wildebeest',
'count': '50+',
'standing': '1'
}],
'images': ["/path/capture_wilde2.jpg"]
})
def testMultiSpeciesCase(self):
self.assertEqual(
self.data["ele_lion"], {
'labels': [{
'species': 'Elephant',
'count': '1',
'standing': '0'
}, {
'species': 'Lion',
'count': '2',
'standing': '1'
}],
'images': ["/path/capture_ele_lion.jpg"]
})
def testMissingImage(self):
""" Test Removal of Records with missing fields """
self.assertNotIn('no_image', self.data)
def testConvertMissingLabels(self):
""" Test Removal of Records with missing fields """
self.assertEqual(self.data['no_species']['labels'][0]['species'], '-1')
self.assertEqual(self.data['no_count']['labels'][0]['count'], '-1')
self.assertEqual(self.data['no_standing']['labels'][0]['standing'],
'-1')
def testInconsistendImage(self):
self.assertEqual(
self.data["ele_zebra_diff_image"], {
'labels': [{
'species': 'Elephant',
'count': '2',
'standing': '0'
}, {
'species': 'Zebra',
'count': '3',
'standing': '0'
}],
'images': ["/path/capture_ele_zebra.jpg"]
})
class ImportFromCSVMultiImageTester(unittest.TestCase):
""" Test Import from CSV """
def setUp(self):
path = './test/test_files/dataset_multi_image_multi_species.csv'
source_type = 'csv'
params = {
'path': path,
'image_path_col_list': ['image1', 'image2', 'image3'],
'capture_id_col': 'capture_id',
'attributes_col_list': ['species', 'count', 'standing']
}
self.importer = DatasetImporter().create(source_type, params)
self.data = self.importer.import_from_source()
def testNormalCase(self):
self.assertEqual(
self.data["ele_1_0"], {
'labels': [{
'species': 'Elephant',
'count': '1',
'standing': '0'
}],
'images': [
"/path/capture_ele1.jpg", "/path/capture_ele2.jpg",
"/path/capture_ele3.jpg"
]
})
def testMultiSpeciesCase(self):
self.assertEqual(
self.data["ele_lion"], {
'labels': [{
'species': 'Elephant',
'count': '1',
'standing': '0'
}, {
'species': 'Lion',
'count': '2',
'standing': '1'
}],
'images': [
"/path/capture_ele_lion1.jpg",
"/path/capture_ele_lion2.jpg",
"/path/capture_ele_lion3.jpg"
]
})
def testNotAllImages(self):
self.assertEqual(
self.data["only_one_image"], {
'labels': [{
'species': 'Elephant',
'count': '1',
'standing': '0'
}],
'images': ["/path/capture_ele3.jpg"]
})
self.assertEqual(
self.data["only_two_images"], {
'labels': [{
'species': 'Elephant',
'count': '1',
'standing': '0'
}],
'images': ["/path/capture_ele1.jpg", "/path/capture_ele2.jpg"]
})
def setUp(self):
path = './test/test_files/json_data_file.json'
source_type = 'json'
params = {'path': path}
self.importer = DatasetImporter().create(source_type, params)
self.data = self.importer.import_from_source()
class ImportFromJsonTester(unittest.TestCase):
""" Test Import from Json """
def setUp(self):
path = './test/test_files/json_data_file.json'
source_type = 'json'
params = {'path': path}
self.importer = DatasetImporter().create(source_type, params)
self.data = self.importer.import_from_source()
def testSingleSpeciesStandard(self):
self.assertEqual(
self.data["single_species_standard"], {
"labels": [{
"class": "cat",
"color_brown": "1",
"color_white": "0",
"counts": "1"
}],
"meta_data": {
"meta_1": "meta_data_1",
"meta_2": "meta_data_2"
},
"images": [
"\\images\\4715\\all\\cat\\10296725_0.jpeg",
"\\images\\4715\\all\\cat\\10296726_0.jpeg",
"\\images\\4715\\all\\cat\\10296727_0.jpeg"
]
})
def testMultiSpeciesStandard(self):
self.assertEqual(
self.data["multi_species_standard"], {
"labels": [{
"class": "cat",
"color_brown": "1",
"color_white": "0",
"counts": "1"
}, {
"class": "dog",
"color_brown": "1",
"color_white": "0",
"counts": "2"
}],
"meta_data": {
"meta_1": "meta_data_1",
"meta_2": "meta_data_2"
},
"images": [
"\\images\\4715\\all\\cat\\10296725_0.jpeg",
"\\images\\4715\\all\\cat\\10296726_0.jpeg",
"\\images\\4715\\all\\cat\\10296727_0.jpeg"
]
})
def testEmptyClassImport(self):
self.assertEqual(
self.data["empty_image"], {
"labels": [{
"class": "empty",
"color_brown": "0",
"color_white": "0",
"counts": "0"
}],
"images": [
"\\images\\4715\\all\\cat\\10296725_0.jpeg",
"\\images\\4715\\all\\cat\\10296726_0.jpeg",
"\\images\\4715\\all\\cat\\10296727_0.jpeg"
]
})
def testMultiColorImport(self):
self.assertEqual(
self.data["single_species_multi_color"], {
"labels": [{
"class": "cat",
"color_brown": "1",
"color_white": "1",
"counts": "1"
}],
"meta_data": {
"meta_1": "meta_data_1",
"meta_2": "meta_data_2"
},
"images": [
"\\images\\4715\\all\\cat\\10296725_0.jpeg",
"\\images\\4715\\all\\cat\\10296726_0.jpeg",
"\\images\\4715\\all\\cat\\10296727_0.jpeg"
]
})
def testEmptyImagesListNotInResults(self):
self.assertNotIn("empty_images_list", self.data)
def testMultiSpeciesNoOtherAttr(self):
self.assertEqual(
self.data["multi_species_no_other_attr"], {
"labels": [{
"class": "cat"
}, {
"class": "dog"
}],
"meta_data": {
"meta_1": "meta_data_1",
"meta_2": "meta_data_2"
},
"images": [
"\\images\\4715\\all\\dog\\10300728_0.jpeg",
"\\images\\4715\\all\\dog\\10300742_0.jpeg"
]
})
def testNonStringLabelNotInResult(self):
self.assertNotIn("non_string_label", self.data)
def testInvalidMetaDataNotInResult(self):
self.assertNotIn("meta_data_invalid", self.data)
def testImportNoMetaData(self):
self.assertEqual(
self.data["no_meta_data"], {
"labels": [{
"class": "cat",
"color_brown": "1",
"color_white": "0",
"counts": "10-50"
}],
"images": [
"\\images\\4715\\all\\cat\\10296725_0.jpeg",
"\\images\\4715\\all\\cat\\10296726_0.jpeg",
"\\images\\4715\\all\\cat\\10296727_0.jpeg"
]
})
def testEmptyImagesNotInResults(self):
self.assertNotIn("only_empty_images_string", self.data)
def create_from_source(self, type, params):
""" Create Dataset Inventory from a specific Source """
importer = DatasetImporter().create(type, params)
self.data_inventory = importer.import_from_source()