with open(output_json,'w') as f:
json.dump(output_data,f,indent=2)
#%% Validate .json file
from data_management.databases import sanity_check_json_db
options = sanity_check_json_db.SanityCheckOptions()
options.baseDir = os.path.join(base_folder,'images'); assert os.path.isdir(options.baseDir)
options.bCheckImageSizes = False
options.bCheckImageExistence = False
options.bFindUnusedImages = False
_, _, _ = sanity_check_json_db.sanity_check_json_db(output_json, options)
#%% Preview labels
from visualization import visualize_db
viz_options = visualize_db.DbVizOptions()
viz_options.num_to_visualize = 100
viz_options.trim_to_images_with_bboxes = False
viz_options.add_search_links = False
viz_options.sort_by_filename = False
viz_options.parallelize_rendering = True
viz_options.include_filename_links = True
# viz_options.classes_to_exclude = ['test']
print('Finished writing .json file with {} images, {} annotations, and {} categories'.format(
len(images),len(annotations),len(categories)))
#%% Validate output
from data_management.databases import sanity_check_json_db
options = sanity_check_json_db.SanityCheckOptions()
options.baseDir = image_directory
options.bCheckImageSizes = False
options.bCheckImageExistence = False
options.bFindUnusedImages = False
sortedCategories, data = sanity_check_json_db.sanity_check_json_db(output_json_file,options)
#%% Preview labels
from visualization import visualize_db
from data_management.databases import sanity_check_json_db
viz_options = visualize_db.DbVizOptions()
viz_options.num_to_visualize = 1000
viz_options.trim_to_images_with_bboxes = False
viz_options.add_search_links = True
viz_options.sort_by_filename = False
viz_options.parallelize_rendering = True
html_output_file,image_db = visualize_db.process_images(db_path=output_json_file,
output_dir=os.path.join(output_base,'preview'),
zip.close()
print('\nFinished writing {}, added {} files'.format(zipfilename,n_images_added))
#%% Sanity-check .json file
from data_management.databases import sanity_check_json_db
options = sanity_check_json_db.SanityCheckOptions()
options.baseDir = output_public_folder
options.bCheckImageSizes = False
options.bCheckImageExistence = True
options.bFindUnusedImages = False
sortedCategories, data, errorInfo = sanity_check_json_db.sanity_check_json_db(json_filename,options)
# This will produce some validation errors, because this zipfile doesn't include humans
assert(len(errorInfo['validationErrors']) == len(human_image_ids))
#%% Zip up .json and .csv files
def zip_single_file(fn,zipfilename=None):
'''
Zips a single file fn, by default to fn.zip
Discards path information, only uses fn's base name.
'''
if zipfilename is None:
zipfilename = fn + '.zip'
json_data['info'] = info
json.dump(json_data, open(output_json_file, 'w'), indent=4)
print('Finished writing .json file with {} images, {} annotations, and {} categories'.format(
len(images), len(annotations), len(categories)))
#%% Validate output
from data_management.databases import sanity_check_json_db
options = sanity_check_json_db.SanityCheckOptions()
options.baseDir = image_directory
options.bCheckImageSizes = False
options.bCheckImageExistence = False
options.bFindUnusedImages = False
data = sanity_check_json_db.sanity_check_json_db(output_json_file,options)
#%% Preview labels
from visualization import visualize_db
from data_management.databases import sanity_check_json_db
viz_options = visualize_db.DbVizOptions()
viz_options.num_to_visualize = None
viz_options.trim_to_images_with_bboxes = False
viz_options.add_search_links = True
viz_options.sort_by_filename = False
viz_options.parallelize_rendering = True
viz_options.classes_to_exclude = ['empty']
html_output_file,image_db = visualize_db.process_images(db_path=output_json_file,
print(
'Finished writing .json file with {} images, {} annotations, and {} categories'
.format(len(images), len(annotations), len(categories)))
#%% Validate output
from data_management.databases import sanity_check_json_db
fn = output_file
options = sanity_check_json_db.SanityCheckOptions()
options.baseDir = image_directory
options.bCheckImageSizes = False
options.bCheckImageExistence = True
options.bFindUnusedImages = True
sortedCategories, data = sanity_check_json_db.sanity_check_json_db(fn, options)
#%% Preview labels
from visualization import visualize_db
from data_management.databases import sanity_check_json_db
viz_options = visualize_db.DbVizOptions()
viz_options.num_to_visualize = None
viz_options.trim_to_images_with_bboxes = False
viz_options.add_search_links = True
viz_options.sort_by_filename = False
viz_options.parallelize_rendering = True
html_output_file, image_db = visualize_db.process_images(
db_path=output_file,
output_dir=os.path.join(base_directory, 'preview'),
# with open(image_json,'r') as f:
# data = json.load(f)
indexedData = IndexedJsonDb(image_json)
print('Done loading data')
#%% Sanity-check data
options = sanity_check_json_db.SanityCheckOptions()
options.baseDir = image_base
options.bCheckImageSizes = False
options.bCheckImageExistence = True
options.bFindUnusedImages = False
sortedCategories = sanity_check_json_db.sanity_check_json_db(
indexedData.db, options)
#%% Label previews
viz_options = visualize_db.DbVizOptions()
viz_options.num_to_visualize = 500
viz_options.trim_to_images_with_bboxes = False
viz_options.add_search_links = True
viz_options.sort_by_filename = False
html_output_file, image_db = visualize_db.process_images(
indexedData.db, os.path.join(output_base, 'preview'), image_base,
viz_options)
os.startfile(html_output_file)
#%% Collect images to annotate
json_data['annotations'] = annotations
json_data['categories'] = categories
json_data['info'] = info
json.dump(json_data,open(outputFile,'w'),indent=4)
print('Finished writing .json file with {} images, {} annotations, and {} categories'.format(
len(images),len(annotations),len(categories)))
#%% Check database integrity
options = sanity_check_json_db.SanityCheckOptions()
options.baseDir = imageBaseDir
options.bCheckImageSizes = False
options.bFindUnusedImages = False
sanity_check_json_db.sanity_check_json_db(outputFile, options)
#%% Preview a few images to make sure labels were passed along sensibly
db_path = outputFile
output_dir = os.path.join(baseDir,'label_preview')
image_base_dir = imageBaseDir
options = visualize_db.DbVizOptions()
options.num_to_visualize = 100
htmlOutputFile = visualize_db.process_images(db_path,output_dir,image_base_dir,options)
#%% One-time processing step: copy images to a flat directory for annotation
if False:
data['annotations'] = train_annotations
json.dump(data, open(train_fn, 'w'), indent=2)
data['images'] = test_images
data['annotations'] = test_annotations
json.dump(data, open(test_fn, 'w'), indent=2)
#%% Validate .json files
options = sanity_check_json_db.SanityCheckOptions()
options.baseDir = input_base_dir
options.bCheckImageSizes = False
options.bCheckImageExistence = True
options.bFindUnusedImages = True
sortedCategories, data = sanity_check_json_db.sanity_check_json_db(
output_json_filename, options)
sortedCategories, data = sanity_check_json_db.sanity_check_json_db(
train_fn, options)
sortedCategories, data = sanity_check_json_db.sanity_check_json_db(
test_fn, options)
#%% Preview labels
viz_options = visualize_db.DbVizOptions()
viz_options.num_to_visualize = 2000
viz_options.trim_to_images_with_bboxes = False
viz_options.add_search_links = False
viz_options.sort_by_filename = False
viz_options.parallelize_rendering = True
viz_options.classes_to_exclude = ['test']
html_output_file, image_db = visualize_db.process_images(
json.dump(data, open(outputJsonFilenameSet1, 'w'), indent=4)
print('Finished writing json to {}'.format(outputJsonFilenameSet1))
data = {}
data['info'] = infoSet2
data['images'] = imagesSet2
data['annotations'] = annotationsSet2
data['categories'] = categoriesSet2
json.dump(data, open(outputJsonFilenameSet2, 'w'), indent=4)
print('Finished writing json to {}'.format(outputJsonFilenameSet2))
#%% Sanity-check final set 1 .json file
from data_management.databases import sanity_check_json_db
options = sanity_check_json_db.SanityCheckOptions()
sortedCategories, data = sanity_check_json_db.sanity_check_json_db(
outputJsonFilenameSet1, options)
sortedCategories
# python sanity_check_json_db.py --bCheckImageSizes --baseDir "E:\wildlife_data\missouri_camera_traps" "E:\wildlife_data\missouri_camera_traps\missouri_camera_traps_set1.json"
#%% Generate previews
from visualization import visualize_db
output_dir = os.path.join(baseDir, 'preview')
options = visualize_db.DbVizOptions()
options.num_to_visualize = 1000
options.sort_by_filename = False
options.classes_to_exclude = None
data = json.load(f)
images = data['images']
annotations = data['annotations']
categories = data['categories']
info = data['info']
print('Finished reading input .json')
#%% Rev version number, update field names and types
assert (info['version'] == 'Caltech Camera Traps - v2')
info['version'] = 2.1
info[
'description'] = 'Caltech Camera Traps: camera trap images collected from the NPS and the USGS with help from Justin Brown and Erin Boydston'
for image in images:
assert 'location' in image and isinstance(image['location'], int)
image['location'] = str(image['location'])
#%% Write json file
json.dump(data, open(outputJsonFile, 'w'), indent=4)
print('Finished writing output .json to {}'.format(outputJsonFile))
#%% Check output data file
sanity_check_json_db.sanity_check_json_db(outputJsonFile)
data['categories'] = categories
with open(output_json_filename, 'w') as f:
json.dump(data, f, indent=1)
print('Finished writing json to {}'.format(output_json_filename))
#%% Validate .json file
options = sanity_check_json_db.SanityCheckOptions()
options.baseDir = output_base
options.bCheckImageSizes = False
options.bCheckImageExistence = False
options.bFindUnusedImages = False
_, _, _ = sanity_check_json_db.sanity_check_json_db(output_json_filename,
options)
#%% Map relative paths to annotation categories
category_id_to_category_names = {
c['id']: c['name']
for c in data['categories']
}
image_id_to_category_names = defaultdict(list)
# ann = data['annotations'][0]
for ann in data['annotations']:
category_name = category_id_to_category_names[ann['category_id']]
image_id_to_category_names[ann['image_id']].append(category_name)
#%% Copy images to output
'Finished writing .json file with {} images, {} annotations, and {} categories'
.format(len(images), len(annotations), len(categories)))
#%% Clean start
### Everything after this should work from a clean start ###
#%% Validate output
fn = output_json_file
options = sanity_check_json_db.SanityCheckOptions()
options.baseDir = image_base_folder
options.bCheckImageSizes = False
options.bCheckImageExistence = False
options.bFindUnusedImages = False
sorted_categories, data, errors = sanity_check_json_db.sanity_check_json_db(
fn, options)
#%% Preview animal labels
viz_options = visualize_db.DbVizOptions()
viz_options.num_to_visualize = 3000
viz_options.trim_to_images_with_bboxes = False
viz_options.add_search_links = False
viz_options.sort_by_filename = False
viz_options.parallelize_rendering = True
html_output_file, image_db = visualize_db.process_images(
db_path=output_json_file,
output_dir=os.path.join(output_dir_base, 'preview_animals'),
image_base_dir=image_base_folder,
options=viz_options)
os.startfile(html_output_file)
