with open(output_json,'w') as f: json.dump(output_data,f,indent=2) #%% Validate .json file from data_management.databases import sanity_check_json_db options = sanity_check_json_db.SanityCheckOptions() options.baseDir = os.path.join(base_folder,'images'); assert os.path.isdir(options.baseDir) options.bCheckImageSizes = False options.bCheckImageExistence = False options.bFindUnusedImages = False _, _, _ = sanity_check_json_db.sanity_check_json_db(output_json, options) #%% Preview labels from visualization import visualize_db viz_options = visualize_db.DbVizOptions() viz_options.num_to_visualize = 100 viz_options.trim_to_images_with_bboxes = False viz_options.add_search_links = False viz_options.sort_by_filename = False viz_options.parallelize_rendering = True viz_options.include_filename_links = True # viz_options.classes_to_exclude = ['test']
print('Finished writing .json file with {} images, {} annotations, and {} categories'.format( len(images),len(annotations),len(categories))) #%% Validate output from data_management.databases import sanity_check_json_db options = sanity_check_json_db.SanityCheckOptions() options.baseDir = image_directory options.bCheckImageSizes = False options.bCheckImageExistence = False options.bFindUnusedImages = False sortedCategories, data = sanity_check_json_db.sanity_check_json_db(output_json_file,options) #%% Preview labels from visualization import visualize_db from data_management.databases import sanity_check_json_db viz_options = visualize_db.DbVizOptions() viz_options.num_to_visualize = 1000 viz_options.trim_to_images_with_bboxes = False viz_options.add_search_links = True viz_options.sort_by_filename = False viz_options.parallelize_rendering = True html_output_file,image_db = visualize_db.process_images(db_path=output_json_file, output_dir=os.path.join(output_base,'preview'),
zip.close() print('\nFinished writing {}, added {} files'.format(zipfilename,n_images_added)) #%% Sanity-check .json file from data_management.databases import sanity_check_json_db options = sanity_check_json_db.SanityCheckOptions() options.baseDir = output_public_folder options.bCheckImageSizes = False options.bCheckImageExistence = True options.bFindUnusedImages = False sortedCategories, data, errorInfo = sanity_check_json_db.sanity_check_json_db(json_filename,options) # This will produce some validation errors, because this zipfile doesn't include humans assert(len(errorInfo['validationErrors']) == len(human_image_ids)) #%% Zip up .json and .csv files def zip_single_file(fn,zipfilename=None): ''' Zips a single file fn, by default to fn.zip Discards path information, only uses fn's base name. ''' if zipfilename is None: zipfilename = fn + '.zip'
json_data['info'] = info json.dump(json_data, open(output_json_file, 'w'), indent=4) print('Finished writing .json file with {} images, {} annotations, and {} categories'.format( len(images), len(annotations), len(categories))) #%% Validate output from data_management.databases import sanity_check_json_db options = sanity_check_json_db.SanityCheckOptions() options.baseDir = image_directory options.bCheckImageSizes = False options.bCheckImageExistence = False options.bFindUnusedImages = False data = sanity_check_json_db.sanity_check_json_db(output_json_file,options) #%% Preview labels from visualization import visualize_db from data_management.databases import sanity_check_json_db viz_options = visualize_db.DbVizOptions() viz_options.num_to_visualize = None viz_options.trim_to_images_with_bboxes = False viz_options.add_search_links = True viz_options.sort_by_filename = False viz_options.parallelize_rendering = True viz_options.classes_to_exclude = ['empty'] html_output_file,image_db = visualize_db.process_images(db_path=output_json_file,
print( 'Finished writing .json file with {} images, {} annotations, and {} categories' .format(len(images), len(annotations), len(categories))) #%% Validate output from data_management.databases import sanity_check_json_db fn = output_file options = sanity_check_json_db.SanityCheckOptions() options.baseDir = image_directory options.bCheckImageSizes = False options.bCheckImageExistence = True options.bFindUnusedImages = True sortedCategories, data = sanity_check_json_db.sanity_check_json_db(fn, options) #%% Preview labels from visualization import visualize_db from data_management.databases import sanity_check_json_db viz_options = visualize_db.DbVizOptions() viz_options.num_to_visualize = None viz_options.trim_to_images_with_bboxes = False viz_options.add_search_links = True viz_options.sort_by_filename = False viz_options.parallelize_rendering = True html_output_file, image_db = visualize_db.process_images( db_path=output_file, output_dir=os.path.join(base_directory, 'preview'),
# with open(image_json,'r') as f: # data = json.load(f) indexedData = IndexedJsonDb(image_json) print('Done loading data') #%% Sanity-check data options = sanity_check_json_db.SanityCheckOptions() options.baseDir = image_base options.bCheckImageSizes = False options.bCheckImageExistence = True options.bFindUnusedImages = False sortedCategories = sanity_check_json_db.sanity_check_json_db( indexedData.db, options) #%% Label previews viz_options = visualize_db.DbVizOptions() viz_options.num_to_visualize = 500 viz_options.trim_to_images_with_bboxes = False viz_options.add_search_links = True viz_options.sort_by_filename = False html_output_file, image_db = visualize_db.process_images( indexedData.db, os.path.join(output_base, 'preview'), image_base, viz_options) os.startfile(html_output_file) #%% Collect images to annotate
json_data['annotations'] = annotations json_data['categories'] = categories json_data['info'] = info json.dump(json_data,open(outputFile,'w'),indent=4) print('Finished writing .json file with {} images, {} annotations, and {} categories'.format( len(images),len(annotations),len(categories))) #%% Check database integrity options = sanity_check_json_db.SanityCheckOptions() options.baseDir = imageBaseDir options.bCheckImageSizes = False options.bFindUnusedImages = False sanity_check_json_db.sanity_check_json_db(outputFile, options) #%% Preview a few images to make sure labels were passed along sensibly db_path = outputFile output_dir = os.path.join(baseDir,'label_preview') image_base_dir = imageBaseDir options = visualize_db.DbVizOptions() options.num_to_visualize = 100 htmlOutputFile = visualize_db.process_images(db_path,output_dir,image_base_dir,options) #%% One-time processing step: copy images to a flat directory for annotation if False:
data['annotations'] = train_annotations json.dump(data, open(train_fn, 'w'), indent=2) data['images'] = test_images data['annotations'] = test_annotations json.dump(data, open(test_fn, 'w'), indent=2) #%% Validate .json files options = sanity_check_json_db.SanityCheckOptions() options.baseDir = input_base_dir options.bCheckImageSizes = False options.bCheckImageExistence = True options.bFindUnusedImages = True sortedCategories, data = sanity_check_json_db.sanity_check_json_db( output_json_filename, options) sortedCategories, data = sanity_check_json_db.sanity_check_json_db( train_fn, options) sortedCategories, data = sanity_check_json_db.sanity_check_json_db( test_fn, options) #%% Preview labels viz_options = visualize_db.DbVizOptions() viz_options.num_to_visualize = 2000 viz_options.trim_to_images_with_bboxes = False viz_options.add_search_links = False viz_options.sort_by_filename = False viz_options.parallelize_rendering = True viz_options.classes_to_exclude = ['test'] html_output_file, image_db = visualize_db.process_images(
json.dump(data, open(outputJsonFilenameSet1, 'w'), indent=4) print('Finished writing json to {}'.format(outputJsonFilenameSet1)) data = {} data['info'] = infoSet2 data['images'] = imagesSet2 data['annotations'] = annotationsSet2 data['categories'] = categoriesSet2 json.dump(data, open(outputJsonFilenameSet2, 'w'), indent=4) print('Finished writing json to {}'.format(outputJsonFilenameSet2)) #%% Sanity-check final set 1 .json file from data_management.databases import sanity_check_json_db options = sanity_check_json_db.SanityCheckOptions() sortedCategories, data = sanity_check_json_db.sanity_check_json_db( outputJsonFilenameSet1, options) sortedCategories # python sanity_check_json_db.py --bCheckImageSizes --baseDir "E:\wildlife_data\missouri_camera_traps" "E:\wildlife_data\missouri_camera_traps\missouri_camera_traps_set1.json" #%% Generate previews from visualization import visualize_db output_dir = os.path.join(baseDir, 'preview') options = visualize_db.DbVizOptions() options.num_to_visualize = 1000 options.sort_by_filename = False options.classes_to_exclude = None
data = json.load(f) images = data['images'] annotations = data['annotations'] categories = data['categories'] info = data['info'] print('Finished reading input .json') #%% Rev version number, update field names and types assert (info['version'] == 'Caltech Camera Traps - v2') info['version'] = 2.1 info[ 'description'] = 'Caltech Camera Traps: camera trap images collected from the NPS and the USGS with help from Justin Brown and Erin Boydston' for image in images: assert 'location' in image and isinstance(image['location'], int) image['location'] = str(image['location']) #%% Write json file json.dump(data, open(outputJsonFile, 'w'), indent=4) print('Finished writing output .json to {}'.format(outputJsonFile)) #%% Check output data file sanity_check_json_db.sanity_check_json_db(outputJsonFile)
data['categories'] = categories with open(output_json_filename, 'w') as f: json.dump(data, f, indent=1) print('Finished writing json to {}'.format(output_json_filename)) #%% Validate .json file options = sanity_check_json_db.SanityCheckOptions() options.baseDir = output_base options.bCheckImageSizes = False options.bCheckImageExistence = False options.bFindUnusedImages = False _, _, _ = sanity_check_json_db.sanity_check_json_db(output_json_filename, options) #%% Map relative paths to annotation categories category_id_to_category_names = { c['id']: c['name'] for c in data['categories'] } image_id_to_category_names = defaultdict(list) # ann = data['annotations'][0] for ann in data['annotations']: category_name = category_id_to_category_names[ann['category_id']] image_id_to_category_names[ann['image_id']].append(category_name) #%% Copy images to output
'Finished writing .json file with {} images, {} annotations, and {} categories' .format(len(images), len(annotations), len(categories))) #%% Clean start ### Everything after this should work from a clean start ### #%% Validate output fn = output_json_file options = sanity_check_json_db.SanityCheckOptions() options.baseDir = image_base_folder options.bCheckImageSizes = False options.bCheckImageExistence = False options.bFindUnusedImages = False sorted_categories, data, errors = sanity_check_json_db.sanity_check_json_db( fn, options) #%% Preview animal labels viz_options = visualize_db.DbVizOptions() viz_options.num_to_visualize = 3000 viz_options.trim_to_images_with_bboxes = False viz_options.add_search_links = False viz_options.sort_by_filename = False viz_options.parallelize_rendering = True html_output_file, image_db = visualize_db.process_images( db_path=output_json_file, output_dir=os.path.join(output_dir_base, 'preview_animals'), image_base_dir=image_base_folder, options=viz_options) os.startfile(html_output_file)