def main(): parser = OptionParser() # This option points to the root directory of the image # dataset. Under the root directory should be category # directories, one per category, with images inside of them. parser.add_option( "--dataset_path", dest="dataset_path", default="", help="A path to the root directory of the image dataset." ) parser.add_option( "--process_limit", type="int", dest="process_limit", default=1, help="The number of processors to use during extraction.", ) # These sift_ parameters are passed onto the sift extractor. parser.add_option("--sift_normalization_threshold", type="float", dest="sift_normalization_threshold", default=2.0) parser.add_option( "--sift_discard_unnormalized", action="store_true", dest="sift_discard_unnormalized", default=True ) parser.add_option("--sift_multiscale", action="store_true", dest="sift_multiscale", default=False) parser.add_option("--sift_minimum_radius", type="int", dest="sift_minimum_radius", default=0) parser.add_option("--sift_grid_type", type="string", dest="sift_grid_type", default="FIXED_3X3") parser.add_option("--sift_first_level_smoothing", type="float", dest="sift_first_level_smoothing", default=0.66) parser.add_option("--sift_nosmooth", action="store_false", dest="sift_smoothed", default=True) parser.add_option("--sift_fast", action="store_true", dest="sift_fast", default=True) parser.add_option("--sift_slow", action="store_false", dest="sift_fast") # TODO(WenDi): This should be a positional argument, as it is mandatory. parser.add_option("--features_directory", action="store", type="string", dest="features_directory") (options, args) = parser.parse_args() print options # Move the sift_ command line arguments into an ExtactionParameters protobuf extraction_parameters = sift_descriptors_pb2.ExtractionParameters() extraction_parameters.normalization_threshold = options.sift_normalization_threshold extraction_parameters.discard_unnormalized = options.sift_discard_unnormalized extraction_parameters.multiscale = options.sift_multiscale extraction_parameters.minimum_radius = options.sift_minimum_radius extraction_parameters.first_level_smoothing = options.sift_first_level_smoothing extraction_parameters.smoothed = options.sift_smoothed extraction_parameters.fractional_xy = True extraction_parameters.fast = options.sift_fast if options.sift_grid_type == "FIXED_3X3": extraction_parameters.grid_method = sift_descriptors_pb2.ExtractionParameters.FIXED_3X3 elif options.sift_grid_type == "SCALED_3X3": extraction_parameters.grid_method = sift_descriptors_pb2.ExtractionParameters.SCALED_3X3 elif options.sift_grid_type == "SCALED_BIN_WIDTH": extraction_parameters.grid_method = sift_descriptors_pb2.ExtractionParameters.SCALED_BIN_WIDTH elif options.sift_grid_type == "SCALED_DOUBLE_BIN_WIDTH": extraction_parameters.grid_method = sift_descriptors_pb2.ExtractionParameters.SCALED_DOUBLE_BIN_WIDTH else: sys.stderr.write("Invalid --sift_grid_type.\n") sys.exit(1) # Get lists of extractions to do. caltech_data = options.dataset_path extraction_list = caltech_util.build_extraction_list(caltech_data, options.features_directory) caltech_util.do_extraction_on_list(extraction_list, extraction_parameters, options.process_limit)
def main(): parser = OptionParser() # This option points to the root directory of the image # dataset. Under the root directory should be category # directories, one per category, with images inside of them. parser.add_option('--dataset_path', dest='dataset_path', default='', help='A path to the root directory of the image dataset.') parser.add_option('-n', '--num_categories', type='int', dest='num_categories', default=None, help=('The number of categories to test on. ' 'If unspecified, use all of them.')) parser.add_option('--work_directory', type='string', dest='work_directory', default='', help='The directory to put all intermediate files.') parser.add_option('-o', '--results-file', type='string', dest='output', default='results.txt', help='The file to store the accuracy results in') parser.add_option('--kernel', type='string', dest='kernel', default='', help='The kernel to use for the spatial pyramids.') parser.add_option('--caltech_256', action='store_true', dest='caltech_256', default=False) parser.add_option('--scenes', action='store_true', dest='scenes', default=False) parser.add_option('--c', type='float', dest='c', default=0) parser.add_option('--clobber', action='store_true', dest='clobber', default=False) parser.add_option('--noclobber', action='store_false', dest='clobber', default=False) # These sift_ parameters are passed onto the sift extractor. parser.add_option('--sift_normalization_threshold', type='float', dest='sift_normalization_threshold', default=0.5) parser.add_option('--sift_discard_unnormalized', action='store_true', dest='sift_discard_unnormalized', default=False) parser.add_option('--sift_multiscale', action='store_true', dest='sift_multiscale', default=False) parser.add_option('--sift_minimum_radius', type='int', dest='sift_minimum_radius', default=0) parser.add_option('--sift_grid_type', type='string', dest='sift_grid_type', default='FIXED_3X3') parser.add_option('--sift_first_level_smoothing', type='float', dest='sift_first_level_smoothing', default=0.5) parser.add_option('--features_directory', action='store', type='string', dest='features_directory') parser.add_option('--process_limit', action='store', type='int', default=1, dest='process_limit') parser.add_option('--num_train', action='store', type='int', dest='num_train') parser.add_option('--num_test', action='store', type='int', dest='num_test') parser.add_option('--codeword_locality', action='store', type='int', dest='codeword_locality') parser.add_option('--pooling', action='store', type='string', dest='pooling') parser.add_option('--dictionary_training_size', action='store', type='int', default=1000000, dest='dictionary_training_size') parser.add_option('--clobber_dictionary', action='store_true', default=False, dest='clobber_dictionary') parser.add_option('--pyramid_levels', type='int', default=3, dest='pyramid_levels') parser.add_option('--single_level', type='int', default=-1, dest='single_level') parser.add_option('--dictionary', action="append", type='string', default=[], dest='dictionary') parser.add_option('--kmeans_accuracy', action='store', type='float', dest='kmeans_accuracy', default=1) (options, args) = parser.parse_args() print options if (len(options.dictionary) == 0): sys.stderr.write('Dictionary argument not provided.\n') sys.exit(1) # Move the sift_ command line arguments into an ExtactionParameters protobuf extraction_parameters = sift_descriptors_pb2.ExtractionParameters() extraction_parameters.normalization_threshold = \ options.sift_normalization_threshold extraction_parameters.discard_unnormalized = \ options.sift_discard_unnormalized extraction_parameters.multiscale = options.sift_multiscale extraction_parameters.minimum_radius = options.sift_minimum_radius extraction_parameters.first_level_smoothing = \ options.sift_first_level_smoothing extraction_parameters.fractional_xy = True extraction_parameters.fast = True if (options.sift_grid_type == 'FIXED_3X3'): extraction_parameters.grid_method = \ sift_descriptors_pb2.ExtractionParameters.FIXED_3X3 elif (options.sift_grid_type == 'FIXED_8X8'): extraction_parameters.grid_method = \ sift_descriptors_pb2.ExtractionParameters.FIXED_8X8 elif (options.sift_grid_type == 'SCALED_3X3'): extraction_parameters.grid_method = \ sift_descriptors_pb2.ExtractionParameters.SCALED_3X3 elif (options.sift_grid_type == 'SCALED_BIN_WIDTH'): extraction_parameters.grid_method = \ sift_descriptors_pb2.ExtractionParameters.SCALED_BIN_WIDTH elif (options.sift_grid_type == 'SCALED_DOUBLE_BIN_WIDTH'): extraction_parameters.grid_method = \ sift_descriptors_pb2.ExtractionParameters.SCALED_DOUBLE_BIN_WIDTH else: sys.stderr.write('Invalid --sift_grid_type.\n') sys.exit(1) extraction_list = caltech_util.build_extraction_list( options.dataset_path, options.features_directory) caltech_util.do_extraction_on_list(extraction_list, extraction_parameters, options.process_limit) if (not os.path.exists(options.work_directory)): os.mkdir(options.work_directory) training_lists = {} testing_lists = {} object_categories = list(set(os.listdir(options.features_directory)) - set(['BACKGROUND_Google']) - set(['257.clutter'])) if options.num_categories is None: options.num_categories = len(object_categories) dictionary_params = [] for d in options.dictionary: (dimensions, alpha) = d.split(':') logging.info('dictionary: %d dimensions, %0.2f alpha' % (int(dimensions), float(alpha))) dictionary_path = os.path.join(options.work_directory, 'caltech_%dd_%0.2fa.dictionary' % (int(dimensions), float(alpha))) dictionary_params.append((dictionary_path, int(dimensions), float(alpha))) logging.info('Splitting Caltech data into training and testing.') for category in object_categories[:options.num_categories]: file_list = glob.glob( os.path.join(os.path.join(options.features_directory, category), "*.sift")) try: capped_num_test = len(file_list) - options.num_train (training_list, testing_list) = \ caltech_util.split_into_train_test_random(file_list, options.num_train, capped_num_test) except caltech_util.CaltechUtilError as err: print err sys.exit(0) training_lists[category] = training_list testing_lists[category] = testing_list descriptor_training_list = os.path.join(options.work_directory, 'descriptor_training_list.txt') # Make the codeword_training_file if it doesn't exist. if (options.clobber or not os.path.exists(descriptor_training_list)): logging.info('Writing list of .sift training files.') codeword_training_file = open(descriptor_training_list, 'w') for category, training_list in training_lists.iteritems(): for t in training_list: codeword_training_file.write('%s\n' % t) codeword_training_file.close() else: logging.info('Training file already exists, skipping.') training_file_list_reread = [] for line in open(descriptor_training_list, 'r').readlines(): training_file_list_reread.append(line.strip()) # Make the descriptor testing file, if it doesn't exist. descriptor_testing_list = os.path.join(options.work_directory, 'descriptor_testing_list.txt') if (options.clobber or not os.path.exists(descriptor_testing_list)): logging.info('Writing list of .sift testing files.') descriptor_testing_file = open(descriptor_testing_list, 'w') for category, testing_list in testing_lists.iteritems(): num_test_found = 0 for t in testing_list: if t not in training_file_list_reread: descriptor_testing_file.write('%s\n' % t) num_test_found += 1 if num_test_found == options.num_test: break descriptor_testing_file.close() else: logging.info('Testing file already exists, skipping.') # Create the first checkpoint. open(os.path.join(options.work_directory, 'checkpoint_a.txt'), 'w').close() # List for holding subprocess objects codebook_builders = [] for dict_id, (dictionary_path, dimensions, alpha) in enumerate(dictionary_params): if (options.clobber_dictionary or not os.path.exists(dictionary_path)): logging.info('Creating dictionary.') codebook_cli_path = 'codebook_cli' command = ('%s --input list:%s --output %s --clusters %d ' '--max_descriptors %d --location_weighting %f ' '--initialization SUBSAMPLED_KMEANSPP --accuracy %f ' '--initialization_checkpoint_file %s ' '--logtostderr' % (codebook_cli_path, descriptor_training_list, dictionary_path, dimensions, options.dictionary_training_size, alpha, options.kmeans_accuracy, os.path.join(options.work_directory, 'checkpoint_b_%d.txt' % dict_id) )) # TODO(sanchom): Limit the number of simultaneous # subprocesses to options.process_limit. # # Create a popen object for this subprocess p = subprocess.Popen(command, shell=True) codebook_builders.append(p) else: logging.info('Dictionary already exists at %s, skipping. ' 'Use --clobber_dictionary to force a rebuild.' % dictionary_path) # Wait for all codebook subprocesses to finish, and checkpoint when they're done finished_processes = [] while len(finished_processes) != len(codebook_builders): for dict_id, p in enumerate(codebook_builders): # If the process is newly finished, if (not dict_id in finished_processes) and (p.poll() is not None): # Checkpoint it. open(os.path.join(options.work_directory, 'checkpoint_c_%d.txt' % dict_id), 'w').close() # Mark this completion as known. finished_processes.append(dict_id) break time.sleep(0.1) # Convert the training files to pyramid representations. logging.info('Converting training files from %s to pyramid representations.' % descriptor_training_list) pyramid_cli_path = 'spatial_pyramid_cli' command = ( '%s --codebooks %s --input list:%s --levels %d --single_level %d --k %d --pooling %s --thread_limit %d --logtostderr' % (pyramid_cli_path, ",".join([a for a, b, c in dictionary_params]), descriptor_training_list, options.pyramid_levels, options.single_level, options.codeword_locality, options.pooling, options.process_limit)) p = subprocess.Popen(command, shell=True) p.wait() # Checkpoint end of training file conversion. open(os.path.join(options.work_directory, 'checkpoint_d.txt'), 'w').close() # Convert the testing files to pyramid representations. logging.info('Converting testing files from %s to pyramid representations.' % descriptor_testing_list) command = ( '%s --codebooks %s --input list:%s --levels %d --single_level %d --k %d --pooling %s --thread_limit %d --logtostderr' % (pyramid_cli_path, ",".join([a for a, b, c in dictionary_params]), descriptor_testing_list, options.pyramid_levels, options.single_level, options.codeword_locality, options.pooling, options.process_limit)) p = subprocess.Popen(command, shell=True) p.wait() # Checkpoint end of testing file conversion. open(os.path.join(options.work_directory, 'checkpoint_e.txt'), 'w').close() # Creating the pyramid lists for training and testing the svm models. pyramid_training_list = os.path.join(options.work_directory, 'pyramid_training_list.txt') f = open(pyramid_training_list, 'w') for descriptor_file in open(descriptor_training_list).readlines(): descriptor_file = descriptor_file.strip() pyramid_file = descriptor_file.replace('.sift', '.pyramid') # Determine the category category = 'None' for c in object_categories[:options.num_categories]: if os.path.splitext(pyramid_file)[0].find(c) != -1: category = c f.write('%s:%s\n' % (pyramid_file, category)) f.close() pyramid_testing_list = os.path.join(options.work_directory, 'pyramid_testing_list.txt') f = open(pyramid_testing_list, 'w') for descriptor_file in open(descriptor_testing_list).readlines(): descriptor_file = descriptor_file.strip() pyramid_file = descriptor_file.replace('.sift', '.pyramid') category = 'None' for c in object_categories[:options.num_categories]: if os.path.splitext(pyramid_file)[0].find(c) != -1: category = c f.write('%s:%s\n' % (pyramid_file, category)) f.close() # Train a model for each category. trainer_cli = 'trainer_cli' command = ( '%s --training_list %s --kernel %s --output_directory /tmp ' '--thread_limit %s --c %f --gram_matrix_checkpoint_file %s ' '--cross_validation_checkpoint_file %s ' '--logtostderr' % (trainer_cli, pyramid_training_list, options.kernel, options.process_limit, options.c, os.path.join(options.work_directory, 'checkpoint_f.txt'), os.path.join(options.work_directory, 'checkpoint_g.txt') )) p = subprocess.Popen(command, shell=True) p.wait() # Checkpoint end of training. open(os.path.join(options.work_directory, 'checkpoint_h.txt'), 'w').close() # Make the model list. model_list_path = os.path.join(options.work_directory, 'model_list.txt') model_list_file = open(model_list_path, 'w') model_list = glob.glob('/tmp/*.svm') for m in model_list: category = os.path.splitext(os.path.basename(m))[0] model_list_file.write('%s:%s\n' % (m, category)) model_list_file.close() # Validate the models on the test data. validate_cli = 'validate_cli' command = \ '%s --training_list %s --model_list %s --testing_list %s --kernel %s --thread_limit %d --result_file %s --logtostderr' % \ (validate_cli, pyramid_training_list, model_list_path, pyramid_testing_list, options.kernel, options.process_limit, os.path.join(options.work_directory, options.output)) p = subprocess.Popen(command, shell=True) p.wait() # Checkpoint end of validation. open(os.path.join(options.work_directory, 'checkpoint_i.txt'), 'w').close()
def main(): parser = OptionParser() # This option points to the root directory of the image # dataset. Under the root directory should be category # directories, one per category, with images inside of them. parser.add_option( '--dataset_path', dest='dataset_path', default='', help='A path to the root directory of the image dataset.') parser.add_option('-n', '--num_categories', type='int', dest='num_categories', default=None, help=('The number of categories to test on. ' 'If unspecified, use all of them.')) parser.add_option('--work_directory', type='string', dest='work_directory', default='', help='The directory to put all intermediate files.') parser.add_option('-o', '--results-file', type='string', dest='output', default='results.txt', help='The file to store the accuracy results in') parser.add_option('--kernel', type='string', dest='kernel', default='', help='The kernel to use for the spatial pyramids.') parser.add_option('--caltech_256', action='store_true', dest='caltech_256', default=False) parser.add_option('--scenes', action='store_true', dest='scenes', default=False) parser.add_option('--c', type='float', dest='c', default=0) parser.add_option('--clobber', action='store_true', dest='clobber', default=False) parser.add_option('--noclobber', action='store_false', dest='clobber', default=False) # These sift_ parameters are passed onto the sift extractor. parser.add_option('--sift_normalization_threshold', type='float', dest='sift_normalization_threshold', default=0.5) parser.add_option('--sift_discard_unnormalized', action='store_true', dest='sift_discard_unnormalized', default=False) parser.add_option('--sift_multiscale', action='store_true', dest='sift_multiscale', default=False) parser.add_option('--sift_minimum_radius', type='int', dest='sift_minimum_radius', default=0) parser.add_option('--sift_grid_type', type='string', dest='sift_grid_type', default='FIXED_3X3') parser.add_option('--sift_first_level_smoothing', type='float', dest='sift_first_level_smoothing', default=0.5) parser.add_option('--features_directory', action='store', type='string', dest='features_directory') parser.add_option('--process_limit', action='store', type='int', default=1, dest='process_limit') parser.add_option('--num_train', action='store', type='int', dest='num_train') parser.add_option('--num_test', action='store', type='int', dest='num_test') parser.add_option('--codeword_locality', action='store', type='int', dest='codeword_locality') parser.add_option('--pooling', action='store', type='string', dest='pooling') parser.add_option('--dictionary_training_size', action='store', type='int', default=1000000, dest='dictionary_training_size') parser.add_option('--clobber_dictionary', action='store_true', default=False, dest='clobber_dictionary') parser.add_option('--pyramid_levels', type='int', default=3, dest='pyramid_levels') parser.add_option('--single_level', type='int', default=-1, dest='single_level') parser.add_option('--dictionary', action="append", type='string', default=[], dest='dictionary') parser.add_option('--kmeans_accuracy', action='store', type='float', dest='kmeans_accuracy', default=1) (options, args) = parser.parse_args() print options if (len(options.dictionary) == 0): sys.stderr.write('Dictionary argument not provided.\n') sys.exit(1) # Move the sift_ command line arguments into an ExtactionParameters protobuf extraction_parameters = sift_descriptors_pb2.ExtractionParameters() extraction_parameters.normalization_threshold = \ options.sift_normalization_threshold extraction_parameters.discard_unnormalized = \ options.sift_discard_unnormalized extraction_parameters.multiscale = options.sift_multiscale extraction_parameters.minimum_radius = options.sift_minimum_radius extraction_parameters.first_level_smoothing = \ options.sift_first_level_smoothing extraction_parameters.fractional_xy = True extraction_parameters.fast = True if (options.sift_grid_type == 'FIXED_3X3'): extraction_parameters.grid_method = \ sift_descriptors_pb2.ExtractionParameters.FIXED_3X3 elif (options.sift_grid_type == 'FIXED_8X8'): extraction_parameters.grid_method = \ sift_descriptors_pb2.ExtractionParameters.FIXED_8X8 elif (options.sift_grid_type == 'SCALED_3X3'): extraction_parameters.grid_method = \ sift_descriptors_pb2.ExtractionParameters.SCALED_3X3 elif (options.sift_grid_type == 'SCALED_BIN_WIDTH'): extraction_parameters.grid_method = \ sift_descriptors_pb2.ExtractionParameters.SCALED_BIN_WIDTH elif (options.sift_grid_type == 'SCALED_DOUBLE_BIN_WIDTH'): extraction_parameters.grid_method = \ sift_descriptors_pb2.ExtractionParameters.SCALED_DOUBLE_BIN_WIDTH else: sys.stderr.write('Invalid --sift_grid_type.\n') sys.exit(1) extraction_list = caltech_util.build_extraction_list( options.dataset_path, options.features_directory) caltech_util.do_extraction_on_list(extraction_list, extraction_parameters, options.process_limit) if (not os.path.exists(options.work_directory)): os.mkdir(options.work_directory) training_lists = {} testing_lists = {} object_categories = list( set(os.listdir(options.features_directory)) - set(['BACKGROUND_Google']) - set(['257.clutter'])) if options.num_categories is None: options.num_categories = len(object_categories) dictionary_params = [] for d in options.dictionary: (dimensions, alpha) = d.split(':') logging.info('dictionary: %d dimensions, %0.2f alpha' % (int(dimensions), float(alpha))) dictionary_path = os.path.join( options.work_directory, 'caltech_%dd_%0.2fa.dictionary' % (int(dimensions), float(alpha))) dictionary_params.append( (dictionary_path, int(dimensions), float(alpha))) logging.info('Splitting Caltech data into training and testing.') for category in object_categories[:options.num_categories]: file_list = glob.glob( os.path.join(os.path.join(options.features_directory, category), "*.sift")) try: capped_num_test = len(file_list) - options.num_train (training_list, testing_list) = \ caltech_util.split_into_train_test_random(file_list, options.num_train, capped_num_test) except caltech_util.CaltechUtilError as err: print err sys.exit(0) training_lists[category] = training_list testing_lists[category] = testing_list descriptor_training_list = os.path.join(options.work_directory, 'descriptor_training_list.txt') # Make the codeword_training_file if it doesn't exist. if (options.clobber or not os.path.exists(descriptor_training_list)): logging.info('Writing list of .sift training files.') codeword_training_file = open(descriptor_training_list, 'w') for category, training_list in training_lists.iteritems(): for t in training_list: codeword_training_file.write('%s\n' % t) codeword_training_file.close() else: logging.info('Training file already exists, skipping.') training_file_list_reread = [] for line in open(descriptor_training_list, 'r').readlines(): training_file_list_reread.append(line.strip()) # Make the descriptor testing file, if it doesn't exist. descriptor_testing_list = os.path.join(options.work_directory, 'descriptor_testing_list.txt') if (options.clobber or not os.path.exists(descriptor_testing_list)): logging.info('Writing list of .sift testing files.') descriptor_testing_file = open(descriptor_testing_list, 'w') for category, testing_list in testing_lists.iteritems(): num_test_found = 0 for t in testing_list: if t not in training_file_list_reread: descriptor_testing_file.write('%s\n' % t) num_test_found += 1 if num_test_found == options.num_test: break descriptor_testing_file.close() else: logging.info('Testing file already exists, skipping.') # Create the first checkpoint. open(os.path.join(options.work_directory, 'checkpoint_a.txt'), 'w').close() # List for holding subprocess objects codebook_builders = [] for dict_id, (dictionary_path, dimensions, alpha) in enumerate(dictionary_params): if (options.clobber_dictionary or not os.path.exists(dictionary_path)): logging.info('Creating dictionary.') codebook_cli_path = 'codebook_cli' command = ( '%s --input list:%s --output %s --clusters %d ' '--max_descriptors %d --location_weighting %f ' '--initialization SUBSAMPLED_KMEANSPP --accuracy %f ' '--initialization_checkpoint_file %s ' '--logtostderr' % (codebook_cli_path, descriptor_training_list, dictionary_path, dimensions, options.dictionary_training_size, alpha, options.kmeans_accuracy, os.path.join(options.work_directory, 'checkpoint_b_%d.txt' % dict_id))) # TODO(sanchom): Limit the number of simultaneous # subprocesses to options.process_limit. # # Create a popen object for this subprocess p = subprocess.Popen(command, shell=True) codebook_builders.append(p) else: logging.info('Dictionary already exists at %s, skipping. ' 'Use --clobber_dictionary to force a rebuild.' % dictionary_path) # Wait for all codebook subprocesses to finish, and checkpoint when they're done finished_processes = [] while len(finished_processes) != len(codebook_builders): for dict_id, p in enumerate(codebook_builders): # If the process is newly finished, if (not dict_id in finished_processes) and (p.poll() is not None): # Checkpoint it. open( os.path.join(options.work_directory, 'checkpoint_c_%d.txt' % dict_id), 'w').close() # Mark this completion as known. finished_processes.append(dict_id) break time.sleep(0.1) # Convert the training files to pyramid representations. logging.info( 'Converting training files from %s to pyramid representations.' % descriptor_training_list) pyramid_cli_path = 'spatial_pyramid_cli' command = ( '%s --codebooks %s --input list:%s --levels %d --single_level %d --k %d --pooling %s --thread_limit %d --logtostderr' % (pyramid_cli_path, ",".join([a for a, b, c in dictionary_params ]), descriptor_training_list, options.pyramid_levels, options.single_level, options.codeword_locality, options.pooling, options.process_limit)) p = subprocess.Popen(command, shell=True) p.wait() # Checkpoint end of training file conversion. open(os.path.join(options.work_directory, 'checkpoint_d.txt'), 'w').close() # Convert the testing files to pyramid representations. logging.info( 'Converting testing files from %s to pyramid representations.' % descriptor_testing_list) command = ( '%s --codebooks %s --input list:%s --levels %d --single_level %d --k %d --pooling %s --thread_limit %d --logtostderr' % (pyramid_cli_path, ",".join([a for a, b, c in dictionary_params]), descriptor_testing_list, options.pyramid_levels, options.single_level, options.codeword_locality, options.pooling, options.process_limit)) p = subprocess.Popen(command, shell=True) p.wait() # Checkpoint end of testing file conversion. open(os.path.join(options.work_directory, 'checkpoint_e.txt'), 'w').close() # Creating the pyramid lists for training and testing the svm models. pyramid_training_list = os.path.join(options.work_directory, 'pyramid_training_list.txt') f = open(pyramid_training_list, 'w') for descriptor_file in open(descriptor_training_list).readlines(): descriptor_file = descriptor_file.strip() pyramid_file = descriptor_file.replace('.sift', '.pyramid') # Determine the category category = 'None' for c in object_categories[:options.num_categories]: if os.path.splitext(pyramid_file)[0].find(c) != -1: category = c f.write('%s:%s\n' % (pyramid_file, category)) f.close() pyramid_testing_list = os.path.join(options.work_directory, 'pyramid_testing_list.txt') f = open(pyramid_testing_list, 'w') for descriptor_file in open(descriptor_testing_list).readlines(): descriptor_file = descriptor_file.strip() pyramid_file = descriptor_file.replace('.sift', '.pyramid') category = 'None' for c in object_categories[:options.num_categories]: if os.path.splitext(pyramid_file)[0].find(c) != -1: category = c f.write('%s:%s\n' % (pyramid_file, category)) f.close() # Train a model for each category. trainer_cli = 'trainer_cli' command = ('%s --training_list %s --kernel %s --output_directory /tmp ' '--thread_limit %s --c %f --gram_matrix_checkpoint_file %s ' '--cross_validation_checkpoint_file %s ' '--logtostderr' % (trainer_cli, pyramid_training_list, options.kernel, options.process_limit, options.c, os.path.join(options.work_directory, 'checkpoint_f.txt'), os.path.join(options.work_directory, 'checkpoint_g.txt'))) p = subprocess.Popen(command, shell=True) p.wait() # Checkpoint end of training. open(os.path.join(options.work_directory, 'checkpoint_h.txt'), 'w').close() # Make the model list. model_list_path = os.path.join(options.work_directory, 'model_list.txt') model_list_file = open(model_list_path, 'w') model_list = glob.glob('/tmp/*.svm') for m in model_list: category = os.path.splitext(os.path.basename(m))[0] model_list_file.write('%s:%s\n' % (m, category)) model_list_file.close() # Validate the models on the test data. validate_cli = 'validate_cli' command = \ '%s --training_list %s --model_list %s --testing_list %s --kernel %s --thread_limit %d --result_file %s --logtostderr' % \ (validate_cli, pyramid_training_list, model_list_path, pyramid_testing_list, options.kernel, options.process_limit, os.path.join(options.work_directory, options.output)) p = subprocess.Popen(command, shell=True) p.wait() # Checkpoint end of validation. open(os.path.join(options.work_directory, 'checkpoint_i.txt'), 'w').close()
def main(): parser = OptionParser() # This option points to the root directory of the image # dataset. Under the root directory should be category # directories, one per category, with images inside of them. parser.add_option( '--dataset_path', dest='dataset_path', default='', help='A path to the root directory of the image dataset.') parser.add_option( '--process_limit', type='int', dest='process_limit', default=1, help='The number of processors to use during extraction.') # These sift_ parameters are passed onto the sift extractor. parser.add_option('--sift_normalization_threshold', type='float', dest='sift_normalization_threshold', default=2.0) parser.add_option('--sift_discard_unnormalized', action='store_true', dest='sift_discard_unnormalized', default=True) parser.add_option('--sift_multiscale', action='store_true', dest='sift_multiscale', default=False) parser.add_option('--sift_minimum_radius', type='int', dest='sift_minimum_radius', default=0) parser.add_option('--sift_grid_type', type='string', dest='sift_grid_type', default='FIXED_3X3') parser.add_option('--sift_first_level_smoothing', type='float', dest='sift_first_level_smoothing', default=0.66) parser.add_option('--sift_nosmooth', action='store_false', dest='sift_smoothed', default=True) parser.add_option('--sift_fast', action='store_true', dest='sift_fast', default=True) parser.add_option('--sift_slow', action='store_false', dest='sift_fast') # TODO(WenDi): This should be a positional argument, as it is mandatory. parser.add_option('--features_directory', action='store', type='string', dest='features_directory') (options, args) = parser.parse_args() print options # Move the sift_ command line arguments into an ExtactionParameters protobuf extraction_parameters = sift_descriptors_pb2.ExtractionParameters() extraction_parameters.normalization_threshold = \ options.sift_normalization_threshold extraction_parameters.discard_unnormalized = \ options.sift_discard_unnormalized extraction_parameters.multiscale = options.sift_multiscale extraction_parameters.minimum_radius = options.sift_minimum_radius extraction_parameters.first_level_smoothing = \ options.sift_first_level_smoothing extraction_parameters.smoothed = \ options.sift_smoothed extraction_parameters.fractional_xy = True extraction_parameters.fast = options.sift_fast if (options.sift_grid_type == 'FIXED_3X3'): extraction_parameters.grid_method = \ sift_descriptors_pb2.ExtractionParameters.FIXED_3X3 elif (options.sift_grid_type == 'SCALED_3X3'): extraction_parameters.grid_method = \ sift_descriptors_pb2.ExtractionParameters.SCALED_3X3 elif (options.sift_grid_type == 'SCALED_BIN_WIDTH'): extraction_parameters.grid_method = \ sift_descriptors_pb2.ExtractionParameters.SCALED_BIN_WIDTH elif (options.sift_grid_type == 'SCALED_DOUBLE_BIN_WIDTH'): extraction_parameters.grid_method = \ sift_descriptors_pb2.ExtractionParameters.SCALED_DOUBLE_BIN_WIDTH else: sys.stderr.write('Invalid --sift_grid_type.\n') sys.exit(1) # Get lists of extractions to do. caltech_data = options.dataset_path extraction_list = caltech_util.build_extraction_list( caltech_data, options.features_directory) caltech_util.do_extraction_on_list(extraction_list, extraction_parameters, options.process_limit)