def main():
dataset_name = 'tide_v12'
root_uri = 'maprfs://data/lpbench/'
configs = GenerateConfigurations(root_uri, dataset_name)
db = resultdb.ResultDatabase()
for config in configs:
print config.iter_graph.tuned_feature_extractor_params.params
if db.ResultReady(config):
LOG(INFO, 'result for config is complete...')
continue
else:
LOG(INFO, 'generating results for config: %s' % (config))
flow = flows.MainFlow(root_uri, config)
CHECK(scheduler.FlowScheduler(flow).RunSequential())
matches = flow.matches
matches_fp = matches.GetFingerprint()
matches_provenance_list = provenance.BuildResourceProvenanceList(
matches_fp)
eval1 = py_eval1.LoadResultOrDie(flow.eval1.GetUri())
eval2 = py_eval2.LoadResultOrDie(flow.eval2.GetUri())
result = lpbench_pb2.ConfigurationResult()
result.config.CopyFrom(config)
result.eval1.CopyFrom(eval1)
result.eval2.CopyFrom(eval2)
result.matches_provenance_list.extend(matches_provenance_list)
CHECK(result.IsInitialized())
db.SetResult(config, result)
return 0
def main(): dataset_name = 'tide_v18_mixed' base_uri = 'maprfs://data/itergraph/' params = GetParamsForDataset(base_uri, dataset_name) root_uri = '%s/%s/' % (base_uri, dataset_name) fs = scheduler.FlowScheduler(itergraph.MainFlow(root_uri, params)) #fs.RunParallel() fs.RunSequential() return 0
def DefaultParamsTideV12(root_uri, dataset_name):
feature_type = 'usift'
desired_features_per_megapixel = 3000
root_sift_normalization = True
tune_flow = util.TuneFeatureExtractorDensityFlow(root_uri, dataset_name, feature_type, desired_features_per_megapixel, root_sift_normalization)
CHECK(scheduler.FlowScheduler(tune_flow).RunSequential())
tuned_feature_extractor_params = iw_pb2.DensityTunedFeatureExtractorParams()
iwutil.LoadProtoFromUriOrDie(tuned_feature_extractor_params, tune_flow.GetOutput().GetUri())
CHECK(tuned_feature_extractor_params.IsInitialized())
params = itergraph_pb2.IterGraphParams()
params.tuned_feature_extractor_params.CopyFrom(tuned_feature_extractor_params)
params.image_matcher_config.CopyFrom(image_matcher_config.GetImageMatcherConfig(feature_type))
params.cbir.full.num_index_shards = 2
params.cbir.full.num_neighbors_per_index_shard = 40
params.cbir.full.scorer.max_results = 100
params.cbir.full.scorer.scoring = full_pb2.QueryScorerParams.ADAPTIVE
params.cbir.full.scorer.normalization = full_pb2.QueryScorerParams.NRC
params.cbir.full.scorer.correspondence_filter = full_pb2.QueryScorerParams.NO_FILTER
params.cbir.full.scorer.bursty_candidates = True
params.cbir.full.flann.algorithm = full_pb2.FlannParams.FLANN_INDEX_KDTREE
params.cbir.full.flann.kdtree_params.trees = 4
params.cbir.full.flann.kdtree_params.checks = 32
# Create template for a "local best" phase
local_best_phase = itergraph_pb2.PhaseParams()
local_best_phase.max_match_candidates = 10000
local_best_phase.max_vertex_degree = 100
local_best_phase.max_match_batch_size = 100
local_best_phase.max_image_replication_factor = 100
local_best_phase.type = itergraph_pb2.PhaseParams.PHASE_TYPE_CBIR_RANK
for _ in range(2):
new_phase = params.phase.add()
new_phase.CopyFrom(local_best_phase)
# motif_phase = params.phase.add()
# motif_phase.max_match_candidates = 5*5000
# motif_phase.max_vertex_degree = 10
# motif_phase.max_match_batch_size = 100
# motif_phase.max_image_replication_factor = 100
# motif_phase.type = itergraph_pb2.PhaseParams.PHASE_TYPE_MOTIF_EXPANSION
# motif_phase.motif_expansion.background_features_uri = 'local://home/ubuntu/Desktop/vol-0449ca74/motif_background_v01/usift/features.pert'
CHECK(params.IsInitialized(), str(params))
return params
def GenerateConfigurations(root_uri, dataset_name):
configs = []
feature_types = [ #'ahess',
#'sift',
'usift'
]
cbir_params = []
full_cbir_params = GenerateFullCbirParams()
cbir_params.extend(full_cbir_params)
#bow_cbir_params = GenerateBowCbirParams()
#cbir_params.extend(bow_cbir_params)
use_motif_expansion_types = [False]
desired_features_per_megapixel_settings = [ #1500,
3000,
#6000
]
root_sift_normalization = True
for feature_type in feature_types:
for desired_features_per_megapixel in desired_features_per_megapixel_settings:
tune_flow = util.TuneFeatureExtractorDensityFlow(
root_uri, dataset_name, feature_type,
desired_features_per_megapixel)
CHECK(scheduler.FlowScheduler(tune_flow).RunSequential())
tuned_feature_extractor_params = iw_pb2.DensityTunedFeatureExtractorParams(
)
iwutil.LoadProtoFromUriOrDie(tuned_feature_extractor_params,
tune_flow.GetOutput().GetUri())
if tuned_feature_extractor_params.params.HasField(
'ocv_sift_params'):
tuned_feature_extractor_params.params.ocv_sift_params.root_sift_normalization = root_sift_normalization
elif tuned_feature_extractor_params.params.HasField(
'vgg_affine_sift_params'):
tuned_feature_extractor_params.params.vgg_affine_sift_params.root_sift_normalization = root_sift_normalization
else:
LOG(FATAL, 'unexpected')
CHECK(tuned_feature_extractor_params.IsInitialized())
for cbir_param in cbir_params:
if cbir_param.HasField('bow') and feature_type != 'ahess':
LOG(
FATAL,
'bow cbir currently required ahess feature type without rootsift because of inria vv'
)
if cbir_param.HasField('bow') and root_sift_normalization:
LOG(
FATAL,
'bow cbir currently required no rootsift because of inria vv'
)
for motif_expansion in use_motif_expansion_types:
c = lpbench_pb2.Configuration()
c.dataset_name = dataset_name
c.iter_graph.tuned_feature_extractor_params.CopyFrom(
tuned_feature_extractor_params)
c.iter_graph.image_matcher_config.CopyFrom(
image_matcher_config.GetImageMatcherConfig(
feature_type))
c.iter_graph.cbir.CopyFrom(cbir_param)
# Create template for a "local best" phase
phase = itergraph_pb2.PhaseParams()
phase.max_match_candidates = 10000
phase.max_vertex_degree = 1000
phase.max_match_batch_size = 100
phase.max_image_replication_factor = 100
phase.type = itergraph_pb2.PhaseParams.PHASE_TYPE_CBIR_RANK
for _ in range(4):
new_phase = c.iter_graph.phase.add()
new_phase.CopyFrom(phase)
if motif_expansion:
motif_phase = c.iter_graph.phase.add()
motif_phase.max_match_candidates = 50000
motif_phase.max_vertex_degree = phase.max_vertex_degree
motif_phase.max_match_batch_size = 100
motif_phase.max_image_replication_factor = 100
motif_phase.type = itergraph_pb2.PhaseParams.PHASE_TYPE_MOTIF_EXPANSION
motif_phase.motif_expansion.background_features_uri = 'local://home/ubuntu/Desktop/vol-0449ca74/motif_background_v01/usift/features.pert'
CHECK(c.IsInitialized())
configs.append(c)
return configs