def eval_feature(fp: GlowProvider, f: str):
images_0 = fp.get_np({'Blond_Hair': 0}, sample=0.1, sample_max=args['max_sampling'])
images_1 = fp.get_np({'Blond_Hair': 1}, sample=0.1, sample_max=args['max_sampling'])
y_0 = np.zeros((len(images_0), 1))
y_1 = np.ones((len(images_1), 1))
x = np.concatenate([[*images_0], [*images_1]])
y = np.append(y_0, y_1)
# Run Perceptron
cm = run_perceptron(x, y)
# show_matrix(cm, f + ': Confusion Matrix - Entire Data')
print(cm)
# Cosine
images_0 = np.average(images_0, axis=0)
images_1 = np.average(images_1, axis=0)
cos_sim = dot(images_0, images_1)/(norm(images_0)*norm(images_1))
print(cos_sim)
# Run SVM
# run_svc(x, y)
return {'feature_name': f, 'cm': cm.tolist(), 'cos': str(cos_sim)}
def eval_features(fp: GlowProvider, result: Path):
features = fp.get_features()
results = dict()
for f, i in zip(features, range(len(features))):
print(f + ': ' + str(i+1) + '/' + str(len(features)))
results[f] = eval_feature(fp, f)
if not result.parent.exists():
result.parent.mkdir()
with open(result, 'w') as o:
json.dump(results, o)
class AvConsumer(multiprocessing.Process):
def __init__(self, task_queue, result_queue, index_path, np_path):
multiprocessing.Process.__init__(self)
self.task_queue = task_queue
self.result_queue = result_queue
self.gp = GlowProvider(index=index_path, np_path=np_path)
def run(self):
proc_name = self.name
the_averages = {}
features = self.gp.get_features()
while True:
# {'img': imgs, 'num_job': num_jobs, 'total_num_job': gp_file_total}
next_task = self.task_queue.get()
if next_task is None:
# Poison pill means shutdown
print('%s: Exiting' % proc_name)
self.result_queue.put(the_averages)
self.task_queue.task_done()
break
print('{}: {} / {}'.format(proc_name, next_task['num_job'],
next_task['total_num_job']))
for np_name in next_task['img']:
np, feature_values = self.gp.get_np_with_features(np_name)
for f in features:
try:
the_averages[(f, feature_values[f])]['np'] += np.copy()
the_averages[(f, feature_values[f])]['count'] += 1
except:
the_averages[(f, feature_values[f])] = {}
the_averages[(f, feature_values[f])]['np'] = np.copy()
the_averages[(f, feature_values[f])]['count'] = 1
self.task_queue.task_done()
return
def __init__(self, task_queue, result_queue, index_path, np_path):
multiprocessing.Process.__init__(self)
self.task_queue = task_queue
self.result_queue = result_queue
self.gp = GlowProvider(index=index_path, np_path=np_path)
ap.add_argument('-m',
'--max_sampling',
required=False,
type=float,
default=20,
help='If sampling, indicate the percentage')
ap.add_argument('-c',
'--chunk_size',
required=False,
type=int,
default=100,
help='Chunk size')
args = vars(ap.parse_args())
# We load the GlowProvider
gp = GlowProvider(index=args['index'], np_path=args['np_files'])
gp_file_names_iterator = chunks(gp.get_image_names(), args['chunk_size'])
gp_file_total = gp.get_np_num() // args['chunk_size']
# Establish communication queues
tasks = multiprocessing.JoinableQueue()
results = multiprocessing.Queue()
# Start consumers
num_consumers = multiprocessing.cpu_count() * 2 - 2
print('Creating %d consumers' % num_consumers)
consumers = [
AvConsumer(tasks,
results,
index_path=args['index'],
'--sampling',
required=False,
type=float,
help='If sampling, indicate the percentage')
ap.add_argument('-m',
'--max_sampling',
required=False,
type=float,
default=20,
help='If sampling, indicate the percentage')
args = vars(ap.parse_args())
if __name__ == '__main__':
fp = GlowProvider(index=args['index'], np_path=args['np_files'])
if False:
eval_features(fp, args['result'])
if True:
result = args['result']
features = fp.get_features()
results = dict()
with Pool(5) as p:
results = p.map(eval_feature_f, features)
results = {
r['feature_name']: {
'cm': r['cm'],
'cos': r['cos']
}
def eval_feature_f(f):
fp = GlowProvider(index=args['index'], np_path=args['np_files'])
return eval_feature(fp, f)