def extract_features_inner_core(image_certificate, model_certificate, feature_hash, image_hash,
model_hash, convolve_func_name,device_id, im_query, m_query, im_skip,
im_limit, m_skip, m_limit):
if im_query is None:
im_query = {}
if m_query is None:
m_query = {}
im_query['__hash__'] = image_hash
m_query['__hash__'] = model_hash
conn = pm.Connection(document_class = SON)
db = conn[DB_NAME]
image_col = db['images.files']
model_col = db['models.files']
image_fs = gridfs.GridFS(db,'images')
model_fs = gridfs.GridFS(db,'models')
feature_fs = gridfs.GridFS(db,'features')
if convolve_func_name == 'pyfft':
context = v1_pyfft.setup_pyfft(device_id)
context.push()
convolve_func = functools.partial(v1f.v1like_filter_pyfft,device_id=device_id)
else:
convolve_func = v1f.v1like_filter_numpy
L1 = get_most_recent_files(image_col,im_query,skip=im_skip,limit=im_limit)
L2 = get_most_recent_files(model_col,m_query,skip=m_skip,limit=m_limit)
for image_config in L1:
for model_config in L2:
features = compute_features(image_config['filename'], image_fs, model_config, model_fs,convolve_func)
features_string = cPickle.dumps(features)
y = SON([('config',SON([('model',model_config['config']['model']),('image',image_config['config']['image'])]))])
filename = get_filename(y['config'])
y['filename'] = filename
y['__hash__'] = feature_hash
feature_fs.put(features_string,**y)
if convolve_func_name == 'pyfft':
context.pop()
def extract_and_evaluate_inner_core(images,m,convolve_func_name,device_id,task,cache_port):
if cache_port:
ctx = zmq.Context()
sock = ctx.socket(zmq.REQ)
sock.connect('tcp://127.0.0.1:' + str(cache_port))
sock.send_pyobj({'alive':True})
poller = zmq.Poller()
poller.register(sock)
poll = poller.poll(timeout=NETWORK_CACHE_TIMEOUT)
if poll != []:
sock.recv_pyobj()
else:
poller = None
else:
poller = None
if convolve_func_name == 'pyfft':
context = v1_pyfft.setup_pyfft(device_id)
context.push()
convolve_func = functools.partial(v1f.v1like_filter_pyfft,device_id=device_id)
else:
convolve_func = v1f.v1like_filter_numpy
conn = pm.Connection(document_class=bson.SON)
db = conn[DB_NAME]
perf_coll = db['performance.files']
model_fs = gridfs.GridFS(db,'models')
image_fs = gridfs.GridFS(db,'images')
L = [get_features(im, image_fs, m, model_fs, convolve_func,task,poller) for im in images]
if convolve_func_name == 'pyfft':
context.pop()
return L
def greedy_optimization(outfile,task,image_certificate_file,initial_model,convolve_func,rep_limit, modifier_args,modifier):
conn = pm.Connection(document_class=bson.SON)
db = conn['v1']
opt_fs = gridfs.GridFS(db,'optimized_performance')
image_coll = db['raw_images.files']
image_fs = gridfs.GridFS(db,'raw_images')
image_certdict = cPickle.load(open(image_certificate_file))
print('using image certificate', image_certificate_file)
image_hash = image_certdict['run_hash']
image_args = image_certdict['out_args']
if convolve_func == v1f.v1like_filter_pyfft:
v1_pyfft.setup_pyfft()
filterbanks = []
perfs = []
model_configs = []
center_config = initial_model
i = 0
improving = True
while ((i < rep_limit) or rep_limit is None):
i += 1
print('Round', i)
next_configs = [m for m in get_consistent_deltas(center_config,modifier) if m not in model_configs]
if next_configs:
next_results = [get_performance(task,image_hash,image_fs,m,convolve_func) for m in next_configs]
next_perfs = [x[0] for x in next_results]
next_filterbanks = [x[1] for x in next_results]
next_perf_ac_max = np.array([x['test_accuracy'] for x in next_perfs]).max()
perf_ac_max = max([x['test_accuracy'] for x in perfs]) if perfs else 0
if next_perf_ac_max > perf_ac_max:
next_perf_ac_argmax = np.array([x['test_accuracy'] for x in next_perfs]).argmax()
center_config = next_configs[next_perf_ac_argmax]
print('\n\n')
print('new best performance is', next_perf_ac_max, 'from model', center_config)
print('\n\n')
perfs.extend(next_perfs)
model_configs.extend(next_configs)
filterbanks.extend(next_filterbanks)
else:
print('Breaking because no further optimization could be done. Best existing performance was', perf_ac_max, 'while best next performance was', next_perf_ac_max)
break
else:
print('Breaking because no next configs')
break
perfargmax = np.array([p['test_accuracy'] for p in perfs]).argmax()
best_model = model_configs[perfargmax]
best_performance = perfs[perfargmax]
out_record = SON([('initial_model',initial_model),
('task',son_escape(task)),
('images',son_escape(image_args)),
('images_hash',image_hash),
('modifier_args',son_escape(modifier_args)),
('modifier',modifier.__class__.__module__ + '.' + modifier.__class__.__name__)
])
filename = get_filename(out_record)
out_record['filename'] = filename
out_record.update(SON([('performances',perfs)]))
out_record.update(SON([('best_model',best_model)]))
out_record.update(SON([('best_performance',best_performance)]))
out_record.update(SON([('num_steps',len(model_configs))]))
out_record.update(SON([('models',model_configs)]))
outdata = cPickle.dumps(filterbanks)
opt_fs.put(outdata,**out_record)
if convolve_func == v1f.v1like_filter_pyfft:
v1_pyfft.cleanup_pyfft()
createCertificateDict(outfile,{'image_file':image_certificate_file})
