def parse_model_definition_file_content(file_content):
'''
Parse protobuf file content
'''
add_caffe_to_path()
from caffe.proto import caffe_pb2
from google import protobuf
model_params = caffe_pb2.NetParameter()
protobuf.text_format.Merge(file_content, model_params)
return model_params
def parse_solver_file_content(file_content):
'''
Returns a protobuf object with the solver parameters
'''
add_caffe_to_path()
from caffe.proto import caffe_pb2
from google import protobuf
solver_params = caffe_pb2.SolverParameter()
protobuf.text_format.Merge(file_content, solver_params)
return solver_params
def load_net_from_snapshot(snapshot_id):
deployfile_relpath, weights_relpath = download_snapshot(
snapshot_id=snapshot_id,
transfer=False,
)
add_caffe_to_path()
import caffe
deployfile_path = os.path.join(settings.CAFFE_ROOT, deployfile_relpath)
weights_path = os.path.join(settings.CAFFE_ROOT, weights_relpath)
return caffe.Net(deployfile_path, weights_path, caffe.TEST)
def transfer_weights(temp_dir, deployfile_source_path, weights_source_path,
deployfile_target_path, weights_target_path,
verbose=True):
deployfile_source = parse_model_definition_file(deployfile_source_path)
# Modify deploy file
deployfile_target, param_mapping = convolutionize_net(deployfile_source)
#save_protobuf_file(deployfile_target_path, deployfile_target)
add_caffe_to_path()
import caffe
net_source = caffe.Net(
deployfile_source_path,
weights_source_path,
caffe.TEST
)
net_target = caffe.Net(
deployfile_target_path,
weights_source_path,
caffe.TEST
)
for t, s in param_mapping.iteritems():
if t not in net_target.params:
print 'WARNING: Couldn\'t find "%s" layer in the target model definition file, skipping...' % t
continue
for blob_idx in (0, 1):
if verbose:
print '%s %s %s <-- %s %s %s' % (
t, blob_idx, net_target.params[t][blob_idx].data.shape,
s, blob_idx, net_source.params[s][blob_idx].data.shape,
)
net_target.params[t][blob_idx].data[...] = (
np.reshape(
net_source.params[s][blob_idx].data,
net_target.params[t][blob_idx].data.shape
)
)
if verbose:
print_net_weight_stats(net_target)
# Download to a temp location, so there is not collision if multiple
# processes are trying to download the same file
f_temp = NamedTemporaryFile(dir=temp_dir, delete=False)
f_temp.close()
# Use the temp file's name
net_target.save(f_temp.name)
# Move it to the final destination in an atomic operation
os.rename(f_temp.name, weights_target_path)
def load_fet_extractor(
deployfile_relpath,
weights_relpath,
do_preprocessing=True,
image_dims=(256, 256),
mean=(104, 117, 123),
device_id=0,
input_scale=1,
):
# Silence Caffe
from os import environ
environ['GLOG_minloglevel'] = '2'
add_caffe_to_path()
import caffe
FeatureExtractor = def_FeatureExtractor(caffe)
if mean is not None:
mean = np.array(mean)
model_file = os.path.join(settings.CAFFE_ROOT, deployfile_relpath)
pretrained_file = os.path.join(settings.CAFFE_ROOT, weights_relpath)
if settings.CAFFE_GPU:
print 'Using GPU'
caffe.set_mode_gpu()
print 'Using device #{}'.format(device_id)
caffe.set_device(device_id)
else:
print 'Using CPU'
caffe.set_mode_cpu()
net = FeatureExtractor(
model_file=model_file,
pretrained_file=pretrained_file,
do_preprocessing=do_preprocessing,
image_dims=image_dims,
mean=mean,
input_scale=input_scale,
raw_scale=255,
channel_swap=(2, 1, 0),
)
return caffe, net
def load_fet_extractor(
deployfile_relpath,
weights_relpath,
do_preprocessing=True,
image_dims=(256, 256),
mean=(104, 117, 123),
device_id=0,
input_scale=1,
):
# Silence Caffe
from os import environ
environ['GLOG_minloglevel'] = '2'
add_caffe_to_path()
import caffe
FeatureExtractor = def_FeatureExtractor(caffe)
if mean is not None:
mean = np.array(mean)
model_file = os.path.join(settings.CAFFE_ROOT, deployfile_relpath)
pretrained_file = os.path.join(settings.CAFFE_ROOT, weights_relpath)
if settings.CAFFE_GPU:
print 'Using GPU'
caffe.set_mode_gpu()
print 'Using device #{}'.format(device_id)
caffe.set_device(device_id)
else:
print 'Using CPU'
caffe.set_mode_cpu()
net = FeatureExtractor(
model_file=model_file,
pretrained_file=pretrained_file,
do_preprocessing=do_preprocessing,
image_dims=image_dims,
mean=mean,
input_scale=input_scale,
raw_scale=255,
channel_swap=(2, 1, 0),
)
return caffe, net
def _load_caffe_net(
deployfile_relpath,
weights_relpath,
image_dims=(256, 256),
mean=(104, 117, 123),
device_id=0,
input_scale=1,
):
add_caffe_to_path()
import caffe
mean = np.array(mean)
model_file = os.path.join(settings.CAFFE_ROOT, deployfile_relpath)
pretrained_file = os.path.join(settings.CAFFE_ROOT, weights_relpath)
if settings.CAFFE_GPU:
print 'Using GPU'
caffe.set_mode_gpu()
print 'Using device #{}'.format(device_id)
caffe.set_device(device_id)
else:
print 'Using CPU'
caffe.set_mode_cpu()
net = caffe.Classifier(
model_file=model_file,
pretrained_file=pretrained_file,
mean=mean,
channel_swap=(2, 1, 0),
raw_scale=255,
input_scale=input_scale,
image_dims=image_dims,
)
return net, caffe
def _load_caffe_net(
deployfile_relpath,
weights_relpath,
image_dims=(256, 256),
mean=(104, 117, 123),
device_id=0,
input_scale=1,
):
add_caffe_to_path()
import caffe
mean = np.array(mean)
model_file = os.path.join(settings.CAFFE_ROOT, deployfile_relpath)
pretrained_file = os.path.join(settings.CAFFE_ROOT, weights_relpath)
if settings.CAFFE_GPU:
print 'Using GPU'
caffe.set_mode_gpu()
print 'Using device #{}'.format(device_id)
caffe.set_device(device_id)
else:
print 'Using CPU'
caffe.set_mode_cpu()
net = caffe.Classifier(
model_file=model_file,
pretrained_file=pretrained_file,
mean=mean,
channel_swap=(2, 1, 0),
raw_scale=255,
input_scale=input_scale,
image_dims=image_dims,
)
return net, caffe
def print_net_weight_stats_file(deployfile_path, weights_path):
add_caffe_to_path()
import caffe
net = caffe.Net(deployfile_path, weights_path, caffe.TEST)
print_net_weight_stats(net)
def handle(self, *args, **option):
if len(args) != 5:
print 'Incorrect number of arguments!'
print 'Usage: ./manage.py cnntools_untie_siamese [options] %s' % Command.args
return
trainingfile_source_relpath = args[0]
weights_source_relpath = args[1]
trainingfile_target_relpath = args[2]
weights_target_relpath = args[3]
suffix = args[4]
trainingfile_source_path = os.path.join(settings.CAFFE_ROOT, trainingfile_source_relpath)
trainingfile_target_path = os.path.join(settings.CAFFE_ROOT, trainingfile_target_relpath)
add_caffe_to_path()
add_bkovacs_to_path()
import caffe
# Change working directory to Caffe
os.chdir(settings.CAFFE_ROOT)
net_source = caffe.Net(
trainingfile_source_path,
os.path.join(settings.CAFFE_ROOT, weights_source_relpath),
caffe.TEST
)
net_target = caffe.Net(
trainingfile_target_path,
caffe.TEST
)
# Get all source names
names = net_target.params.keys()
print 'Computing param mapping...'
param_mapping = {}
for name in names:
if suffix in name:
orig_name = name.replace(suffix, '')
if orig_name not in names:
print 'Warning: the corresponding name to {} ({}) is not among the blob names, skipping weight transfer!'
else:
param_mapping[name] = orig_name
for i, (t, s) in enumerate(param_mapping.iteritems()):
if s not in net_source.params:
print 'Couldn\'nt find {} among the source net params, skipping...'.format(s)
continue
# Weights and biases
for blob_idx in (0, 1):
print '%s %s %s <-- %s %s %s' % (
t, blob_idx, net_target.params[t][blob_idx].data.shape,
s, blob_idx, net_source.params[s][blob_idx].data.shape,
)
net_target.params[s][blob_idx].data[...] = net_source.params[s][blob_idx].data
net_target.params[t][blob_idx].data[...] = net_source.params[s][blob_idx].data
net_target.save(
os.path.join(settings.CAFFE_ROOT, weights_target_relpath),
)
print 'Done.'
def setup_solverfile(model_name, model_file_content, solver_file_content,
options, caffe_cnn_trrun_id, device_id):
rand_name = '%d' % caffe_cnn_trrun_id
root_path = os.path.join(
settings.CAFFE_ROOT,
'training_runs',
'-'.join([rand_name, model_name])
)
ensuredir(root_path)
trainfilename = 'train_val.prototxt'
solverfilename = 'solver.prototxt'
trainfile_path = os.path.join(root_path, trainfilename)
solverfile_path = os.path.join(root_path, solverfilename)
# Save the model_file_content to a file, so Caffe can read it
with open(trainfile_path, 'w') as f:
f.write(model_file_content)
# copy the sample solver file and modify it
solver_params = caffefileproc.parse_solver_file_content(solver_file_content)
# modify solver params according to the command line parameters
solver_params.net = trainfile_path
if 'base_lr' in options and options['base_lr'] is not None:
solver_params.base_lr = options['base_lr']
# Switch on debug_info to facilitate debugging
if 'debug_info' in options and options['debug_info'] is not None:
solver_params.debug_info = options['debug_info']
if 'weight_decay' in options and options['weight_decay'] is not None:
solver_params.weight_decay = options['weight_decay']
if 'max_iter' in options and options['max_iter'] is not None:
solver_params.max_iter = options['max_iter']
snapshot_path = os.path.join(root_path, 'snapshots')
ensuredir(snapshot_path)
solver_params.snapshot_prefix = os.path.join(
snapshot_path,
'train_{}-base_lr{}'.format(model_name, solver_params.base_lr)
)
# compute the proper test_iter
batch_size, testset_size = extract_batchsize_testsetsize(model_file_content)
if batch_size and testset_size:
if options['verbose']:
print 'Extracted batch_size ({0}) and testset_size ({1})'.format(
batch_size, testset_size)
# Note the solver file should have exactly one test_iter
solver_params.test_iter[0] = int(testset_size/batch_size)
else:
if options['verbose']:
print 'WARNING: Couldn\'t find the batch_size or the source file ' + \
'containing the testset, please set the test_iter to ' + \
'testset_size / batch_size!'
# Setting random seed value for reproducible results
solver_params.random_seed = settings.CAFFE_SEED
# Silence Caffe
from os import environ
environ['GLOG_minloglevel'] = '2'
add_caffe_to_path()
import caffe
from caffe.proto import caffe_pb2
if settings.CAFFE_GPU and (options['cpu'] is None or not options['cpu']):
if options['verbose']:
print 'Using GPU'
caffe.set_mode_gpu()
caffe.set_device(device_id)
solver_params.solver_mode = caffe_pb2.SolverParameter.GPU
else:
if options['verbose']:
print 'Using CPU'
caffe.set_mode_cpu()
solver_params.solver_mode = caffe_pb2.SolverParameter.CPU
caffefileproc.save_protobuf_file(solverfile_path, solver_params)
return solver_params, solverfile_path
def train_network(solver_params, solverfile_path, options,
caffe_cnn_trrun_id):
add_caffe_to_path()
import caffe
for p in settings.TRAINING_EXTRA_PYTHON_PATH:
add_to_path(p)
restore = False
solver = get_solver_type(caffe, solver_params)(str(solverfile_path))
solver_nets = [solver.net] + list(solver.test_nets)
if 'weights' in options and options['weights'] is not None:
_, ext = os.path.splitext(options['weights'])
if ext == '.solverstate':
solver.restore(
os.path.join(settings.CAFFE_ROOT, options['weights'])
)
restore = True
else:
for n in solver_nets:
n.copy_from(
os.path.join(settings.CAFFE_ROOT, options['weights'])
)
print 'solver_net count:', len(solver_nets)
for n in solver_nets:
data_layer = n.layers[0]
# If the data layer is python, we try to set the random seed for
# reproducibility
if data_layer.type == 'Python':
# Note: The python implementation of the layer should have a
# "set_random_seed" function. If we can't find a function with this name,
# we won't set the random seed
set_random_seed_func = getattr(data_layer, 'set_random_seed', None)
if callable(set_random_seed_func):
set_random_seed_func(settings.CAFFE_SEED)
# Note: The python implementation of the layer should have a
# "set_params" function.
set_params_func = getattr(data_layer, 'set_params', None)
if 'data_layer_params' in options and callable(set_params_func):
set_params_func(options['data_layer_params'])
n.reshape()
# {key: output_num, value: output_name}
output_names = [{}, {}]
# for backward compatibility
name_to_num = [{}, {}]
# {key: output_num, value: {key: it_num value: output_value}}
outputs = [{}, {}]
for i, op in enumerate(solver.net.outputs):
output_names[0][i] = op
name_to_num[0][op] = i
outputs[0][i] = {}
op_num = 0
for test_net in solver.test_nets:
for op in test_net.outputs:
output_names[1][op_num] = op
name_to_num[1][op] = op_num
outputs[1][op_num] = {}
op_num += 1
if restore:
# Load back the saved outputs, so we can start from the figures where
# we left off
outputs, max_iter = _load_training_run(caffe_cnn_trrun_id)
start_it = int(solver.iter)
# Filter out data which happened after the snapshot
for op in outputs:
for op_num, its in op.iteritems():
to_remove = []
for it in its:
if it > start_it:
to_remove.append(it)
for it in to_remove:
its.pop(it)
else:
max_iter = solver_params.max_iter
start_it = 0
final_snapshot_id = None
for it in range(start_it, max_iter+1):
#start = time.clock()
solver.step(1) # SGD by Caffe
#elapsed = time.clock() - start
#if options['verbose']:
#print 'One iteration took {:.2f} seconds'.format(elapsed)
display = solver_params.display and it % solver_params.display == 0
if display:
for op in solver.net.outputs:
val = solver.net.blobs[op].data
op_num = name_to_num[0][op]
outputs[0][op_num][it] = float(val)
test_display = solver_params.test_interval and \
it % solver_params.test_interval == 0 and \
(it != 0 or solver_params.test_initialization)
if test_display:
for i, test_net in enumerate(solver.test_nets):
for op in test_net.outputs:
val = solver.test_mean_scores[i][op]
op_num = name_to_num[1][op]
outputs[1][op_num][it] = float(val)
if display or test_display:
_refresh_training_run(
caffe_cnn_trrun_id,
outputs,
output_names,
it,
max_iter,
)
snapshot_path = os.path.join(
settings.CAFFE_ROOT,
'{}_iter_{}.caffemodel'.format(
solver_params.snapshot_prefix,
it
)
)
snapshot = it % solver_params.snapshot == 0 and (it != 0 or options.get('start_snapshot', False))
if snapshot:
if it == 0:
solver.net.save(snapshot_path)
final_snapshot = upload_snapshot(
caffe_cnn_trrun_id=caffe_cnn_trrun_id,
snapshot_path=snapshot_path,
it=it,
verbose=options['verbose'],
)
final_snapshot_id = final_snapshot.id
return final_snapshot_id