def load_net(fname):
cudaconv_net = IGPUModel.load_checkpoint(fname)
layers = cudaconv_net["model_state"]["layers"]
#Note, data dimensions are hardcoded here - not sure we have that info in the cudaconv object?
decafnet = translator.translate_cuda_network(layers, {'data': (32, 32, 3)})
return decafnet
def load_net(fname):
cudaconv_net = IGPUModel.load_checkpoint(fname)
layers = cudaconv_net["model_state"]["layers"]
#Note, data dimensions are hardcoded here - not sure we have that info in the cudaconv object?
decafnet = translator.translate_cuda_network(layers, {'data': (32, 32, 3)})
return decafnet
def __init__(self, params):
# import only necessary for ConvNet
from gpumodel import IGPUModel
from shownet import ShowConvNet
import options
load_dic = IGPUModel.load_checkpoint(params['mdl_file'][0])
op = load_dic['op']
op2 = ShowConvNet.get_options_parser()
op.merge_from(op2)
op.eval_expr_defaults()
op.set_value('load_file', params['mdl_file'][0])
op.set_value('gpu', [params['gpu']], False)
op.set_value('minibatch_size', params['batch'], False)
op.set_value('write_features', params['layer'], False)
#for k in op.options.keys():
# print k, op.options[k].value
self.model = ShowConvNet(op, load_dic)
layers = [_['name'] for _ in self.model.layers]
params['layer_idx'] = layers.index(params['layer'])
params['output_size'] = params['input_size'] - params['border_size']*2
self.params = params
#initial dummy running
indata = np.zeros((self.params['input_size']**2, self.params['batch']), dtype=np.float32)
outdata = np.zeros((self.params['output_size']**2, self.params['batch']), dtype=np.float32)
data = [indata, outdata]
ftrs = np.zeros((self.params['batch'], self.params['output_size']**2), dtype=np.float32)
self.model.do_write_one_feature(data, ftrs, self.params['layer_idx'])
def load_from_convnet(filename):
cudaconv_net = IGPUModel.load_checkpoint(filename)
layers = cudaconv_net["model_state"]["layers"]
data_layer = [l for l in layers if l["name"] == "data"][0]
data_consumer = [l for l in layers if data_layer in l.get("inputLayers", [])][0]
input_dim = int(sqrt(data_consumer["imgPixels"][0])) # The width/height of the square images
input_channels = data_consumer["channels"][0] # The number of channels in the input images
return translator.translate_cuda_network(layers, {"data": (input_dim, input_dim, input_channels)})
def plot_predictions(self, data_provider, output_file='/tmp/predictions.png', train=True, only_errors=True):
op = shownet.ShowConvNet.get_options_parser()
local_train = train
predict_dict = {
'--write-features': 'probs',
'--feature-path' : '/tmp/feature_path',
'--test-range': '2',
'--train-range': '1',
'--show-preds' : 'probs',
'-f': self.last_model,
'--data-provider': 'dp_scikit',
'--multiview-test': 0,
'--logreg-name': 'aaa'
}
op.parse_from_dictionary(predict_dict)
load_dic = None
options = op.options
if options["load_file"].value_given:
print 'load file option provided'
load_dic = IGPUModel.load_checkpoint(options["load_file"].value)
old_op = load_dic["op"]
old_op.merge_from(op)
op = old_op
op.eval_expr_defaults()
class MyConvNet(shownet.ShowConvNet):
def get_data_dims(self, idx):
return data_provider.get_data_dims(idx)
def get_num_classes(self):
return data_provider.get_num_classes()
def get_next_batch(self, train=True):
return data_provider.get_next_batch(local_train)
def get_num_test_batches(self):
return data_provider.get_num_test_batches()
def get_plottable_data(self, data):
return data_provider.get_plottable_data(data)
def init_data_providers(self):
data_provider.init_data_providers()
def get_label_names(self):
return data_provider.get_label_names()
model = MyConvNet(op, load_dic=load_dic)
model.only_errors = only_errors
model.plot_predictions()
pl.savefig(output_file)
model.cleanup()
def get_model():
global _model
if _model is None:
# This code is adapted from gpumodel.py and shownet.py
load_dic = IGPUModel.load_checkpoint(app.config["TRAINED_MODEL_PATH"])
op = ShowConvNet.get_options_parser()
old_op = load_dic["op"]
old_op.merge_from(op)
op = old_op
_model = ShowConvNet(op, load_dic)
return _model
def get_model():
global _model
if _model is None:
# This code is adapted from gpumodel.py and shownet.py
load_dic = IGPUModel.load_checkpoint(app.config["TRAINED_MODEL_PATH"])
op = ShowConvNet.get_options_parser()
old_op = load_dic["op"]
old_op.merge_from(op)
op = old_op
_model = ShowConvNet(op, load_dic)
return _model
def load_from_convnet(filename):
cudaconv_net = IGPUModel.load_checkpoint(filename)
layers = cudaconv_net["model_state"]["layers"]
data_layer = [l for l in layers if l['name'] == 'data'][0]
data_consumer = [
l for l in layers if data_layer in l.get('inputLayers', [])
][0]
input_dim = int(sqrt(data_consumer['imgPixels']
[0])) # The width/height of the square images
input_channels = data_consumer['channels'][
0] # The number of channels in the input images
return translator.translate_cuda_network(
layers, {'data': (input_dim, input_dim, input_channels)})
def initBFrom(name, shape, params=None):
assert(params != None)
assert(len(params) > 0)
(checkPointFile, layerName) = params[0].split('.')
net = IGPUModel.load_checkpoint(checkPointFile)
layernames = [ layer['name'] for layer in net['model_state']['layers'] ]
if not layerName in layernames:
raise initWError("There is layer named '%s' in file '%s'" % (layerName, checkPointFile))
else:
weightlist = net['model_state']['layers'][layernames.index(layerName)]['biases']
assert(len(weightlist) > 0)
assert(weightlist.shape == shape)
return weightlist
def __init__(self, model_path, data_processor, gpu, layers):
op = ConvNetPredict.get_options_parser()
op.set_value('load_file', model_path)
op.set_value('gpu', str(gpu))
load_dic = IGPUModel.load_checkpoint(model_path)
old_op = load_dic["op"]
old_op.merge_from(op)
op = old_op
op.eval_expr_defaults()
ConvNet.__init__(self, op, load_dic)
self.dp = data_processor
self.ftr_layer_idx = map(self.get_layer_idx, layers)
def initBFrom(name, shape, params=None):
assert (params != None)
assert (len(params) > 0)
(checkPointFile, layerName) = params[0].split('.')
net = IGPUModel.load_checkpoint(checkPointFile)
layernames = [layer['name'] for layer in net['model_state']['layers']]
if not layerName in layernames:
raise initWError("There is layer named '%s' in file '%s'" %
(layerName, checkPointFile))
else:
weightlist = net['model_state']['layers'][layernames.index(
layerName)]['biases']
assert (len(weightlist) > 0)
assert (weightlist.shape == shape)
return weightlist
def get_test_error(self):
data, filenames = self.get_test_patches_and_filenames()
if self.test_on_images:
print 'testing on images'
probabilities, test_results = self.get_predictions(data)
fileProbs, fileLabels, fileIDs = ModelEvaluation.GetUnormalizedJointLogProbability(probabilities, data[1].reshape(-1), filenames)
filePredictions = ModelEvaluation.GetPredictions(fileProbs)
fileAccuracy, misclassifiedFiles = ModelEvaluation.CalculateAccuracy(filePredictions, fileLabels)
nExamples = test_results[1]
results = ({'logprob' : [test_results[0]['logprob'][0], (1-fileAccuracy) * test_results[1]]}, test_results[1])
if self.test_only: # Print the individual batch results for safety
print str(results)
else:
print 'not testing on images'
self.libmodel.startBatch(data, True)
results = self.finish_batch()
self.regular_test_outputs += [IGPUModel.get_test_error(self)]
return results
Transition_matrix = dic['Transition_matrix']
Prior = dic['Prior']
outPred = './training/pred/'
####################################
##################load CNN here######
import getopt as opt
from gpumodel import IGPUModel
from options import *
op = ShowConvNet.get_options_parser()
op.options['load_file'].value = r'.\tmp\tmp\ConvNet__2014-05-14_21.42.41'
op.options['feature_path'].value = r'.\prediction_feature'
op.options['test_batch_range'].value = 1
op.options['write_features'].value = 'probs'
load_dic = IGPUModel.load_checkpoint(op.options["load_file"].value)
old_op = load_dic["op"]
old_op.merge_from(op)
op = old_op
op.eval_expr_defaults()
#op, load_dic = IGPUModel.parse_options(op)
op.options['train_batch_range'].value = [1]
op.options['test_batch_range'].value = [1]
model = ShowConvNet(op, load_dic)
meta = pickle.load(open(r'.\storage\batches.meta'))
data_mean = meta['data_mean']
###############################
# load prestore template
ref_depth = numpy.load('distance_median.npy')
template = cv2.imread('template.png')
# result.append(item + ';' +model.ClassNameTest(item)+';'+ door + '\n')
ground_truth=model.ClassNameTest(item)
print ground_truth,door
if not model.ClassNameTest(item)=='0':
P_num +=1
if not door == ground_truth:
error += 1
erro_ratio = float(error)/i
print erro_ratio
print i,P_num,len(result),error
# result.append('error_ratio:'+str(erro_ratio)+' Positive_num:'+str(P_num)+' total_num:'+str(i))
# myreslut = sorted(result, key=lambda result:result[0])
# if P_num<2000:
# my_result = file('myresult_p.txt', 'wb')
# else:
# my_result = file('myresult_n.txt', 'wb')
# my_result.writelines(myreslut)
# my_result.close()
except (UnpickleError, ShowNetError, opt.GetoptError), e:
print "----------------"
print "Error:"
print e
print 'finish_8'
op = ShowPredction.get_options_parser()
op, load_dic = IGPUModel.parse_options(op)
model = ShowPredction(op, load_dic)
print os.path.exists("G:\\door_data_sampling\\posture\\data_pos\\test\\test_value_p\\")
show_predict_dir('G:\\door_data_sampling\\posture\\test\\org_data\\')
if not os.path.exists(outPred):
os.makedirs(outPred)
####################################
##################load CNN here######
import getopt as opt
from gpumodel import IGPUModel
from options import *
op = ShowConvNet.get_options_parser()
#op.options['load_file'].value=r'.\ConvNet_3DCNN\tmp\ConvNet__2014-05-28_01.59.00'
### old
op.options['load_file'].value=r'I:\Kaggle_multimodal\StartingKit_track3\Final_project\ConvNet_3DCNN\tmp\ConvNet__2014-05-26_03.40.18'
op.options['write_features'].value ='probs'
load_dic = IGPUModel.load_checkpoint(op.options["load_file"].value)
old_op = load_dic["op"]
old_op.merge_from(op)
op = old_op
op.eval_expr_defaults()
op.options['train_batch_range'].value=[1]
op.options['test_batch_range'].value=[1]
op.options['data_path'].value=r'.\ConvNet_3DCNN\storage_sk_final'
model = ShowConvNet(op, load_dic)
model.crop_border = 0
meta = pickle.load(open(r'.\ConvNet_3DCNN\storage_sk_final\batches.meta'))
data_mean = meta['data_mean']
def fit(self, X, y, use_starting_point=True, **kwargs):
print 'about to fit ConvNetLearn'
if use_starting_point and self.last_model is not None:
self.dict['-f']=self.last_model
op = convnet.ConvNet.get_options_parser()
op.parse_from_dictionary(self.dict)
load_dic = None
options = op.options
if options["load_file"].value_given:
print 'load file option provided'
load_dic = IGPUModel.load_checkpoint(options["load_file"].value)
name_to_weights = {}
if self.init_states_models is not None:
name_to_weights = {}
for init_model in self.init_states_models:
load_dic_local = IGPUModel.load_checkpoint(init_model)
for k, v in load_dic_local['model_state'].iteritems():
if k == 'layers':
for elem in v:
name = elem.get('name')
weights = elem.get('weights')
if weights is not None:
print 'adding weights for layer {}'.format(name)
if name not in name_to_weights:
name_to_weights[name] = {}
name_to_weights[name]['weights'] = weights
name_to_weights[name]['biases'] = elem.get('biases')
name_to_weights[name]['weightsInc'] = elem.get('weightsInc')
name_to_weights[name]['biasesInc'] = elem.get('biasesInc')
if len(name_to_weights) > 0:
print 'layer names with init arrays: {}'.format(name_to_weights.keys())
for k, v in load_dic['model_state'].iteritems():
if k == 'layers':
for elem in v:
name = elem.get('name')
print 'name of layer to possibly be updated {}'.format(name)
weights = elem.get('weights')
if weights is not None:
if name in name_to_weights:
print 'changing init point of model for layer {}'.format(name)
coefs_name = name_to_weights.get(name)
if coefs_name is None or 'weights' not in coefs_name:
raise Exception('coeef names doent have weights for {}, coef names fields: {}'.format(name, coefs_name.keys()))
elem['weights'] = coefs_name['weights']
elem['biases'] = coefs_name['biases']
elem['weightsInc'] = coefs_name['weightsInc']
elem['biasesInc'] = coefs_name['biasesInc']
old_op = load_dic["op"]
old_op.merge_from(op)
op = old_op
op.eval_expr_defaults()
try:
self.dict.pop('-f')
except:
pass
if hasattr(X, 'get_next_batch'):
data_provider = X
else:
data_provider = InMemorySplitDataProvider(X,y,fraction_test=self.fraction_test)
data_provider.init_data_providers()
num_classes = data_provider.get_num_classes()
logger.info('num classes {}'.format(num_classes))
#we adjust the number of classes dynamically
for name in self.mcp_layers.sections():
if self.mcp_layers.has_option(name,'outputs'):
if self.mcp_layers.get(name, 'outputs') == 'num_classes':
self.mcp_layers.set( name, 'outputs', value='{}'.format(num_classes))
##################
class MyConvNet(convnet.ConvNet):
def __init__(self, op, load_dic, mcp_layers, mcp_params, fraction_test):
self.layer_def_dict = mcp_layers
self.layer_params_dict = mcp_params
convnet.ConvNet.__init__(self,op,load_dic=load_dic,initialize_from_file=False)
self.test_one = True
self.epoch = 1
self.max_filesize_mb = 5000
def get_data_dims(self, idx):
return data_provider.get_data_dims(idx)
def get_num_classes(self):
return data_provider.get_num_classes()
def get_next_batch(self, train=True):
return data_provider.get_next_batch(train)
def get_num_test_batches(self):
return data_provider.get_num_test_batches()
def init_data_providers(self):
data_provider.init_data_providers()
model = MyConvNet(op, load_dic=load_dic, mcp_layers=self.mcp_layers, mcp_params=self.mcp_params, fraction_test=self.fraction_test)
self.last_model = join(self.output_folder, model.save_file)
print 'last model name {}'.format(self.last_model)
model.start()
def predict_proba(self, X, train=True):
op = shownet.ShowConvNet.get_options_parser()
predict_dict = {
'--write-features': 'probs',
'--feature-path' : '/tmp/feature_path',
'--test-range': '2',
'--train-range': '1',
'-f': self.last_model,
'--data-provider': 'dp_scikit',
'--show-preds' : '',
'--multiview-test': 0,
'--logreg-name': 'aaa'
}
op.parse_from_dictionary(predict_dict)
load_dic = None
options = op.options
if options["load_file"].value_given:
print 'load file option provided'
load_dic = IGPUModel.load_checkpoint(options["load_file"].value)
old_op = load_dic["op"]
old_op.merge_from(op)
op = old_op
op.eval_expr_defaults()
if hasattr(X, 'get_next_batch'):
data_provider = X
else:
data_provider = InMemorySplitDataProvider(X,None,fraction_test=0.0)
data_provider.init_data_providers()
class MyConvNet(shownet.ShowConvNet):
def init_data_providers(self):
self.dp_params['convnet'] = self
def compute_probs(self, X):
out = None
while True:
data_all = data_provider.get_next_batch(train=train)
epoch, batch = data_all[0], data_all[1]
if epoch != 1:
break
print 'working on epoch: {}, batch: {}'.format(epoch, batch)
data = data_all[2]
if isinstance(data[0], list):
data_point = data[0][4]
else:
data_point = data[0].shape[1]
print 'data points {}'.format(data_point)
num_ftrs = self.layers[self.ftr_layer_idx]['outputs']
ftrs = np.zeros((data_point, num_ftrs), dtype=np.single)
self.libmodel.startFeatureWriter(data + [ftrs], self.ftr_layer_idx)
self.finish_batch()
if out is None:
out = ftrs
else:
out = np.vstack((out,ftrs))
return out
model = MyConvNet(op, load_dic=load_dic)
probs = model.compute_probs(data_provider)
model.cleanup()
return probs
"Queue key")
op.add_option("ensemble-id", "ensemble_id", IntegerOptionParser,
"Id of predict ensemble")
op.add_option("iteration-id", "iteration_id", IntegerOptionParser,
"Id of predict iteration")
op.add_option("data-dir", "data_dir", StringOptionParser,
"Id of predict ensemble")
op.add_option("is-dataset",
"is_dataset",
BooleanOptionParser,
"Format output as file and upload on s3",
default=False)
op.options['load_file'].default = None
return op
if __name__ == "__main__":
try:
op = ShowConvNet.get_options_parser()
op, load_dic, batch_meta = IGPUModel.parse_options(op)
model = ShowConvNet(op, load_dic, {
'batch_meta': batch_meta,
'data_dir': op.options['data_dir'].value
})
model.start()
except (UnpickleError, ShowNetError, opt.GetoptError), e:
print "----------------"
print "Error:"
print e
