def load_params(self, filename):
cwd = os.path.dirname(os.path.realpath(__file__))
with open(os.path.join(cwd,filename+'.zip'),'rb') as f:
params = load(f)
#print params,self.params[0].get_value()
for i,param in enumerate(params):
self.full_params[i].set_value(param)
def test_save_the_best():
with NamedTemporaryFile(dir=config.temp_dir) as dst,\
NamedTemporaryFile(dir=config.temp_dir) as dst_best:
track_cost = TrackTheBest("cost", after_epoch=False, after_batch=True)
main_loop = MockMainLoop(
extensions=[FinishAfter(after_n_epochs=1),
WriteCostExtension(),
track_cost,
Checkpoint(dst.name, after_batch=True,
save_separately=['log'])
.add_condition(
["after_batch"],
OnLogRecord(track_cost.notification_name),
(dst_best.name,))])
main_loop.run()
assert main_loop.log[4]['saved_to'] == (dst.name, dst_best.name)
assert main_loop.log[5]['saved_to'] == (dst.name, dst_best.name)
assert main_loop.log[6]['saved_to'] == (dst.name,)
with open(dst_best.name, 'rb') as src:
assert load(src).log.status['iterations_done'] == 5
root, ext = os.path.splitext(dst_best.name)
log_path = root + "_log" + ext
with open(log_path, 'rb') as src:
assert cPickle.load(src).status['iterations_done'] == 5
def test_save_the_best():
with NamedTemporaryFile() as dst,\
NamedTemporaryFile() as dst_best:
track_cost = TrackTheBest("cost", after_epoch=False, after_batch=True)
main_loop = MockMainLoop(
extensions=[FinishAfter(after_n_epochs=1),
WriteCostExtension(),
track_cost,
Checkpoint(dst.name, after_batch=True,
save_separately=['log'])
.add_condition(
["after_batch"],
OnLogRecord(track_cost.notification_name),
(dst_best.name,))])
main_loop.run()
assert main_loop.log[4]['saved_to'] == (dst.name, dst_best.name)
assert main_loop.log[5]['saved_to'] == (dst.name, dst_best.name)
assert main_loop.log[6]['saved_to'] == (dst.name,)
with open(dst_best.name, 'rb') as src:
assert load(src).log.status['iterations_done'] == 5
root, ext = os.path.splitext(dst_best.name)
log_path = root + "_log" + ext
with open(log_path, 'rb') as src:
assert cPickle.load(src).status['iterations_done'] == 5
def test_save_the_best():
skip_if_configuration_set("log_backend", "sqlite", "Known to be flaky with SQLite log backend.")
with NamedTemporaryFile(dir=config.temp_dir) as dst, NamedTemporaryFile(dir=config.temp_dir) as dst_best:
track_cost = TrackTheBest("cost", after_epoch=False, after_batch=True)
main_loop = MockMainLoop(
extensions=[
FinishAfter(after_n_epochs=1),
WriteCostExtension(),
track_cost,
Checkpoint(dst.name, after_batch=True, save_separately=["log"]).add_condition(
["after_batch"], OnLogRecord(track_cost.notification_name), (dst_best.name,)
),
]
)
main_loop.run()
assert main_loop.log[4]["saved_to"] == (dst.name, dst_best.name)
assert main_loop.log[5]["saved_to"] == (dst.name, dst_best.name)
assert main_loop.log[6]["saved_to"] == (dst.name,)
with open(dst_best.name, "rb") as src:
assert load(src).log.status["iterations_done"] == 5
root, ext = os.path.splitext(dst_best.name)
log_path = root + "_log" + ext
with open(log_path, "rb") as src:
assert cPickle.load(src).status["iterations_done"] == 5
def post(self, *args, **kwargs):
"""
Returns a JSON with requested template based on IDs sent
"""
dbresponse = ""
model_id = ""
# db connection variable
session = kwargs["db_connection"]
dataset = request.files['dataset']
model_id = request.form['info']
# check if ids list is populated
try:
fetch = session.query(Model).filter(Model.MODEL == model_id).one()
finally:
# releases the database cursor
session.close()
with open("models/" + fetch.MODEL + ".zip", "rb") as f:
model = pkl.load(f)
response = list()
data, _, _ = api_utils.file_treatment(dataset)
for line in data:
response.append(int(model.pred(line)))
return response
def loads(self, file_path):
with open(file_path, 'rb') as f:
self.params = load(f)
index = 0
for layer in self.layer_list:
if layer.trainable:
for i in range(len(layer.params())):
layer.set_params(i, self.params[index])
index += 1
def load_model_params(model, model_path):
with open(os.path.join(model_path, "params.zip"), 'r') as f:
saved_parameters = pkl_utils.load(f)
for i, param in enumerate(saved_parameters[0]):
model.parameters[i].set_value(param.get_value())
for i, param in enumerate(saved_parameters[1]):
model.update_rule.parameters[i].set_value(param.get_value())
def load_model_params(model, model_path):
with open(os.path.join(model_path, "params.zip"), 'r') as f:
saved_parameters = pkl_utils.load(f)
for i, param in enumerate(saved_parameters[0]):
model.parameters[i].set_value(param.get_value())
for i, param in enumerate(saved_parameters[1]):
model.update_rule.parameters[i].set_value(param.get_value())
def test_dump_load_mrg(self):
rng = MRG_RandomStreams()
with open('test', 'wb') as f:
dump(rng, f)
with open('test', 'rb') as f:
rng = load(f)
assert type(rng) == MRG_RandomStreams
def test_dump_load_mrg(self):
rng = MRG_RandomStreams(use_cuda=cuda_ndarray.cuda_enabled)
with open('test', 'wb') as f:
dump(rng, f)
with open('test', 'rb') as f:
rng = load(f)
assert type(rng) == MRG_RandomStreams
def test_dump_load_mrg(self):
rng = MRG_RandomStream()
with open("test", "wb") as f:
dump(rng, f)
with open("test", "rb") as f:
rng = load(f)
assert type(rng) == MRG_RandomStream
def test_dump_load_mrg(self):
rng = MRG_RandomStreams(use_cuda=cuda_ndarray.cuda_enabled)
with open('test', 'wb') as f:
dump(rng, f)
with open('test', 'rb') as f:
rng = load(f)
assert type(rng) == MRG_RandomStreams
def test_dump_zip_names():
foo_1 = theano.shared(0, name='foo')
foo_2 = theano.shared(1, name='foo')
with open('model.zip', 'wb') as f:
dump((foo_1, foo_2, numpy.array(2)), f)
keys = numpy.load('model.zip').keys()
assert keys == ['foo', 'foo_2', 'array_0', 'pkl']
foo = numpy.load('model.zip')['foo']
assert foo == numpy.array(0)
with open('model.zip', 'rb') as f:
foo_1, foo_2, array = load(f)
assert array == numpy.array(2)
def load_feature_maps(self, filename, src_layer, src_fmap_nums, dst_layer, dst_fmap_nums):
cwd = os.path.dirname(os.path.realpath(__file__))
with open(os.path.join(cwd,filename+'.zip'),'rb') as f:
params = load(f)
curr_fmaps = self.layers[dst_layer].w.get_value()
curr_biases = self.layers[dst_layer].b.get_value()
src_w_param_num = self.full_params.index(self.layers[src_layer].w)
src_b_param_num = self.full_params.index(self.layers[src_layer].b)
for src_fmap_num, dst_fmap_num in zip(src_fmap_nums,dst_fmap_nums):
curr_fmaps[dst_fmap_num] = params[src_w_param_num][src_fmap_num]
curr_biases[dst_fmap_num] = params[src_b_param_num][src_fmap_num]
self.layers[dst_layer].w.set_value(curr_fmaps)
self.layers[dst_layer].b.set_value(curr_biases)
def test_dump_zip_names(self):
foo_1 = theano.shared(0, name='foo')
foo_2 = theano.shared(1, name='foo')
foo_3 = theano.shared(2, name='foo')
with open('model.zip', 'wb') as f:
dump((foo_1, foo_2, foo_3, np.array(3)), f)
keys = list(np.load('model.zip').keys())
assert keys == ['foo', 'foo_2', 'foo_3', 'array_0', 'pkl']
foo_3 = np.load('model.zip')['foo_3']
assert foo_3 == np.array(2)
with open('model.zip', 'rb') as f:
foo_1, foo_2, foo_3, array = load(f)
assert array == np.array(3)
def test_dump_zip_names(self):
foo_1 = theano.shared(0, name="foo")
foo_2 = theano.shared(1, name="foo")
foo_3 = theano.shared(2, name="foo")
with open("model.zip", "wb") as f:
dump((foo_1, foo_2, foo_3, np.array(3)), f)
keys = list(np.load("model.zip").keys())
assert keys == ["foo", "foo_2", "foo_3", "array_0", "pkl"]
foo_3 = np.load("model.zip")["foo_3"]
assert foo_3 == np.array(2)
with open("model.zip", "rb") as f:
foo_1, foo_2, foo_3, array = load(f)
assert array == np.array(3)
def test_dump_zip_names(self):
foo_1 = theano.shared(0, name='foo')
foo_2 = theano.shared(1, name='foo')
foo_3 = theano.shared(2, name='foo')
with open('model.zip', 'wb') as f:
dump((foo_1, foo_2, foo_3, np.array(3)), f)
keys = list(np.load('model.zip').keys())
assert keys == ['foo', 'foo_2', 'foo_3', 'array_0', 'pkl']
foo_3 = np.load('model.zip')['foo_3']
assert foo_3 == np.array(2)
with open('model.zip', 'rb') as f:
foo_1, foo_2, foo_3, array = load(f)
assert array == np.array(3)
def test_dump_load():
x = GpuArraySharedVariable(
'x',
GpuArrayType('float32', (1, 1), name='x', context_name=test_ctx_name),
[[1]], False)
with open('test', 'wb') as f:
dump(x, f)
with open('test', 'rb') as f:
x = load(f)
assert x.name == 'x'
np.testing.assert_allclose(x.get_value(), [[1]])
def test_dump_load():
x = GpuArraySharedVariable('x',
GpuArrayType('float32', (1, 1), name='x',
context_name=test_ctx_name),
[[1]], False)
with open('test', 'wb') as f:
dump(x, f)
with open('test', 'rb') as f:
x = load(f)
assert x.name == 'x'
np.testing.assert_allclose(x.get_value(), [[1]])
def test_dump_load(self):
if not cuda_ndarray.cuda_enabled:
raise SkipTest('Optional package cuda disabled')
x = CudaNdarraySharedVariable('x', CudaNdarrayType((1, 1), name='x'),
[[1]], False)
with open('test', 'wb') as f:
dump(x, f)
with open('test', 'rb') as f:
x = load(f)
assert x.name == 'x'
assert_allclose(x.get_value(), [[1]])
def test_dump_load(self):
if not cuda_ndarray.cuda_enabled:
raise SkipTest('Optional package cuda disabled')
x = CudaNdarraySharedVariable('x', CudaNdarrayType((1, 1), name='x'),
[[1]], False)
with open('test', 'wb') as f:
dump(x, f)
with open('test', 'rb') as f:
x = load(f)
assert x.name == 'x'
assert_allclose(x.get_value(), [[1]])
def test_dump_load():
x = GpuArraySharedVariable(
"x",
GpuArrayType("float32", (1, 1), name="x", context_name=test_ctx_name),
[[1]],
False,
)
with open("test", "wb") as f:
dump(x, f)
with open("test", "rb") as f:
x = load(f)
assert x.name == "x"
np.testing.assert_allclose(x.get_value(), [[1]])
def LoadModel(self, file):
print 'Loading model paramters'
with open(file, 'rb') as f:
model = pkl_utils.load(f)
for value, param in zip(model, self.params):
param.set_value(np.asarray(value, dtype=theano.config.floatX))
if __name__ == '__main__':
temp_folder = tempfile.mkdtemp()
print('Creating temporary folder: {}'.format(temp_folder))
streaming_metrics = [
metric(topn=ARGS.eval_metrics_top_n) for metric in [HitRate, MRR]
]
if ARGS.warmup_model_dump_file:
print('Loading pre-trained GRU model from: {}'.format(
ARGS.warmup_model_dump_file))
gru_file = open(ARGS.warmup_model_dump_file, "rb+")
try:
gru = pkl.load(gru_file)
finally:
gru_file.close()
else:
gru = GRU4Rec(
n_epochs=ARGS.n_epochs,
loss='bpr-max-0.5',
final_act='linear',
hidden_act='tanh',
layers=[300],
adapt=ARGS.optimizer,
decay=0.0,
batch_size=ARGS.batch_size,
dropout_p_embed=0.0,
dropout_p_hidden=ARGS.dropout_p_hidden,
def __init__(self, modelfile):
#config.experimental.unpickle_gpu_on_cpu = True
self.model = pickle.load(open(modelfile, 'rb'))
def __init__(self,
rng,
input,
in_channels,
dropout_rates,
batch_size,
use_bias=True):
# rectified_linear_activation = lambda x: T.maximum(0.0, x)
# Set up all the hidden layers
# weight_matrix_sizes = zip(layer_sizes, layer_sizes[1:])
self.layers = []
self.dropout_layers = []
# with open('/mnt/md1/a503denglei/weights/weight_3DCNN_gpu_20181217.zip','rb') as f:
with open('/home/a503tongxueheng/jupyter_project/weight_3DCNN.zip',
'rb') as f:
w0, w1, w2, w3, w4, w5, b0, b1, b2, b3, b4, b5 = load(f)
#######
filter_w = 3
num_3d_pixel = 20
layer0_w = num_3d_pixel
layer0_h = num_3d_pixel
layer0_d = num_3d_pixel
layer1_w = (layer0_w - 3 + 1) # 14
layer1_h = (layer0_h - 3 + 1)
layer1_d = (layer0_d - 3 + 1)
layer2_w = (layer1_w - 3 + 1) / 2 # 14
layer2_h = (layer1_h - 3 + 1) / 2
layer2_d = (layer1_d - 3 + 1) / 2
layer3_w = (layer2_w - 3 + 1) / 2
layer3_h = (layer2_h - 3 + 1) / 2
layer3_d = (layer2_d - 3 + 1) / 2
######################
# BUILD ACTUAL MODEL #
######################
print '... building the model'
# image sizes
batchsize = batch_size
in_time = num_3d_pixel
in_width = num_3d_pixel
in_height = num_3d_pixel
# filter sizes
flt_channels = 100
flt_time = filter_w
flt_width = filter_w
flt_height = filter_w
signals_shape0 = (batchsize, in_time, in_channels, in_height, in_width)
filters_shape0 = (flt_channels, 3, in_channels, 3, 3)
signals_shape1 = (batchsize, layer1_d, flt_channels, layer1_h,
layer1_w)
filters_shape1 = (flt_channels * 2, 3, flt_channels, 3, 3)
signals_shape2 = (batchsize, layer2_d, flt_channels * 2, layer2_h,
layer2_w)
filters_shape2 = (flt_channels * 4, 3, flt_channels * 2, 3, 3)
# next_layer_input = input
# first_layer = True
# dropout the input
layer0_input = input.reshape(signals_shape0) # 20
# layer0_dropout_input = _dropout_from_layer(rng, layer0_input, p=dropout_rates[0])
# W0: flt_channels, flt_time, in_channels, flt_height, flt_width
###################################
Dropout_layer0 = Dropout_Conv_3d_Layer(rng=rng,
input=layer0_input,
image_shape=signals_shape0,
filter_shape=filters_shape0,
dropout_rate=dropout_rates[0],
W=w0,
b=b0)
self.dropout_layers.append(Dropout_layer0)
next_dropout_input = Dropout_layer0.output
layer0 = Conv_3d_Layer(
rng=rng,
input=layer0_input, # 18
image_shape=signals_shape0,
filter_shape=filters_shape0,
W=Dropout_layer0.W * (1 - dropout_rates[0]),
b=Dropout_layer0.b * (1 - dropout_rates[0]),
)
self.layers.append(layer0)
next_layer_input = layer0.output
##################################
###################################
Dropout_layer1 = Dropout_Conv_3d_Layer(rng=rng,
input=next_dropout_input,
image_shape=signals_shape1,
filter_shape=filters_shape1,
dropout_rate=dropout_rates[1],
W=w1,
b=b1)
Dropout_layer1_pool = PoolLayer3D(
input=Dropout_layer1.output.dimshuffle(0, 2, 1, 3, 4),
pool_shape=(2, 2, 2)) # 4
self.dropout_layers.append(Dropout_layer1)
# self.dropout_layers.append(Dropout_layer1_pool)
next_dropout_input = Dropout_layer1_pool.output.dimshuffle(
0, 2, 1, 3, 4)
layer1 = Conv_3d_Layer(
rng=rng,
input=next_layer_input, # N*4*12*12*12 => N*7*10*10*10
image_shape=signals_shape1,
filter_shape=filters_shape1,
W=Dropout_layer1.W * (1 - dropout_rates[1]),
b=Dropout_layer1.b * (1 - dropout_rates[1]),
)
layer1_pool = PoolLayer3D(input=layer1.output.dimshuffle(
0, 2, 1, 3, 4),
pool_shape=(2, 2, 2)) # 4
self.layers.append(layer1)
# self.layers.append(layer1_pool)
next_layer_input = layer1_pool.output.dimshuffle(0, 2, 1, 3, 4)
##################################
###################################
Dropout_layer2 = Dropout_Conv_3d_Layer(rng=rng,
input=next_dropout_input,
image_shape=signals_shape2,
filter_shape=filters_shape2,
dropout_rate=dropout_rates[2],
W=w2,
b=b2)
Dropout_layer2_pool = PoolLayer3D(
input=Dropout_layer2.output.dimshuffle(0, 2, 1, 3, 4),
pool_shape=(2, 2, 2)) # 4
self.dropout_layers.append(Dropout_layer2)
# self.dropout_layers.append(Dropout_layer2_pool)
next_dropout_input = Dropout_layer2_pool.output.dimshuffle(
0, 2, 1, 3, 4).flatten(2)
layer2 = Conv_3d_Layer(
rng=rng,
input=next_layer_input, # N*4*12*12*12 => N*7*10*10*10
image_shape=signals_shape2,
filter_shape=filters_shape2,
W=Dropout_layer2.W * (1 - dropout_rates[2]),
b=Dropout_layer2.b * (1 - dropout_rates[2]),
)
layer2_pool = PoolLayer3D(input=layer2.output.dimshuffle(
0, 2, 1, 3, 4),
pool_shape=(2, 2, 2)) # 4
self.layers.append(layer2)
# self.layers.append(layer2_pool)
next_layer_input = layer2_pool.output.dimshuffle(0, 2, 1, 3,
4).flatten(2)
##################################
# W4: 200*layer4_w*layer4_h, 500
Dropout_layer3 = DropoutHiddenLayer(rng=rng,
input=next_dropout_input,
activation=relu,
n_in=(flt_channels * 4 * layer3_d *
layer3_w * layer3_h),
n_out=1000,
dropout_rate=dropout_rates[3],
W=w3,
b=b3)
self.dropout_layers.append(Dropout_layer3)
next_dropout_input = Dropout_layer3.output
# Reuse the paramters from the dropout layer here, in a different
# path through the graph.
layer3 = HiddenLayer(
rng=rng,
input=next_layer_input,
activation=relu,
# scale the weight matrix W with (1-p)
W=Dropout_layer3.W * (1 - dropout_rates[3]),
b=Dropout_layer3.b * (1 - dropout_rates[3]),
n_in=(flt_channels * 4 * layer3_d * layer3_w * layer3_h),
n_out=1000,
)
self.layers.append(layer3)
next_layer_input = layer3.output
##################################
# layer4_input = layer2.output.flatten(2) # N*200
# W4: 200*layer4_w*layer4_h, 500
Dropout_layer4 = DropoutHiddenLayer(rng=rng,
input=next_dropout_input,
activation=relu,
n_in=1000,
n_out=100,
dropout_rate=dropout_rates[4],
W=w4,
b=b4)
self.dropout_layers.append(Dropout_layer4)
next_dropout_input = Dropout_layer4.output
# Reuse the paramters from the dropout layer here, in a different
# path through the graph.
layer4 = HiddenLayer(
rng=rng,
input=next_layer_input,
activation=relu,
# scale the weight matrix W with (1-p)
W=Dropout_layer4.W * (1 - dropout_rates[4]),
b=Dropout_layer4.b * (1 - dropout_rates[4]),
n_in=1000,
n_out=100,
)
self.layers.append(layer4)
next_layer_input = layer4.output
##################### TODO #######################
Dropout_layer5 = LogisticRegression(input=next_dropout_input,
n_in=100,
n_out=20,
W=w5,
b=b5)
self.dropout_layers.append(Dropout_layer5)
# Again, reuse paramters in the dropout output.
layer5 = LogisticRegression(
input=next_layer_input,
# scale the weight matrix W with (1-p)
W=Dropout_layer5.W,
b=Dropout_layer5.b,
n_in=100,
n_out=20)
self.layers.append(layer5)
# Use the negative log likelihood of the logistic regression layer as
# the objective.
self.dropout_negative_log_likelihood = self.dropout_layers[
-1].negative_log_likelihood
self.dropout_errors = self.dropout_layers[-1].errors
self.negative_log_likelihood = self.layers[-1].negative_log_likelihood
self.errors = self.layers[-1].errors
self.L2_sqr = T.sum(Dropout_layer0.W**2) + T.sum(
Dropout_layer1.W**2) + T.sum(Dropout_layer2.W**2) + T.sum(
Dropout_layer3.W**2) + T.sum(Dropout_layer4.W**2) + T.sum(
Dropout_layer5.W**2)
# Grab all the parameters together.
self.params = [
param for layer in self.dropout_layers for param in layer.params
]
#my code
self.class_score = self.layers[-1].class_score
self.p_y_given_x = self.layers[-1].p_y_given_x
pass
sys.stdout = Logger( os.environ['HOME' ] + '/theano.log' )
PATH_TO_TRAIN = os.environ['HOME']+'/rsc15_train_full.txt'
PATH_TO_TEST = os.environ['HOME']+'/rsc15_test.txt'
LAYERS = 1
if __name__ == '__main__':
data = pd.read_csv(PATH_TO_TRAIN, sep='\t', dtype={'ItemId':np.int64})
valid = pd.read_csv(PATH_TO_TEST, sep='\t', dtype={'ItemId':np.int64})
#load model
fd = open('model','rb')
gru = load(fd)
print('Model: {}'.format(gru))
batch_size=10
session_ids = valid.SessionId.values[0:batch_size]
input_item_ids = valid.ItemId.values[0:batch_size]
out_idx = valid.ItemId.values[0:batch_size]
uniq_out = np.unique(np.array(out_idx, dtype=np.int32))
#predict_for_item_ids = np.hstack([data, uniq_out[~np.in1d(uniq_out,data)]])
predict_for_item_ids = None
print('session_ids: {}'.format(session_ids))
def load_params_pickle(file):
with open(file, 'rb') as f:
loaded_params = load(f)
return loaded_params
def load_model(params, filename):
from theano.misc import pkl_utils
with open(filename, 'rb') as fin:
for param in params:
param.set_value(pkl_utils.load(fin))
def LoadModel(self,file): print 'Loading model paramters' with open(file,'rb') as f: model = pkl_utils.load(f) for value,param in zip(model,self.params): param.set_value(np.asarray(value,dtype=theano.config.floatX))
def load_model(filename):
with open(filename, 'rb') as f:
saved = load(f)
print "\nSuccessfully loaded architecture from file: %s" % filename
return saved
def load_params_pickle(file):
with open(file, 'rb') as f:
loaded_params = load(f)
return loaded_params
