def restore_params(network, path='models'):
logging.info("Restore pre-trained parameters")
maybe_download_and_extract(
'resnet50_weights_tf_dim_ordering_tf_kernels.h5',
path,
'https://github.com/fchollet/deep-learning-models/releases/download/v0.2/',
) # ls -al
try:
import h5py
except Exception:
raise ImportError('h5py not imported')
f = h5py.File(
os.path.join(path, 'resnet50_weights_tf_dim_ordering_tf_kernels.h5'),
'r')
for layer in network.all_layers:
if len(layer.all_weights) == 0:
continue
w_names = list(f[layer.name])
params = [f[layer.name][n][:] for n in w_names]
# if 'bn' in layer.name:
# params = [x.reshape(1, 1, 1, -1) for x in params]
assign_weights(params, layer)
del params
f.close()
def restore_model(model, layer_type):
logging.info("Restore pre-trained weights")
# download weights
maybe_download_and_extract(model_saved_name[layer_type], 'models', model_urls[layer_type])
weights = []
if layer_type == 'vgg16':
npz = np.load(os.path.join('models', model_saved_name[layer_type]))
# get weight list
for val in sorted(npz.items()):
logging.info(" Loading weights %s in %s" % (str(val[1].shape), val[0]))
weights.append(val[1])
if len(model.weights) == len(weights):
break
elif layer_type == 'vgg19':
npz = np.load(os.path.join('models', model_saved_name[layer_type]), encoding='latin1').item()
# get weight list
for val in sorted(npz.items()):
logging.info(" Loading %s in %s" % (str(val[1][0].shape), val[0]))
logging.info(" Loading %s in %s" % (str(val[1][1].shape), val[0]))
weights.extend(val[1])
if len(model.weights) == len(weights):
break
# assign weight values
assign_weights(weights, model)
del weights
def restore_params(self, sess):
# For no pre-trained model link
# logging.info("Restore pre-trained parameters")
# vgg19_npy_path = os.path.join('models', 'vgg19.npy')
# if not os.path.isfile(vgg19_npy_path):
# print("Please download vgg19.npy from : https://github.com/machrisaa/tensorflow-vgg")
# exit()
# npz = np.load(vgg19_npy_path, encoding='latin1').item()
# For existing pre-trained model link
logging.info("Restore pre-trained parameters")
maybe_download_and_extract(
'vgg19.npy',
'models',
'https://media.githubusercontent.com/media/tensorlayer/pretrained-models/master/models/',
expected_bytes=574670860)
vgg19_npy_path = os.path.join('models', 'vgg19.npy')
npz = np.load(vgg19_npy_path, encoding='latin1').item()
params = []
for val in sorted(npz.items()):
W = np.asarray(val[1][0])
b = np.asarray(val[1][1])
print(" Loading %s: %s, %s" % (val[0], W.shape, b.shape))
params.extend([W, b])
print("Restoring model from npz file")
assign_params(sess, params, self.net)
del params
def restore_model(model, layer_type):
logging.info("Restore pre-trained weights")
# download weights
maybe_download_and_extract(model_saved_name[layer_type], 'models', model_urls[layer_type])
weights = []
if layer_type == 'vgg16':
npz = np.load(os.path.join('models', model_saved_name[layer_type]), allow_pickle=True)
# get weight list
for val in sorted(npz.items()):
logging.info(" Loading weights %s in %s" % (str(val[1].shape), val[0]))
weights.append(val[1])
if len(model.all_weights) == len(weights):
break
elif layer_type == 'vgg19':
npz = np.load(os.path.join('models', model_saved_name[layer_type]), allow_pickle=True, encoding='latin1').item()
# get weight list
for val in sorted(npz.items()):
logging.info(" Loading %s in %s" % (str(val[1][0].shape), val[0]))
logging.info(" Loading %s in %s" % (str(val[1][1].shape), val[0]))
weights.extend(val[1])
if len(model.all_weights) == len(weights):
break
else:
raise TypeError(f'layer type not supported for restore_model(): {layer_type}')
# assign weight values
# UPDATE: weights must be shorter in len than model.all_weights (caller's duty to check)
# assign_weights(weights, model)
assign_weights(weights[:len(model.all_weights)], model)
del weights
def restore_params(self, sess, path='models'):
logging.info("Restore pre-trained parameters")
maybe_download_and_extract(
'mobilenet.npz', path, 'https://github.com/tensorlayer/pretrained-models/raw/master/models/',
expected_bytes=25600116
) # ls -al
params = load_npz(name=os.path.join(path, 'mobilenet.npz'))
assign_params(sess, params[:len(self.net.all_params)], self.net)
del params
def restore_params(network, path='models'):
logging.info("Restore pre-trained parameters")
maybe_download_and_extract(
'squeezenet.npz', path, 'https://github.com/tensorlayer/pretrained-models/raw/master/models/',
expected_bytes=7405613
) # ls -al
params = load_npz(name=os.path.join(path, 'squeezenet.npz'))
assign_weights(params[:len(network.all_weights)], network)
del params
def restore_params(network, path='models'):
logging.info("Restore pre-trained parameters")
maybe_download_and_extract(
'squeezenet.npz', path, 'https://github.com/tensorlayer/pretrained-models/raw/master/models/',
expected_bytes=7405613
) # ls -al
params = load_npz(name=os.path.join(path, 'squeezenet.npz'))
assign_weights(params[:len(network.weights)], network)
del params
def restore_params(network, path='models'):
logging.info("Restore pre-trained parameters")
maybe_download_and_extract(
'mobilenet.npz',
path,
'https://github.com/tensorlayer/pretrained-models/raw/master/models/',
expected_bytes=25600116) # ls -al
params = load_npz(name=os.path.join(path, 'mobilenet.npz'))
for idx, net_weight in enumerate(network.all_weights):
if 'batchnorm' in net_weight.name:
params[idx] = params[idx].reshape(1, 1, 1, -1)
assign_weights(params[:len(network.all_weights)], network)
del params
def restore_params(self, sess):
logging.info("Restore pre-trained parameters")
maybe_download_and_extract(
'vgg16_weights.npz', 'models', 'http://www.cs.toronto.edu/~frossard/vgg16/', expected_bytes=553436134
)
npz = np.load(os.path.join('models', 'vgg16_weights.npz'))
params = []
for val in sorted(npz.items()):
logging.info(" Loading params %s" % str(val[1].shape))
params.append(val[1])
if len(self.all_params) == len(params):
break
assign_params(sess, params, self.net)
del params
def restore_weights(self):
logging.info("Restore pre-trained weights")
## download weights
maybe_download_and_extract(
'vgg16_weights.npz', 'models', 'http://www.cs.toronto.edu/~frossard/vgg16/', expected_bytes=553436134
)
npz = np.load(os.path.join('models', 'vgg16_weights.npz'))
## get weight list
weights = []
for val in sorted(npz.items()):
logging.info(" Loading weights %s in %s" % (str(val[1].shape), val[0]))
weights.append(val[1])
if len(self.weights) == len(weights):
break
## assign weight values
# print(self.weights)
assign_weights(weights, self)
del weights
def restore_weights(self):
logging.info("Restore pre-trained weights")
## download weights
maybe_download_and_extract(
'vgg16_weights.npz',
'models',
'http://www.cs.toronto.edu/~frossard/vgg16/',
expected_bytes=553436134)
npz = np.load(os.path.join('models', 'vgg16_weights.npz'))
## get weight list
weights = []
for val in sorted(npz.items()):
logging.info(" Loading weights %s in %s" %
(str(val[1].shape), val[0]))
weights.append(val[1])
if len(self.weights) == len(weights):
break
## assign weight values
# print(self.weights)
assign_weights(weights, self)
del weights
def restore_weights(self, sess=None):
logging.info("Restore pre-trained weights")
## download weights
maybe_download_and_extract(
'vgg19.npy', 'models',
'https://media.githubusercontent.com/media/tensorlayer/pretrained-models/master/models/',
expected_bytes=574670860
)
vgg19_npy_path = os.path.join('models', 'vgg19.npy')
npz = np.load(vgg19_npy_path, encoding='latin1').item()
weights = []
for val in sorted(npz.items()):
W = np.asarray(val[1][0])
b = np.asarray(val[1][1])
print(" Loading %s: %s, %s" % (val[0], W.shape, b.shape))
weights.extend([W, b])
if len(self.all_params) == len(weights):
break
## assign weight values
print(self.weights)
assign_weights(sess, weights, self)
del weights
groundtruth_negatives = tf.placeholder(shape=[None, all_default_boxs_len],
dtype=tf.float32,
name='groundtruth_negatives')
groundtruth_count = tf.add(groundtruth_positives, groundtruth_negatives)
pre_cls, pre_loc, net_work = ssd_net(imageinput, True, False)
loss_all, loss_class, loss_loc = mixedloss(pre_cls, pre_loc,
groundtruth_class,
groundtruth_location,
groundtruth_positives,
groundtruth_count)
train_op = tf.train.AdamOptimizer(learning_rate).minimize(loss_all)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
maybe_download_and_extract('vgg16_weights.npz',
'models',
'http://www.cs.toronto.edu/~frossard/vgg16/',
expected_bytes=553436134)
npz = np.load(os.path.join('models', 'vgg16_weights.npz'))
params = []
idx = 0
for val in sorted(npz.items()):
idx = idx + 1
print(" Loading params %s" % (str(val[1].shape)))
params.append(val[1])
if idx == 17:
break
assign_params(sess, params, net_work['block4'])
for epoch in range(train_epoch):
idx = 0
for batch_imgs, batch_ann in tl.iterate.minibatches(laod_imgs,
load_ann,
def load_cifar100_dataset(shape=(-1, 32, 32, 3), path='./data/cifar100', plotable=False, second=3):
print("Load or Download cifar100 > {}".format(path))
#Helper function to unpickle the data
def unpickle(file):
fp = open(file, 'rb')
if sys.version_info.major == 2:
data = pickle.load(fp)
elif sys.version_info.major == 3:
data = pickle.load(fp, encoding='latin-1')
fp.close()
return data
# http://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz
filename = 'cifar-100-python.tar.gz'
url = 'https://www.cs.toronto.edu/~kriz/'
# #Download and uncompress file
maybe_download_and_extract(filename, path, url, extract=True)
#Unpickle file and fill in data
X_train = None
y_train = []
data_dic = unpickle(os.path.join(path,'cifar-100-python/', 'train'))
X_train = data_dic['data']
y_train = data_dic['fine_labels']
test_data_dic = unpickle(os.path.join(path,'cifar-100-python', 'test'))
X_test = test_data_dic['data']
y_test = np.array(test_data_dic['fine_labels'])
if shape == (-1, 3, 32, 32):
X_test = X_test.reshape(shape)
X_train = X_train.reshape(shape)
elif shape == (-1, 32, 32, 3):
X_test = X_test.reshape(shape, order='F')
X_train = X_train.reshape(shape, order='F')
X_test = np.transpose(X_test, (0, 2, 1, 3))
X_train = np.transpose(X_train, (0, 2, 1, 3))
else:
X_test = X_test.reshape(shape)
X_train = X_train.reshape(shape)
y_train = np.array(y_train)
if plotable == True:
print('\nCIFAR-100')
import matplotlib.pyplot as plt
fig = plt.figure(1)
print('Shape of a training image: X_train[0]',X_train[0].shape)
plt.ion() # interactive mode
count = 1
for row in range(10):
for col in range(10):
a = fig.add_subplot(10, 10, count)
if shape == (-1, 3, 32, 32):
plt.imshow(np.transpose(X_train[count-1], (1, 2, 0)), interpolation='nearest')
elif shape == (-1, 32, 32, 3):
plt.imshow(X_train[count-1], interpolation='nearest')
else:
raise Exception("Do not support the given 'shape' to plot the image examples")
plt.gca().xaxis.set_major_locator(plt.NullLocator()) # 不显示刻度(tick)
plt.gca().yaxis.set_major_locator(plt.NullLocator())
count = count + 1
plt.draw() # interactive mode
plt.pause(3) # interactive mode
print("X_train:",X_train.shape)
print("y_train:",y_train.shape)
print("X_test:",X_test.shape)
print("y_test:",y_test.shape)
X_train = np.asarray(X_train, dtype=np.float32)
X_test = np.asarray(X_test, dtype=np.float32)
y_train = np.asarray(y_train, dtype=np.int32)
y_test = np.asarray(y_test, dtype=np.int32)
return X_train, y_train, X_test, y_test