def load_from_file(fname, batch_size=32, layer_range=None):
t = time.time()
logging.info("Loading model from %s" % fname)
model = load_from_json(common.json_from_gz(fname), batch_size, layer_range)
model.fname = fname
logging.verbose("Loading model took %.2f sec" % (time.time() - t))
return model
def predict_output(self, dataset):
dataset_x, dataset_y, dataset_size = dataset.export(self.batch_size)
#dummy call to build function
self.predict_output_step(dataset_x[:self.batch_size])
#evaluate function
timer = common.Timer()
n = math.ceil(dataset_size / self.batch_size)
pr = []
for index in range(n):
data_x = dataset_x[index * self.batch_size:(index + 1) *
self.batch_size]
pr_batch = self.predict_output_step(data_x)
pr.append(pr_batch)
pr = numpy.concatenate(pr, axis=0)
logging.verbose("Prediction took %.3f sec for %i samples" %
(timer.current(), pr.shape[0]))
#crop dummy data
if (dataset_size % self.batch_size) != 0:
s = [dataset_size] + list(pr.shape[1:])
pr.resize(tuple(s), refcheck=False)
return pr
def __init__(self,
layers,
size=(2, 2),
stride=None,
pad=(0, 0),
mode="max",
ignore_border=True,
json_param={}):
super().__init__(layer_index=len(layers))
self.input = layers[-1].output
self.input_shape = layers[-1].output_shape
self.size = json_param.get("size", size)
self.pad = json_param.get("pad", pad)
self.ignore_border = json_param.get("ignoreBorder", ignore_border)
self.mode = json_param.get("mode", mode)
self.stride = json_param.get("stride", stride)
if self.stride is None:
self.stride = self.size
#output dim
if self.ignore_border:
h = int(
math.floor(
(self.input_shape[2] + 2 * self.pad[0] - self.size[0]) /
self.stride[0])) + 1
w = int(
math.floor(
(self.input_shape[3] + 2 * self.pad[1] - self.size[1]) /
self.stride[1])) + 1
else:
h = int(
math.ceil(
(self.input_shape[2] + 2 * self.pad[0]) / self.stride[0]))
w = int(
math.ceil(
(self.input_shape[3] + 2 * self.pad[1]) / self.stride[1]))
#theano optimizer is sometimes failing to use cudnn pooling!
use_cudnn = (dnn.dnn_available() and dnn.version() >= (4000, 4000)
and self.ignore_border)
if use_cudnn:
self.output = dnn.dnn_pool(self.input,
ws=self.size,
pad=self.pad,
stride=self.stride,
mode=self.mode)
else:
self.output = tensor.signal.pool.pool_2d(
self.input,
ds=self.size,
padding=self.pad,
ignore_border=self.ignore_border,
st=self.stride,
mode=self.mode)
self.output_shape = (self.input_shape[0], self.input_shape[1], h, w)
logging.verbose("Adding", self)
def __init__(self,
layers,
sample_feat=512,
cost_factor=1,
dropout=0.0,
use_center=False,
json_param={}):
super().__init__(layer_index=len(layers))
#pass though layer
self.input = layers[-1].output
self.input_shape = layers[-1].output_shape
self.output = layers[-1].output
self.output_shape = layers[-1].output_shape
self.batch_size, self.features, self.height, self.width = self.input_shape
#get param
self.sample_feat = json_param.get("sampleFeat", sample_feat)
self.cost_factor = json_param.get("costFactor", cost_factor)
self.use_center = json_param.get("useCenter", use_center)
self.dropout = json_param.get("dropout", dropout)
self.corner_num = 5 if self.use_center else 4
self.layers = [InitialLayer(self.input, self.input_shape)]
self.layers.append(
ConvLayer(
self.layers,
(self.corner_num + self.sample_feat, self.features, 1, 1),
(1, 1), True, False))
#initialize corner_pr weights / biases
omega = self.layers[-1].omega.get_value()
omega[:self.corner_num, :, :, :] = 0.0
self.layers[-1].omega.set_value(omega)
beta = self.layers[-1].beta.get_value()
beta[:self.corner_num] = 5.0
self.layers[-1].beta.set_value(beta)
#extract corner probabilities
self.corner_shape = (self.batch_size, 2, self.corner_num, self.height,
self.width)
self.corner_lh = self.layers[-1].output[:, :self.corner_num, :, :]
self.corner_lh = tensor.concatenate(
[self.corner_lh[:, None, ...], -self.corner_lh[:, None, ...]],
axis=1)
self.corner_pr = theano_util.log_softmax(self.corner_lh, axis=[1])
#extract sample
self.sample_shape = (self.batch_size, self.sample_feat, self.height,
self.width)
self.sample_shared = theano.shared(
numpy.zeros(self.sample_shape, dtype=numpy.float32),
"shared sample")
self.sample = self.layers[-1].output[:, self.corner_num:, :, :]
logging.verbose("Adding", self)
def __init__(self,
layers,
class_num=10,
overlap_threshold=0.5,
cost_factor=1.0,
bbox_factor=0.0,
json_param={}):
super().__init__(layer_index=len(layers))
#passthrough layer
self.input = layers[-1].output
self.input_shape = layers[-1].output_shape
self.output = layers[-1].output
self.output_shape = layers[-1].output_shape
self.cost_factor = json_param.get("costFactor", cost_factor)
self.bbox_factor = json_param.get("bboxFactor", bbox_factor)
self.class_num = json_param.get("classNum", class_num)
self.overlap_threshold = json_param.get("overlapThreshold",
overlap_threshold)
#find sparse / instcount / compare layer
self.sparse_layer = common.find_layers(layers, "denet-sparse", False)
assert self.sparse_layer != None, "Error: Requires denet-sparse layer to be specified before denet-detect layer!"
self.use_bbox_reg = (self.bbox_factor > 0.0)
self.batch_size = self.sparse_layer.batch_size
self.sample_num = self.sparse_layer.sample_num
self.null_class = self.class_num
#
s0 = self.class_num + 1
s1 = 4 if self.use_bbox_reg else 0
self.layers = [
ConvLayer([InitialLayer(self.input, self.input_shape)],
(s0 + s1, self.input_shape[1], 1, 1), (1, 1), True,
"valid", 0.0)
]
#class assignment log(Pr({c,null} | sample_j, sample_i))
self.det_shape = (self.batch_size, s0, self.sample_num,
self.sample_num)
self.det_lh = self.layers[-1].output[:, :s0, ...]
self.det_pr = theano_util.log_softmax(self.det_lh, axis=[1])
#bbox regression
if self.use_bbox_reg:
self.bbox_shape = (self.batch_size, s1, self.sample_num,
self.sample_num)
self.bbox_reg = self.layers[-1].output[:, s0:(s0 + s1), ...]
self.detect_func = None
self.nms_func = None
logging.verbose("Adding", self)
def __init__(self, layers, activation="relu", json_param={}):
super().__init__(layer_index=len(layers))
self.input = layers[-1].output
self.input_shape = layers[-1].output_shape
self.activation=json_param.get("activation", activation)
self.output_shape = self.input_shape
self.output = ActivationLayer.apply(self.input, self.activation)
#apply activation
logging.verbose("Adding", self)
def __init__(self, layers, json_param={}):
super().__init__(layer_index=len(layers))
self.enabled = json_param.get("enabled", True)
self.has_split = self.enabled
self.input = layers[-1].output
self.output_shape = self.input_shape = layers[-1].output_shape
if self.enabled:
self.output = theano.shared(numpy.zeros(self.output_shape).astype(theano.config.floatX), str(self) + " - output")
else:
self.output = self.input
logging.verbose("Adding", self, "layer - input:", self.input_shape, "enabled:", self.enabled)
def __init__(self, layers, skip_index=0, split=False, json_param={}):
super().__init__(layer_index=len(layers))
self.skip_index = json_param.get("index", skip_index)
self.has_split = json_param.get("split", split)
self.input = layers[-1].output
self.input_shape = layers[-1].output_shape
self.output_shape = self.input_shape
if self.has_split:
self.output = theano.shared(numpy.zeros(self.output_shape).astype(theano.config.floatX), str(self) + " - output")
self.skip = theano.shared(numpy.zeros(self.output_shape).astype(theano.config.floatX), str(self) + " - skip")
else:
self.skip = self.output = self.input
logging.verbose("Adding", self, "split:", self.has_split)
def get_target(self, model, data_x, metas):
sample_bboxs = self.get_samples(data_x, train=True)
total_cover = 0
total_bbox = 0
for b, meta in enumerate(metas):
#compute groundtruth coverage
cover = 0
for meta_bbox in meta["bbox"]:
for _, sample_bbox in sample_bboxs[b]:
if common.overlap_iou(meta_bbox, sample_bbox) > 0.5:
cover += 1
break
logging.verbose(
"%i: corner detector found %i samples (%i/%i coverage)" %
(b, len(sample_bboxs[b]), cover, len(meta["bbox"])))
total_cover += cover
total_bbox += len(meta["bbox"])
n = self.sample_count - math.floor(
self.random_sample * self.sample_count)
if len(sample_bboxs[b]) > n:
# logging.verbose("%i: removing %i samples to make room for random samples"%(b, len(sample_bboxs[b]) - n))
sample_bboxs[b] = random.sample(sample_bboxs[b], n)
#add random samples if bbox detector produces too few
while len(sample_bboxs[b]) < self.sample_count:
x0 = random.uniform(0.0, 1.0)
y0 = random.uniform(0.0, 1.0)
x1 = random.uniform(x0, 1.0)
y1 = random.uniform(y0, 1.0)
bbox = (x0, y0, x1, y1)
sample_bboxs[b].append((0.0, bbox))
#insert groundtruth
if self.sample_gt:
for index, bbox in enumerate(meta["bbox"]):
sample_bboxs[b][-(index + 1)] = (1.0, bbox)
logging.verbose(
"Overall %i/%i (%.2f%%) coverage" %
(total_cover, total_bbox, 100.0 * total_cover / total_bbox))
self.set_samples(sample_bboxs)
return None
def __init__(self, layers, size=(2, 2), json_param={}):
super().__init__(layer_index=len(layers))
self.input = layers[-1].output
self.input_shape = layers[-1].output_shape
self.size=json_param.get("size", size)
#output dim
self.output_shape = (self.input_shape[0], self.input_shape[1], self.size[1]*self.input_shape[2], self.size[0]*self.input_shape[3])
self.use_optimized = theano.sandbox.cuda.cuda_enabled
if self.use_optimized:
self.output = PoolInvOp(self.size)(self.input)
else:
self.output = tensor.repeat(tensor.repeat(self.input, self.size[1], axis=2), self.size[0], axis=3)
logging.verbose("Adding", self)
def build(self,
model_desc,
data_shape,
activation="relu",
border_mode="valid",
weight_init="he-forward"):
self.model_desc = " ".join(model_desc)
self.data_shape = data_shape
self.layers = [InitialLayer(self.input, self.get_input_shape())]
for i, layer_desc in enumerate(model_desc):
wb = weight_init[min(len(weight_init) - 1, i)]
self.build_layer(layer_desc, self.layers, activation, border_mode,
wb)
logging.verbose("Number of parameters in model: %d" %
self.get_parameter_num())
def get_samples(self, data_x, train=False, store_shared=False):
global profile
if self.corner_func is None:
logging.verbose("Building corner function - store samples:",
store_shared, "train:", train)
updates = [(self.corner_layer.sample_shared,
self.corner_layer.sample)] if store_shared else []
self.corner_func = theano.function(
[self.model_input],
self.corner_layer.corner_pr,
updates=updates,
profile=profile,
givens=[(get_train(), tensor.cast(int(train), 'int8'))],
on_unused_input='ignore')
#find corners
timer = common.Timer()
logging.debug("Running corner function")
corner_pr = self.corner_func(data_x)
if profile:
logging.debug("Profiling corner function")
theano_util.profile(self.corner_func, 10, data_x)
theano_util.export_graph("./corner.graph", self.corner_func)
logging.debug("Done")
exit(0)
#build sampling bounding boxs
timer.mark()
logging.debug("Build samples (%i threads)" % self.thread_num)
samples = c_code.build_samples(self.thread_num, corner_pr,
self.corner_threshold, self.sample_num,
self.corner_max, self.local_max,
self.nms_threshold)
timer.mark()
logging.verbose(
"Took %i ms to get_samples (%i model, %i build, %i max corners) " %
(timer.current_ms(), timer.delta_ms(0), timer.delta_ms(1),
self.corner_max))
return samples
def __init__(self, layers, dropout_rate=1.0, json_param={}):
super().__init__(layer_index=len(layers))
self.input = layers[-1].output
self.input_shape = layers[-1].output_shape
#get parameters
self.dropout_rate = json_param.get("dropoutRate", dropout_rate)
scale = 1.0 / (1.0 - self.dropout_rate)
mask = get_rng().binomial(n=1,
p=1.0 - self.dropout_rate,
size=self.input_shape)
mask = tensor.cast(mask, theano.config.floatX)
self.output = theano.ifelse.ifelse(get_train(),
self.input * mask * scale,
self.input)
self.output_shape = self.input_shape
logging.verbose("Adding", self, "layer - input:", self.input_shape,
"dropout: %.0f%%" % (100.0 * self.dropout_rate))
def __init__(self, layers, use_center=True, valid=[], json_param={}):
super().__init__(layer_index=len(layers))
self.input = layers[-1].output
self.input_shape = layers[-1].output_shape
#used for determining multiview map
self.multiview_layers = layers
if use_center:
yc = self.input_shape[-2] // 2
xc = self.input_shape[-1] // 2
valid = [(0,yc,xc)]
self.valid = json_param.get("valid", valid)
#could be done faster with advanced indexing
if len(self.valid) > 0:
x_shape = (self.input_shape[0], self.input_shape[1], len(self.valid))
x = tensor.zeros(x_shape)
for i, offset in enumerate(self.valid):
x = tensor.set_subtensor(x[:,:,i], self.input[:,:,offset[1],offset[2]])
else:
x = self.input
x_shape = self.input_shape
self.log_likelihood = x
self.log_pr = self.log_softmax(x, axis=1)
self.log_pr_shape = x_shape
#return probabilities as output
self.output_shape = (self.log_pr_shape[0], self.log_pr_shape[1])
if len(self.log_pr_shape) > 2:
self.output = tensor.exp(self.log_pr).mean(axis=range(2, len(self.log_pr_shape)))
else:
self.output = tensor.exp(self.log_pr)
logging.verbose("Adding", self, "layer - input:", self.input_shape, "valid:", self.valid)
def train_epoch(self,
dataset,
epoch,
learning_rate,
momentum=[0, 1, 0],
decay=0.0,
solver_mode="sgd"):
#train over batches (assume dataset size is mulitple of batch_size!)
logging.info("Evaluating training function")
dataset_x, dataset_m, dataset_size = dataset.export(self.batch_size)
index_num = math.ceil(dataset_size / self.batch_size)
total_cost = 0
for index in range(index_num):
#upload data to GPU and perform train step
timer = common.Timer()
data_x = dataset_x[index * self.batch_size:(index + 1) *
self.batch_size]
data_m = dataset_m[index * self.batch_size:(index + 1) *
self.batch_size]
cost, _ = self.train_step(data_x, data_m, epoch, self.iteration,
learning_rate, momentum, decay)
#watch out for GPU's randomly producing NaN!
if math.isnan(cost):
raise Exception("ERROR: Cost is NaN")
logging.verbose(
"Batch %i.%i - iteration: %i cost:" %
(epoch, index * self.batch_size, self.iteration), cost,
"took: %i ms" % timer.current_ms())
total_cost += cost
self.iteration += 1
return total_cost
def __init__(self, layers, skip_index=0, combine_mode="proj-add", json_param={}):
super().__init__(layer_index=len(layers))
self.combine_mode = json_param.get("combineMode", combine_mode)
self.skip_index = json_param.get("index", skip_index)
self.skip_layer=None
for layer in layers:
if layer.type_name == "skip-src" and layer.skip_index == self.skip_index:
self.skip_layer=layer
break
assert self.skip_layer != None
self.x = layers[-1].output
self.x_shape = layers[-1].output_shape
self.y = self.skip_layer.skip
self.y_shape = self.skip_layer.output_shape
if self.combine_mode == "proj-add":
self.output_shape = self.x_shape
if self.y_shape[1] != self.x_shape[1]:
self.layers = [InitialLayer(self.y, self.y_shape)]
self.layers.append(ConvLayer(self.layers, filter_shape = (self.x_shape[1], self.y_shape[1], 1, 1)))
self.output = self.x + self.layers[-1].output
else:
self.output = self.x + self.y
elif self.combine_mode == "concat":
self.output_shape = (self.x_shape[0], self.x_shape[1] + self.y_shape[1], self.x_shape[2], self.x_shape[3])
self.output = tensor.concatenate([self.x, self.y], axis=1)
else:
raise Exception("Unknown combine mode: %s"%self.combine_mode)
logging.verbose("Adding", self, "layer - x:", self.x_shape, "y:", self.y_shape, "skip index:", self.skip_index, "combine mode:", self.combine_mode)
def train_step(self, data_x, data_m, epoch, it, learning_rate, momentum,
decay):
#assert "train" in self.func, "Call build_train_func() before calling train_step()"
denet.layer.get_iteration().set_value(it)
momentum = numpy.array(momentum, dtype=numpy.float32)
#for split mode explicitly perform bwd/fwd propagation
if self.use_split_mode:
#only get_targets after train_fwd has been performed
targets = []
fwd_index = 0
for layer in self.layers:
if layer.has_split:
logging.verbose("Forward prop %i" % fwd_index)
self.func["train_fwd"][fwd_index](data_x)
fwd_index += 1
target = layer.get_target(self, data_x, data_m)
if not target is None:
print(layer.type_name)
targets += target
# print(targets)
logging.verbose("Backward prop %i" % 0)
costs = self.func["train_bwd"][0](epoch, it, learning_rate,
momentum, decay, data_x,
*targets)
for i, f_bwd in enumerate(self.func["train_bwd"][1:]):
logging.verbose("Backward prop %i" % (i + 1))
f_bwd(epoch, it, learning_rate, momentum, decay, data_x,
*targets)
else:
targets = []
for layer in self.layers:
target = layer.get_target(self, data_x, data_m)
if not target is None:
targets += target
costs = self.func["train_step"](epoch, it, learning_rate, momentum,
decay, data_x, *targets)
return costs[0], costs[1:]
def run_sync(self):
#connect to clients
server_socket, client_sockets = self.connect_clients()
#construct update object for each client / server
client_updates = [
shared.ModelUpdate(self.model_dims) for _ in range(self.client_num)
]
server_update = shared.ModelUpdate(self.model_dims)
#perform synchronization
while True:
try:
logging.info("Waiting for updates...")
for i, sock in enumerate(client_sockets):
update_json = network.recv_json(sock)
client_updates[i].import_json(update_json["data"])
logging.info("Synchronising...")
ts = time.time()
server_update.set_mean(client_updates, self.thread_num)
logging.verbose("mean calc took %.2f sec" % (time.time() - ts))
ts = time.time()
server_json = server_update.export_json()
logging.verbose("json export took %.2f sec" %
(time.time() - ts))
#send mean update to clients
ts = time.time()
for sock in client_sockets:
network.send_json(sock, server_json)
logging.verbose("transferring data to clients took %.2f sec" %
(time.time() - ts))
except (KeyboardInterrupt, SystemExit):
logging.info("Done")
sys.exit(0)
except Exception as e:
logging.error("Encounter exception: ", e)
def __init__(self,
layers,
grid_size=3,
sample_num=16,
corner_threshold=0.01,
random_sample=0.0,
local_max=0,
nms_threshold=0.7,
sample_gt=True,
version="v2",
json_param={}):
super().__init__(layer_index=len(layers))
self.input = layers[-1].output
self.input_shape = layers[-1].output_shape
self.batch_size = self.input_shape[0]
self.model_input = layers[0].output
self.grid_size = json_param.get("gridSize", grid_size)
self.sample_num = json_param.get("sampleNum", sample_num)
self.sample_gt = json_param.get("sampleGT", sample_gt)
self.corner_threshold = json_param.get("cornerThreshold",
corner_threshold)
self.nms_threshold = json_param.get("nmsThreshold", nms_threshold)
self.random_sample = json_param.get("randomSample", random_sample)
self.local_max = json_param.get("localMax", local_max)
self.version = json_param.get("version", version)
self.corner_max = 1024
self.thread_num = self.batch_size
self.sample_count = self.sample_num * self.sample_num
#find corner layer
self.corner_func = None
self.corner_layer = common.find_layers(layers, "denet-corner", True)
assert not self.corner_layer is None, "denet-corner layer required before spare layer!"
#sampling bounding boxs
self.sample_bbox_list = []
self.sample_bbox = theano.shared(
value=numpy.zeros((self.batch_size, self.sample_num,
self.sample_num, 4),
dtype=numpy.float32))
self.output_feat = self.grid_size * self.grid_size * self.corner_layer.sample_shape[
1] + 2
self.output_shape = (self.batch_size, self.output_feat,
self.sample_num, self.sample_num)
#Use optimized op!
sample_input = tensor.switch(get_train(), self.corner_layer.sample,
self.corner_layer.sample_shared)
self.use_optimized = theano.sandbox.cuda.cuda_enabled
if self.use_optimized:
self.output = DeNetSparseOp(self.grid_size)(sample_input,
self.sample_bbox)
else:
sample_w = self.sample_bbox[:, :, :, 2] - self.sample_bbox[:, :, :,
0]
sample_h = self.sample_bbox[:, :, :, 3] - self.sample_bbox[:, :, :,
1]
sample_x = self.sample_bbox[:, :, :, 0, None] + tensor.arange(
self.grid_size, dtype='float32')[
None, None,
None, :] * sample_w[:, :, :, None] / (self.grid_size - 1)
sample_y = self.sample_bbox[:, :, :, 1, None] + tensor.arange(
self.grid_size, dtype='float32')[
None, None,
None, :] * sample_h[:, :, :, None] / (self.grid_size - 1)
batch_range = tensor.arange(self.batch_size,
dtype='int64')[:, None, None, None,
None]
grid_range = tensor.arange(self.grid_size * self.grid_size,
dtype='int64').reshape(
(self.grid_size,
self.grid_size))[None, None,
None, :, :]
#extract samples
b, f, h, w = self.corner_layer.sample_shape
#get sample position in feature map (b,sj,si,gy/gx)
sample_xf = tensor.maximum(0, tensor.minimum(sample_x * w, w - 1))
sample_yf = tensor.maximum(0, tensor.minimum(sample_y * h, h - 1))
syf = tensor.cast(sample_yf.round(), 'int64')
sxf = tensor.cast(sample_xf.round(), 'int64')
byxc = batch_range * h * w + syf[:, :, :, :,
None] * w + sxf[:, :, :, None, :]
#extract sample
sample = sample_input.dimshuffle((1, 0, 2, 3)).flatten(2)
sample = sample[:, byxc.flatten()]
sample = sample.reshape((f, b, self.sample_num, self.sample_num,
self.grid_size * self.grid_size))
sample = sample.dimshuffle((1, 2, 3, 4, 0))
sample = sample.reshape((b, self.sample_num, self.sample_num,
self.grid_size * self.grid_size * f))
sample = sample.dimshuffle((0, 3, 1, 2))
#add width / height
self.output = tensor.concatenate(
[sample, sample_h[:, None, :, :], sample_w[:, None, :, :]],
axis=1)
global c_code
logging.verbose("Adding", self)
def load(self,
input_dir,
data_format,
is_training,
thread_num,
class_labels=None):
from .basic import DatasetFromDir
self.input_dir = input_dir
if self.input_dir[-1] == '/':
self.input_dir = self.input_dir[:-1]
self.data_format = data_format
self.thread_num = thread_num
#generate class labels
self.class_labels = class_labels
fname = os.path.join(os.path.dirname(self.input_dir),
"class_labels.txt")
if os.path.isfile(fname) and self.class_labels is None:
logging.info("Loading class labels from:", fname)
self.class_labels = {}
with open(fname, "r") as f:
for line in f.readlines():
tokens = line.rstrip('\n').split(" ")
self.class_labels[tokens[1]] = int(tokens[0])
elif self.class_labels is None:
self.class_labels = DatasetFromDir.find_class_labels(input_dir)
#check to see if buffered file list is present
list_fname = os.path.join(input_dir, "image_list.json")
if os.path.isfile(list_fname):
logging.info("Loading dataset metadata:", list_fname)
json_data = common.json_from_file(list_fname)
if json_data.get("version", 0) < 1:
logging.warning(
"Warning: image_list.json is old version, missing bounding boxs!"
)
self.images = [{
"fname": fname,
"bboxs": []
} for fname in json_data["images"]]
else:
self.images = json_data["images"]
else:
bbox_dir = os.path.join(os.path.dirname(input_dir), "bbox")
if not os.path.isdir(bbox_dir):
raise Exception("ERROR: cannot find bbox dir:" + bbox_dir)
fnames = []
for i, c in enumerate(os.listdir(input_dir).sort()):
images_cls = DatasetFromDir.find_paths(
os.path.join(input_dir, c), "*.JPEG")
logging.info("Found %i images for class" % len(images_cls), c)
fnames += images_cls
logging.info("Finding bboxs in:", bbox_dir)
self.images = []
for i, fname in enumerate(fnames):
logging.verbose("%i/%i" % (i, len(fnames)))
cls_name = os.path.basename(os.path.dirname(fname))
obj_fname = os.path.join(
bbox_dir, cls_name,
os.path.splitext(os.path.basename(fname))[0] + ".xml")
bboxs = []
if os.path.isfile(obj_fname):
obj_tree = xml.parse(obj_fname).getroot()
size = obj_tree.find("size")
width = int(size.find("width").text)
height = int(size.find("height").text)
for obj in obj_tree.iter("object"):
bndbox = obj.find("bndbox")
min_x = int(bndbox.find("xmin").text)
min_y = int(bndbox.find("ymin").text)
max_x = int(bndbox.find("xmax").text)
max_y = int(bndbox.find("ymax").text)
bboxs.append({
"x0": min_x,
"x1": max_x,
"y0": min_y,
"y1": max_y
})
self.images.append({"fname": fname, "bboxs": bboxs})
try:
logging.info("Saving dataset metadata:", list_fname)
common.json_to_file(list_fname, {
"images": self.images,
"version": 1
})
except Exception as e:
logging.warning(
"Warning: failed to write buffered image list - ", e)
#add/fix fields to fit new image_loader interface
for image in self.images:
fname = image["fname"]
cls = self.class_labels[os.path.basename(os.path.dirname(fname))]
image["class"] = cls
image["bboxs"] = [(cls, (bb["x0"], bb["y0"], bb["x1"], bb["y1"]))
for bb in image["bboxs"]]
param_str = ",".join(data_format.split(",")[1:])
format_params = common.get_params_dict(param_str)
self.image_loader = ImageLoader(thread_num, is_training, format_params)
#from facebook resnet implementation
self.image_loader.rgb_mean = numpy.array([0.485, 0.456, 0.406],
dtype=numpy.float32)
self.image_loader.rgb_std = numpy.array([0.229, 0.224, 0.225],
dtype=numpy.float32)
self.image_loader.rgb_eigen_val = numpy.array([0.2175, 0.0188, 0.0045],
dtype=numpy.float32)
self.image_loader.rgb_eigen_vec = numpy.array(
[[-0.5675, 0.7192, 0.4009], [-0.5808, -0.0045, -0.8140],
[-0.5836, -0.6948, 0.4203]],
dtype=numpy.float32)
#others
self.subset_size = format_params.get("images_per_subset", 10000)
self.use_null_class = format_params.get("null", False)
self.subset_num = format_params.get("subset_num", sys.maxsize)
self.bbox_only = format_params.get("bbox_only", False)
#only use samples with bounding boxes
if self.image_loader.is_training and self.bbox_only:
images_bbox = []
for image in self.images:
if len(image["bboxs"]) > 0:
images_bbox.append(image)
self.images = images_bbox
#append null class
if self.use_null_class and not "null" in self.class_labels:
self.class_labels["null"] = len(self.class_labels)
self.subset_index = -1
self.subset_total_size = len(self.images)
self.subset_num = min(
self.subset_num,
int(math.ceil(self.subset_total_size / self.subset_size)))
logging.info("Using Imagenet dataset - size:", self.subset_total_size,
"subset_num", self.subset_num, "images per subset:",
self.subset_size, self.image_loader)
def export_graph(fname, func):
logging.verbose("Saving function graph: " + fname)
with open(fname, "w") as f:
theano.printing.debugprint(func, file=f, print_type=True)
def load(self,
input_dir,
data_format,
is_training,
thread_num,
class_labels=None):
self.data = []
self.thread_num = thread_num
param_str = ",".join(data_format.split(",")[1:])
format_params = common.get_params_dict(param_str)
self.data_types = []
if format_params.get("2014-train", False):
self.data_types.append("train2014")
if format_params.get("2014-val", False):
self.data_types.append("val2014")
if format_params.get("2014-test", False):
self.data_types.append("test2014")
if format_params.get("2015-test", False):
self.data_types.append("test2015")
if format_params.get("2015-test-dev", False):
self.data_types.append("test-dev2015")
if len(self.data_types) == 0:
raise Exception("please specify mscoco subset")
bbox_hist = [0 for _ in range(32)]
self.images = []
self.class_labels = {}
self.categories = None
for data_type in self.data_types:
if "test" in data_type:
fname = os.path.join(
input_dir, "annotations/image_info_%s.json" % data_type)
else:
fname = os.path.join(
input_dir, "annotations/instances_%s.json" % data_type)
json_data = common.json_from_file(fname)
#get class labels
data_categories = {}
for i, json_cat in enumerate(json_data["categories"]):
data_categories[json_cat["id"]] = json_cat["name"]
if not json_cat["name"] in self.class_labels:
self.class_labels[json_cat["name"]] = len(
self.class_labels)
assert (self.categories is None) or (self.categories
== data_categories)
self.categories = data_categories
logging.verbose("Found %i labels:" % len(self.class_labels))
#collect bounding boxes
bboxs = {}
for json_ann in json_data.get("annotations", []):
cls_id = self.class_labels[self.categories[
json_ann["category_id"]]]
image_id = json_ann["image_id"]
bbox = json_ann["bbox"]
if not image_id in bboxs:
bboxs[image_id] = []
bboxs[image_id].append(
(cls_id, (bbox[0], bbox[1], bbox[0] + bbox[2],
bbox[1] + bbox[3])))
logging.verbose("Found %i bboxs" %
(sum([len(bbox) for bbox in bboxs.values()])))
#collect images
if data_type == "test-dev2015":
data_type = "test2015"
for image in json_data["images"]:
fname = image["file_name"]
image_id = image["id"]
bbox_list = bboxs.get(image_id, [])
bbox_hist[min(len(bbox_list), 31)] += 1
self.images.append({
"fname":
os.path.join(input_dir, data_type, fname),
"bboxs":
bbox_list,
"id":
image_id
})
print("BBox histogram (%):",
[round(100.0 * x / sum(bbox_hist), 1) for x in bbox_hist])
#setup image loader
self.image_loader = ImageLoader(thread_num, is_training, format_params)
#subset
self.output_size = self.image_loader.crop
self.images_per_subset = format_params.get("images_per_subset", 10000)
self.subset_total_size = len(self.images)
self.subset_num = format_params.get("subset_num", sys.maxsize)
self.subset_num = min(
self.subset_num,
int(math.ceil(self.subset_total_size / self.images_per_subset)))
self.subset_index = -1
self.subset_size = self.images_per_subset
self.bbox_only = format_params.get("bbox_only", False)
#only use samples with bounding boxes
if self.image_loader.is_training and self.bbox_only:
images_bbox = []
for image in self.images:
if len(image["bboxs"]) > 0:
images_bbox.append(image)
logging.info("Removed %i images without bboxs" %
(len(self.images) - len(images_bbox)))
self.images = images_bbox
logging.info("Using MSCOCO dataset - size:", self.subset_total_size,
"subset_num", self.subset_num, "images per subset:",
self.subset_size, self.image_loader)
def save_to_file(model, fname, compresslevel=9):
logging.info("Saving model to %s" % fname)
t = time.time()
common.json_to_gz(fname, model.export_json(), compresslevel)
logging.verbose("Saving model took %.2f sec" % (time.time() - t))
def main():
#load arguments:
parser = argparse.ArgumentParser(
description='Train a convolutional network using labelled data.')
logging.add_arguments(parser)
parser.add_argument("--model",
required=False,
default=None,
help="Model to continue training.")
parser.add_argument("--cost-factors",
default=[],
nargs="+",
help="Multiplicative factors for model costs")
parser.add_argument(
"--thread-num",
type=int,
default=1,
help=
"Number of threads to use for supported opeartions (e.g. loading/distorting datasets)"
)
parser.add_argument("--extension",
default="ppm",
help="Image file extension")
parser.add_argument("--train",
default=None,
help="The folder with training / validation data")
parser.add_argument("--test",
default=None,
help="The folder with testing data (optional)")
parser.add_argument("--test-epochs",
type=int,
default=1,
help="Epochs between each test evaluation")
parser.add_argument("--test-mode",
default="default",
help="Mode to use for testing")
parser.add_argument(
"--border-mode",
default="valid",
help="Border mode for convolutional layers (full, valid)")
parser.add_argument("--output-prefix",
default="./model",
help="Output prefix for model files")
parser.add_argument(
"--activation",
default="relu",
help=
"Activation function used in convolution / hidden layers (tanh, relu, leaky-relu)"
)
parser.add_argument("--solver", type=str, default="nesterov", help="")
parser.add_argument("--weight-init",
nargs="+",
default=["he-backward"],
help="Weight initialization scheme")
parser.add_argument("--learn-rate",
type=float,
default=0.1,
help="Learning rate for weights and biases.")
parser.add_argument(
"--learn-momentum",
type=float,
default=[0.0, 0.0],
nargs="+",
help="Learning momentum for weights and biases (0.0 - 1.0).")
parser.add_argument(
"--learn-anneal",
type=float,
default=1,
help="Annealing factor per epoch for weight and bias learning rate")
parser.add_argument(
"--learn-anneal-epochs",
nargs="+",
type=int,
default=[],
help="Epochs to apply learning rate annealing (default every epoch)")
parser.add_argument("--learn-decay",
type=float,
default=0.0,
help="L2 weight decay (not applied to biases). ")
parser.add_argument("--epochs",
type=int,
default=30,
help="The number of training epochs")
parser.add_argument("--max-samples",
type=int,
default=None,
help="Maximum samples to load from training set")
parser.add_argument("--batch-size",
type=int,
default=32,
help="Size of processing batchs")
parser.add_argument("--seed",
type=int,
default=23455,
help="Random Seed for weights")
parser.add_argument(
"--distort-mode",
default=[],
nargs="+",
help="Distortions to apply to training data (default, cifar10, disable)"
)
parser.add_argument("--disable-intermediate",
default=False,
action="store_true",
help="Disable outputting of intermediate model files")
parser.add_argument(
"--augment-mirror",
default=False,
action="store_true",
help="Augment training data with horizontally mirrored copies")
parser.add_argument("--skip-train",
default=False,
action="store_true",
help="Skip training of model")
parser.add_argument("--skip-layer-updates",
type=int,
nargs="+",
default=[],
help="Skip training updates to specified layers")
parser.add_argument("--model-desc",
default=[
"C[100,7]", "P[2]", "C[150,4]", "P[2]", "C[250,4]",
"P[2]", "C[300,1]", "R"
],
nargs="+",
type=str,
help="Network layer description")
args = parser.parse_args()
logging.init(args)
#set random seeds
random.seed(args.seed)
numpy.random.seed(args.seed)
#load training dataset
logging.info("Loading training data:", args.train)
train_data = dataset.load(args.train,
args.extension,
is_training=True,
thread_num=args.thread_num)
data_shape = train_data.get_data_shape()
class_num = train_data.get_class_num()
class_labels = train_data.class_labels
logging.info("Found %i class labels:\n" % class_num, class_labels)
#hack for reducing training data size
if not args.max_samples is None:
train_data.data = random.sample(train_data.data, args.max_samples)
#mirror training data
if args.augment_mirror:
train_data.augment_mirror()
logging.info("Training: %i samples" % len(train_data))
#load test dataset
if args.test:
logging.info("Loading test: " + args.test)
test_data = dataset.load(args.test,
args.extension,
is_training=False,
thread_num=args.thread_num,
class_labels=class_labels)
#initialize model
model = model_cnn.initialize(args, data_shape, class_labels, class_num)
model.build_train_func(args.solver, args.cost_factors)
#Run training
best_test_error = 100.0
learn_rate = args.learn_rate
for epoch in range(args.epochs):
logging.info("----- Training Epoch: %i -----" % epoch)
#perform training
if not args.skip_train:
logging.info("Training with solver " + args.solver +
", learning rate " + str(learn_rate) +
" and momentum " + str(args.learn_momentum))
#shuffle dataset:
train_data.shuffle()
for subset in range(train_data.subset_num):
timer = common.Timer()
train_data.load_from_subset(subset)
logging.info("Performing Gradient Descent...")
cost = model.train_epoch(train_data, epoch, learn_rate,
args.learn_momentum, args.learn_decay)
nbatch = math.ceil(len(train_data) / model.batch_size)
logging.info("Training subset %i - Cost: %.3f, Took %.1f sec" %
(subset, cost, timer.current()))
if len(args.learn_anneal_epochs) == 0 or (
epoch + 1) in args.learn_anneal_epochs:
logging.verbose("Annealing learning rate")
learn_rate *= args.learn_anneal
#perform testing
test_error = 0
if not args.test is None and ((epoch % args.test_epochs) == 0
or epoch == (args.epochs - 1)):
test_error, test_class_errors = compute_error(test_data, model)
logging.info(
"Epoch %i test error: %.2f%% (%i samples)" %
(epoch, test_error, int(test_error * len(test_data) / 100.0)))
save_results(args.output_prefix + "_epoch%03i.test" % epoch,
test_error, test_class_errors)
#save intermediate models
if not args.disable_intermediate:
model_cnn.save_to_file(
model, args.output_prefix + "_epoch%03i.mdl.gz" % (epoch))
#save final model
model_cnn.save_to_file(
model, args.output_prefix + "_epoch%03i_final.mdl.gz" % epoch)
logging.info("Finished Training")
def build_train_func(self,
solver_mode="sgd",
cost_factors=[],
use_acc_mode=False,
skip_build=False):
#arguments to function
logging.info(
"Building training functions - solver: %s, use_acc_mode: %s" %
(solver_mode, use_acc_mode))
iteration = tensor.fscalar()
learn_rate = tensor.fscalar()
momentum = tensor.fvector()
decay = tensor.fscalar()
#find costs
self.yt = []
self.cost_list = []
self.cost_layers = []
self.cost_layer_names = []
for layer in self.layers:
yt_index = tensor.lvector("target index %i" %
len(self.cost_layers))
yt_value = tensor.fvector("target value %i" %
len(self.cost_layers))
cost = layer.cost(yt_index, yt_value)
if not cost is None:
self.yt += [yt_index, yt_value]
self.cost_list.append(cost)
self.cost_layers.append(layer)
self.cost_layer_names.append(layer.type_name)
self.cost_factors = [1.0] * len(self.cost_list) if len(
cost_factors) == 0 else cost_factors
assert len(self.cost_factors) == len(
self.cost_list
), "Different number of cost factors (%i) and cost layers (%i)" % (len(
self.cost_factors), len(self.cost_layers))
logging.info("Found %i costs in model:" % len(self.cost_layers),
list(zip(self.cost_layer_names, self.cost_factors)))
self.train_cost = tensor.as_tensor_variable(0)
for i, cost in enumerate(self.cost_list):
self.train_cost += self.cost_factors[i] * cost
if self.gradient_clip > 0.0:
logging.info("Clipping gradient to [%f,%f]" %
(-self.gradient_clip, self.gradient_clip))
self.train_cost = theano.gradient.grad_clip(
self.train_cost, -self.gradient_clip, self.gradient_clip)
#find split points
split_points = [0]
self.use_split_mode = False
for index, layer in enumerate(self.layers):
if layer.has_split:
self.use_split_mode = True
split_points.append(index)
split_points.append(len(self.layers))
if self.use_split_mode:
logging.verbose("Using split mode with split points:",
split_points)
self.func["train_fwd"] = []
self.func["train_bwd"] = []
self.updates = []
for sp in range(len(split_points) - 1):
logging.info("Building training functions for layers %i-%i" %
(split_points[sp], split_points[sp + 1]))
split_start = self.layers[split_points[sp]] if sp > 0 else None
split_end = self.layers[split_points[sp + 1]] if (
sp + 2) < len(split_points) else None
split_cost = self.train_cost if split_end is None else None
split_layers = []
for i, layer in enumerate(self.layers):
if (i > split_points[sp]) and (i < split_points[sp + 1]):
split_layers.append(layer)
#determine known_grads provided by previous backward passes
from collections import OrderedDict
split_known_grads = OrderedDict()
for i in range(sp + 1, len(split_points) - 1):
split_known_grads.update(
self.layers[split_points[i]].split_known_grads())
if len(split_known_grads) == 0:
split_known_grads = None
# print(split_known_grads)
# print(split_known_grads)
# print(sp+1, len(split_points)-1)
#
def get_sgd_updates(p, g):
m = theano.shared(numpy.zeros(p.shape.eval(),
dtype=theano.config.floatX),
broadcastable=p.broadcastable,
borrow=True)
rho = tensor.switch(tensor.gt(iteration, 0), momentum[0], 0.0)
m_update = rho * m + (1.0 - rho) * g
p_update = p - learn_rate * m_update
return [(p, p_update), (m, m_update)]
def get_torch_updates(p, g):
m = theano.shared(numpy.zeros(p.shape.eval(),
dtype=theano.config.floatX),
broadcastable=p.broadcastable,
borrow=True)
rho = tensor.switch(tensor.gt(iteration, 0), momentum[0], 0.0)
m_update = rho * m + g
p_update = p - learn_rate * (g + momentum[0] * m_update)
return [(p, p_update), (m, m_update)]
def get_adam_updates(p, g):
eps = 1e-8
m = theano.shared(numpy.zeros(p.shape.eval(),
dtype=theano.config.floatX),
broadcastable=p.broadcastable,
borrow=True)
v = theano.shared(numpy.zeros(p.shape.eval(),
dtype=theano.config.floatX),
broadcastable=p.broadcastable,
borrow=True)
m_update = momentum[0] * m + (1.0 - momentum[0]) * g
v_update = momentum[1] * v + (1.0 - momentum[1]) * (g * g)
m_hat = m_update / (1.0 -
tensor.pow(momentum[0], iteration + 1))
v_hat = v_update / (1.0 -
tensor.pow(momentum[1], iteration + 1))
p_update = p - learn_rate * m_hat / (tensor.sqrt(v_hat) + eps)
return [(p, p_update), (m, m_update), (v, v_update)]
#append parameter updates
params = []
params_decay = []
for layer in split_layers:
params += layer.weights()
params_decay += [True] * len(layer.weights())
params += layer.biases()
params_decay += [False] * len(layer.biases())
#build updates
print("known grads:", split_known_grads)
grads = tensor.grad(split_cost,
params,
known_grads=split_known_grads)
solver_updates = []
for p, g, p_decay in zip(params, grads, params_decay):
#add L2 weight decay if needed
if p_decay or self.bias_decay:
g += decay * p
if solver_mode == "adam":
solver_updates += get_adam_updates(p, g)
elif solver_mode == "torch" or solver_mode == "nesterov":
solver_updates += get_torch_updates(p, g)
else:
solver_updates += get_sgd_updates(p, g)
#append per layer updates
local_updates = solver_updates + sum(
[layer.updates(self.train_cost) for layer in split_layers], [])
#all updates
self.updates += local_updates
#skipping actual theano function building (if you just want updates, etc)
if skip_build:
continue
global debug_train
if debug_train:
logging.warning("WARNING: Debug mode is active!")
from theano.compile.nanguardmode import NanGuardMode
debug_mode = theano.compile.MonitorMode(
post_func=debug_detect_errors)
else:
debug_mode = None
if self.use_split_mode:
if not split_end is None:
updates = sum(
[layer.split_forward() for layer in split_layers], [])
updates += split_end.split_forward()
print("fwd updates:", updates)
f = theano.function([self.input], [],
updates=updates,
givens=[(denet.layer.get_train(),
tensor.cast(1, 'int8'))],
on_unused_input='ignore',
mode=debug_mode)
self.func["train_fwd"].append(f)
outputs = ([self.train_cost] +
self.cost_list) if split_end is None else []
updates = sum([
layer.split_backward(split_cost, split_known_grads)
for layer in split_layers
], [])
if not split_start is None:
updates += split_start.split_backward(
split_cost, split_known_grads)
print("bwd updates:", updates)
updates += local_updates
f = theano.function([
denet.layer.get_epoch(), iteration, learn_rate, momentum,
decay, self.input
] + self.yt,
outputs,
updates=updates,
givens=[(denet.layer.get_train(),
tensor.cast(1, 'int8'))],
on_unused_input='ignore',
mode=debug_mode)
self.func["train_bwd"].insert(0, f)
elif use_acc_mode:
acc_counter = theano.shared(
numpy.array(0, dtype=theano.config.floatX))
begin_updates = [(acc_counter, tensor.zeros_like(acc_counter))]
step_updates = [(acc_counter, acc_counter + 1)]
end_updates = []
self.acc_params = []
for p_dest, p_src in self.updates:
p_acc = theano.shared(numpy.zeros(
p_dest.shape.eval(), dtype=theano.config.floatX),
broadcastable=p_dest.broadcastable,
borrow=True)
begin_updates.append((p_acc, tensor.zeros_like(p_acc)))
step_updates.append((p_acc, p_acc + p_src))
end_updates.append((p_dest, p_acc / acc_counter))
self.acc_params.append(p_acc)
logging.info(
"Constructing parameter accumulate update functions (solver=%s)"
% solver_mode)
self.func["train_begin"] = theano.function(
[], [], updates=begin_updates)
self.func["train_step"] = theano.function(
[
denet.layer.get_epoch(), iteration, learn_rate,
momentum, decay, self.input
] + self.yt, [self.train_cost] + self.cost_list,
updates=step_updates,
givens=[(denet.layer.get_train(), tensor.cast(1, 'int8'))],
on_unused_input='ignore',
allow_input_downcast=True,
mode=debug_mode)
self.func["train_end"] = theano.function([], [],
updates=end_updates)
else:
logging.info(
"Constructing parameter update function (solver=%s)" %
solver_mode)
#making
f_input = theano.In(self.input, borrow=True)
f_yt = [theano.In(yt, borrow=True) for yt in self.yt]
self.func["train_step"] = theano.function(
[
denet.layer.get_epoch(), iteration, learn_rate,
momentum, decay, f_input
] + f_yt, [self.train_cost] + self.cost_list,
updates=self.updates,
givens=[(denet.layer.get_train(), tensor.cast(1, 'int8'))],
on_unused_input='ignore',
allow_input_downcast=True,
mode=debug_mode)
logging.verbose("Exporting graph...")
with open("graph.txt", "w") as f:
theano.printing.debugprint(self.func["train_step"],
file=f,
print_type=True)
def run_worker(gpu, args, data_shape, class_labels, class_num, task_queue,
active, epoch, learn_rate, cost, timer, data_x, data_y, data_m,
model_update):
#redirect output (unbuffered)
sys.stdout = open(gpu + ".out", 'w')
sys.stderr = open(gpu + ".err", 'w')
logging.init(args, flush=True)
sys.setrecursionlimit(10000)
#create thread to flush stdout / stderr every 5 seconds
flush_logs()
logging.info(gpu + ": initializing")
#remove all openmpi variables!
for v in os.environ.keys():
if v[:5] == "OMPI_":
del os.environ[v]
#set compile dir and gpu (possible since theano hasn't been imported yet!)
if not "THEANO_FLAGS" in os.environ:
os.environ["THEANO_FLAGS"] = ""
import socket
os.environ["THEANO_FLAGS"] += "," + args.theano_flags + ","
os.environ["THEANO_FLAGS"] += "device=" + gpu + ","
os.environ["THEANO_FLAGS"] += "force_device=True,"
os.environ["THEANO_FLAGS"] += "compiledir=~/.theano/" + socket.gethostname(
) + "-" + gpu + "/,"
#os.environ["THEANO_FLAGS"] += "lib.cnmem=1,";
os.environ["THEANO_FLAGS"] += "nvcc.flags=-D_FORCE_INLINES,"
logging.info(gpu + ": Using THEANO_FLAGS:", os.environ["THEANO_FLAGS"])
#initialize local model
import denet.model.model_cnn as model_cnn
model = model_cnn.initialize(args, data_shape, class_labels, class_num)
#pre-initialize training function
use_acc_mode = args.batch_size_factor > 1 and args.use_acc_mode
model.build_train_func(args.solver,
args.cost_factors,
use_acc_mode=use_acc_mode)
if use_acc_mode:
train_begin_func = model.func["train_begin"]
train_end_func = model.func["train_end"]
#begin processing loop
iteration = 0
while (True):
#try to start next task immediately otherwise wait for task
wait_time = time.time()
try:
task = task_queue.get(block=False)
except queue.Empty:
logging.verbose(gpu + ": waiting for task")
with active.get_lock():
active.value = 0
task = task_queue.get(block=True)
wait_time = time.time() - wait_time
logging.verbose(gpu + ": " + task + " (wait time=%i ms)" %
(1000 * wait_time))
#calculate updates
ts = time.time()
if task == "predict":
with data_x.lock, data_y.lock:
data_y.get_array()[...] = model.predict_output_step(
data_x.get_array())
elif task == "model-read":
model_update.import_updates(model)
elif task == "model-write":
model_update.export_updates(model)
elif task == "train-begin":
if use_acc_mode:
train_begin_func()
with cost.get_lock():
cost.value = 0
elif task == "train-step":
with cost.get_lock(), epoch.get_lock(), learn_rate.get_lock(
), data_x.lock:
data_meta = data_m.get(block=True)
c, _ = model.train_step(data_x.get_array(), data_meta,
epoch.value, iteration,
learn_rate.value, args.learn_momentum,
args.learn_decay)
if math.isnan(c):
raise Exception("Encountered NaN cost for worker")
cost.value += c
iteration += 1
elif task == "train-end":
if use_acc_mode:
train_end_func()
with cost.get_lock():
cost.value /= args.batch_size_factor
elif task == "done":
exit(0)
with timer.get_lock():
timer.value = time.time() - ts
logging.info(gpu + ": %s took %i ms" % (task, 1000 * timer.value))
def __init__(self, layers, filter_shape=None, stride = (1,1), bottleneck=0, activation = "relu", version="original", json_param={}):
super().__init__(layer_index=len(layers))
self.input = layers[-1].output
self.input_shape = layers[-1].output_shape
#get parameters
self.filter_shape = json_param.get("shape", filter_shape)
self.stride = json_param.get("stride", stride)
self.bottleneck = json_param.get("bottleneck", bottleneck)
self.version = json_param.get("version", version)
self.activation = json_param.get("activation", activation)
self.bn_json_param = json_param.get("bnParam", {
"enabled":json_param.get("enableBatchNorm", True),
})
#determing convolution shapes
if self.bottleneck > 0:
self.size = (self.filter_shape[2], self.filter_shape[3])
shape0 = (self.bottleneck, self.filter_shape[1], 1, 1)
shape1 = (self.bottleneck, self.bottleneck, self.filter_shape[2], self.filter_shape[3])
shape2 = (self.filter_shape[0], self.bottleneck, 1, 1)
else:
self.size = (self.filter_shape[2]*2 - 1, self.filter_shape[3]*2 - 1)
shape0 = self.filter_shape
shape1 = (self.filter_shape[0], self.filter_shape[0], self.filter_shape[2], self.filter_shape[3])
shape2 = None
logging.verbose("Adding", self)
# logging.verbose("Adding", self, "layer - input:", self.input_shape, "shape:", self.filter_shape, "stride:", self.stride, "bottleneck:", self.bottleneck,
# "activation:", self.activation, "version:", self.version, "bn param:", self.bn_json_param)
logging.verbose("--------RESNET BLOCK----------")
self.layers = [InitialLayer(self.input, self.input_shape)]
if "pre-activation" in self.version:
if "bnrelu" in self.version and self.activation == "relu":
self.layers.append(BatchNormReluLayer(self.layers, json_param = self.bn_json_param))
else:
self.layers.append(BatchNormLayer(self.layers, json_param = self.bn_json_param))
self.layers.append(ActivationLayer(self.layers, self.activation))
self.layers.append(ConvLayer(self.layers, filter_shape = shape0, filter_stride=self.stride, border_mode="half", use_bias=False))
if "bnrelu" in self.version and self.activation == "relu":
self.layers.append(BatchNormReluLayer(self.layers, json_param = self.bn_json_param))
else:
self.layers.append(BatchNormLayer(self.layers, json_param = self.bn_json_param))
self.layers.append(ActivationLayer(self.layers, self.activation))
self.layers.append(ConvLayer(self.layers, filter_shape = shape1, border_mode="half", use_bias=False))
#for bottleneck design add additional conv
if self.bottleneck > 0:
if "bnrelu" in self.version and self.activation == "relu":
self.layers.append(BatchNormReluLayer(self.layers, json_param = self.bn_json_param))
else:
self.layers.append(BatchNormLayer(self.layers, json_param = self.bn_json_param))
self.layers.append(ActivationLayer(self.layers, self.activation))
self.layers.append(ConvLayer(self.layers, filter_shape = shape2, border_mode="half", use_bias=False))
if not "pre-activation" in self.version:
self.layers.append(BatchNormLayer(self.layers, json_param = self.bn_json_param))
y = self.layers[-1].output
y_shape = self.layers[-1].output_shape
#project input shape to output shape dimensions
if self.input_shape != y_shape:
logging.verbose("---------SHORTCUT----------")
#handle resnet models with batchnorm in shortcut route
if "pre-activation" in self.version:
input_layers = self.layers[0:2]
else:
input_layers = [InitialLayer(self.input, self.input_shape)]
self.layers.append(ConvLayer(input_layers, filter_shape=(y_shape[1], self.input_shape[1], 1, 1), filter_stride=self.stride, use_bias=False, border_mode="half"))
#original model has batch norm after shortcut
if "original" in self.version:
self.layers.append(BatchNormLayer(self.layers, json_param = self.bn_json_param))
x = self.layers[-1].output
else:
x = self.input
logging.verbose("------------------------------")
#add residual
self.output_shape = y_shape
if "pre-activation" in self.version:
self.output = x + y
else:
self.output = ActivationLayer.apply(x + y, self.activation)
def __init__(self,
layers,
filter_shape=None,
filter_stride=(1, 1),
use_bias=True,
border_mode="valid",
wb="he-backward",
json_param={}):
super().__init__(layer_index=len(layers))
self.input = layers[-1].output
self.input_shape = layers[-1].output_shape
#get parameters
self.border_mode = json_param.get("border", border_mode)
self.filter_shape = tuple(json_param.get("shape", filter_shape))
self.stride = tuple(json_param.get("stride", filter_stride))
self.use_bias = json_param.get("useBias", use_bias)
self.size = (self.filter_shape[2], self.filter_shape[3])
#use initialization
if type(wb) is float:
self.w_bound = float(wb)
elif "he-forward" in wb:
self.w_bound = math.sqrt(
2.0 / (self.filter_shape[2] * self.filter_shape[3] *
self.filter_shape[1]))
elif "he-backward" in wb:
self.w_bound = math.sqrt(
2.0 / (self.filter_shape[2] * self.filter_shape[3] *
self.filter_shape[0]))
elif "xavier-forward" in wb:
self.w_bound = math.sqrt(
1.0 / (self.filter_shape[2] * self.filter_shape[3] *
self.filter_shape[1]))
elif "xavier-backward" in wb:
self.w_bound = math.sqrt(
1.0 / (self.filter_shape[2] * self.filter_shape[3] *
self.filter_shape[0]))
#initialize weights
if self.w_bound > 0:
if "uniform" in wb:
w = numpy.random.uniform(-self.w_bound,
self.w_bound,
size=self.filter_shape)
else:
w = numpy.random.normal(0.0,
self.w_bound,
size=self.filter_shape)
else:
w = numpy.zeros(shape=self.filter_shape)
self.omega = theano.shared(numpy.asarray(w,
dtype=theano.config.floatX),
name="deconv omega")
#initialize bias
if self.use_bias:
self.beta = theano.shared(value=numpy.zeros(
(self.filter_shape[0], ), dtype=theano.config.floatX),
name="deconv beta")
#calculate output shape
if self.border_mode == "half":
fh = self.filter_shape[2] // 2
fw = self.filter_shape[3] // 2
h = self.input_shape[2] * self.stride[
0] - 2 * fh + self.filter_shape[2] - 1
w = self.input_shape[3] * self.stride[
1] - 2 * fw + self.filter_shape[3] - 1
else:
raise Exception("Unknown border mode: " + str(self.border_mode))
self.output_shape = (self.input_shape[0], self.filter_shape[0], h, w)
self.output = conv2d_grad_wrt_inputs(
self.input, self.omega.dimshuffle((1, 0, 2, 3)), self.output_shape,
(self.filter_shape[1], self.filter_shape[0], self.filter_shape[2],
self.filter_shape[3]), self.border_mode, self.stride)
if self.use_bias:
self.output += self.beta[None, :, None, None]
logging.verbose("Adding", self)
def __init__(self,
layers,
crop_size=None,
mirror_pr=0.0,
flip_pr=0.0,
json_param={}):
super().__init__(layer_index=len(layers))
self.input = layers[-1].output
self.input_shape = layers[-1].output_shape
self.crop_size = json_param.get("crop", crop_size)
self.mirror_pr = json_param.get("mirror", mirror_pr)
self.flip_pr = json_param.get("flip", flip_pr)
self.output_shape = (self.input_shape[0], self.input_shape[1],
self.crop_size[0], self.crop_size[1])
self.output = []
zero = tensor.zeros((self.input_shape[0], ), dtype=numpy.int8)
index_b = tensor.arange(self.input_shape[0])
index_c = tensor.arange(self.input_shape[1])
index_x = tensor.arange(self.crop_size[0])[None, :]
index_y = tensor.arange(self.crop_size[1])[None, :]
#randomly mirror (y-axis) data
if self.mirror_pr > 0.0:
mirror = tensor.gt(get_rng().uniform(size=(self.input_shape[0], )),
1.0 - self.mirror_pr)
mirror = tensor.switch(get_train(), mirror, zero)
index_y = tensor.switch(mirror[:, None],
-index_y + self.crop_size[1] - 1, index_y)
#randomly flip (x-axis) data
if self.flip_pr > 0.0:
flip = tensor.gt(get_rng().uniform(size=(self.input_shape[0], )),
1.0 - self.flip_pr)
flip = tensor.switch(get_train(), flip, zero)
index_x = tensor.switch(flip[:, None],
-index_x + self.crop_size[0] - 1, index_x)
#randomly offset crop
dx = self.input_shape[2] - self.crop_size[0]
dy = self.input_shape[3] - self.crop_size[1]
if self.crop_size[0] != self.input_shape[2] or self.crop_size[
1] != self.input_shape[3]:
center_x = theano.shared(numpy.full(shape=(self.input_shape[0], ),
fill_value=dx // 2,
dtype=numpy.int32),
borrow=False)
center_y = theano.shared(numpy.full(shape=(self.input_shape[0], ),
fill_value=dy // 2,
dtype=numpy.int32),
borrow=False)
offset_x = get_rng().random_integers(size=(self.input_shape[0], ),
low=0,
high=dx)
offset_y = get_rng().random_integers(size=(self.input_shape[0], ),
low=0,
high=dy)
index_x += tensor.switch(get_train(), offset_x, center_x)[:, None]
index_y += tensor.switch(get_train(), offset_y, center_y)[:, None]
#perform advanced indexing
self.output = self.input[index_b[:, None, None,
None], index_c[None, :, None, None],
index_x[:, None, :, None], index_y[:, None,
None, :]]
logging.verbose("Adding", self)
def __init__(self,
layers,
filter_shape=None,
filter_stride=(1, 1),
use_bias=False,
border_mode="half",
wb="he-backward",
json_param={}):
super().__init__(layer_index=len(layers))
self.input = layers[-1].output
self.input_shape = layers[-1].output_shape
#get parameters
self.border_mode = json_param.get("border", border_mode)
self.filter_shape = tuple(json_param.get("shape", filter_shape))
self.stride = tuple(json_param.get("stride", filter_stride))
self.use_bias = json_param.get("useBias", use_bias)
self.size = (self.filter_shape[2], self.filter_shape[3])
self.enabled = json_param.get("enabled", True)
#use initialization
if type(wb) is float:
self.w_bound = float(wb)
elif "he-forward" in wb:
self.w_bound = math.sqrt(
2.0 / (self.filter_shape[2] * self.filter_shape[3] *
self.filter_shape[1]))
elif "he-backward" in wb:
self.w_bound = math.sqrt(
2.0 / (self.filter_shape[2] * self.filter_shape[3] *
self.filter_shape[0]))
elif "xavier-forward" in wb:
self.w_bound = math.sqrt(
1.0 / (self.filter_shape[2] * self.filter_shape[3] *
self.filter_shape[1]))
elif "xavier-backward" in wb:
self.w_bound = math.sqrt(
1.0 / (self.filter_shape[2] * self.filter_shape[3] *
self.filter_shape[0]))
#initialize weights
if self.w_bound > 0:
if "uniform" in wb:
w = numpy.random.uniform(-self.w_bound,
self.w_bound,
size=self.filter_shape)
else:
w = numpy.random.normal(0.0,
self.w_bound,
size=self.filter_shape)
else:
w = numpy.zeros(shape=self.filter_shape)
self.omega = theano.shared(numpy.asarray(w,
dtype=theano.config.floatX),
name="conv omega")
#initialize bias
if self.use_bias:
self.beta = theano.shared(value=numpy.zeros(
(self.filter_shape[0], ), dtype=theano.config.floatX),
name="conv beta")
#calculate output shape
if self.border_mode == 'valid':
w = math.ceil((self.input_shape[-2] - self.filter_shape[2] + 1) /
self.stride[0])
h = math.ceil((self.input_shape[-1] - self.filter_shape[3] + 1) /
self.stride[1])
elif self.border_mode == "full":
w = math.ceil((self.input_shape[-2] + self.filter_shape[2] - 1) /
self.stride[0])
h = math.ceil((self.input_shape[-1] + self.filter_shape[3] - 1) /
self.stride[1])
elif self.border_mode == "half":
fh = self.filter_shape[2] // 2
fw = self.filter_shape[3] // 2
w = math.ceil(
(self.input_shape[-2] + 2 * fh - self.filter_shape[2] + 1) /
self.stride[0])
h = math.ceil(
(self.input_shape[-1] + 2 * fw - self.filter_shape[3] + 1) /
self.stride[1])
elif self.border_mode == "same":
assert self.stride == (1, 1)
w = self.input_shape[-2]
h = self.input_shape[-1]
elif isinstance(self.border_mode, int):
w = math.ceil((self.input_shape[-2] + 2 * self.border_mode -
self.filter_shape[2] + 1) / self.stride[0])
h = math.ceil((self.input_shape[-1] + 2 * self.border_mode -
self.filter_shape[3] + 1) / self.stride[1])
else:
raise Exception("Unknown border mode: " + str(self.border_mode))
#handle "same" mode specially
if self.border_mode == "same":
y0 = (self.filter_shape[2] - 1) // 2
x0 = (self.filter_shape[3] - 1) // 2
y = tensor.nnet.conv2d(input=self.input,
filters=self.omega,
filter_shape=self.filter_shape,
subsample=self.stride,
input_shape=self.input_shape,
border_mode="full")[:, :, y0:(y0 + h),
x0:(x0 + w)]
else:
# y = tensor.nnet.conv2d(input=self.input, filters=self.omega, filter_shape=self.filter_shape, subsample=self.stride, input_shape=self.input_shape, border_mode=self.border_mode)
y = tensor.nnet.conv2d(input=self.input,
filters=self.omega,
filter_shape=self.filter_shape,
subsample=self.stride,
border_mode=self.border_mode)
y_shape = (self.input_shape[0], self.filter_shape[0], w, h)
self.output_unbiased = y
if self.use_bias:
y += self.beta[None, :, None, None]
self.output_shape = y_shape
self.output = y
logging.verbose("Adding", self)
