def load_from_file(fname, batch_size=32, layer_range=None): t = time.time() logging.info("Loading model from %s" % fname) model = load_from_json(common.json_from_gz(fname), batch_size, layer_range) model.fname = fname logging.verbose("Loading model took %.2f sec" % (time.time() - t)) return model
def predict_output(self, dataset): dataset_x, dataset_y, dataset_size = dataset.export(self.batch_size) #dummy call to build function self.predict_output_step(dataset_x[:self.batch_size]) #evaluate function timer = common.Timer() n = math.ceil(dataset_size / self.batch_size) pr = [] for index in range(n): data_x = dataset_x[index * self.batch_size:(index + 1) * self.batch_size] pr_batch = self.predict_output_step(data_x) pr.append(pr_batch) pr = numpy.concatenate(pr, axis=0) logging.verbose("Prediction took %.3f sec for %i samples" % (timer.current(), pr.shape[0])) #crop dummy data if (dataset_size % self.batch_size) != 0: s = [dataset_size] + list(pr.shape[1:]) pr.resize(tuple(s), refcheck=False) return pr
def __init__(self, layers, size=(2, 2), stride=None, pad=(0, 0), mode="max", ignore_border=True, json_param={}): super().__init__(layer_index=len(layers)) self.input = layers[-1].output self.input_shape = layers[-1].output_shape self.size = json_param.get("size", size) self.pad = json_param.get("pad", pad) self.ignore_border = json_param.get("ignoreBorder", ignore_border) self.mode = json_param.get("mode", mode) self.stride = json_param.get("stride", stride) if self.stride is None: self.stride = self.size #output dim if self.ignore_border: h = int( math.floor( (self.input_shape[2] + 2 * self.pad[0] - self.size[0]) / self.stride[0])) + 1 w = int( math.floor( (self.input_shape[3] + 2 * self.pad[1] - self.size[1]) / self.stride[1])) + 1 else: h = int( math.ceil( (self.input_shape[2] + 2 * self.pad[0]) / self.stride[0])) w = int( math.ceil( (self.input_shape[3] + 2 * self.pad[1]) / self.stride[1])) #theano optimizer is sometimes failing to use cudnn pooling! use_cudnn = (dnn.dnn_available() and dnn.version() >= (4000, 4000) and self.ignore_border) if use_cudnn: self.output = dnn.dnn_pool(self.input, ws=self.size, pad=self.pad, stride=self.stride, mode=self.mode) else: self.output = tensor.signal.pool.pool_2d( self.input, ds=self.size, padding=self.pad, ignore_border=self.ignore_border, st=self.stride, mode=self.mode) self.output_shape = (self.input_shape[0], self.input_shape[1], h, w) logging.verbose("Adding", self)
def __init__(self, layers, sample_feat=512, cost_factor=1, dropout=0.0, use_center=False, json_param={}): super().__init__(layer_index=len(layers)) #pass though layer self.input = layers[-1].output self.input_shape = layers[-1].output_shape self.output = layers[-1].output self.output_shape = layers[-1].output_shape self.batch_size, self.features, self.height, self.width = self.input_shape #get param self.sample_feat = json_param.get("sampleFeat", sample_feat) self.cost_factor = json_param.get("costFactor", cost_factor) self.use_center = json_param.get("useCenter", use_center) self.dropout = json_param.get("dropout", dropout) self.corner_num = 5 if self.use_center else 4 self.layers = [InitialLayer(self.input, self.input_shape)] self.layers.append( ConvLayer( self.layers, (self.corner_num + self.sample_feat, self.features, 1, 1), (1, 1), True, False)) #initialize corner_pr weights / biases omega = self.layers[-1].omega.get_value() omega[:self.corner_num, :, :, :] = 0.0 self.layers[-1].omega.set_value(omega) beta = self.layers[-1].beta.get_value() beta[:self.corner_num] = 5.0 self.layers[-1].beta.set_value(beta) #extract corner probabilities self.corner_shape = (self.batch_size, 2, self.corner_num, self.height, self.width) self.corner_lh = self.layers[-1].output[:, :self.corner_num, :, :] self.corner_lh = tensor.concatenate( [self.corner_lh[:, None, ...], -self.corner_lh[:, None, ...]], axis=1) self.corner_pr = theano_util.log_softmax(self.corner_lh, axis=[1]) #extract sample self.sample_shape = (self.batch_size, self.sample_feat, self.height, self.width) self.sample_shared = theano.shared( numpy.zeros(self.sample_shape, dtype=numpy.float32), "shared sample") self.sample = self.layers[-1].output[:, self.corner_num:, :, :] logging.verbose("Adding", self)
def __init__(self, layers, class_num=10, overlap_threshold=0.5, cost_factor=1.0, bbox_factor=0.0, json_param={}): super().__init__(layer_index=len(layers)) #passthrough layer self.input = layers[-1].output self.input_shape = layers[-1].output_shape self.output = layers[-1].output self.output_shape = layers[-1].output_shape self.cost_factor = json_param.get("costFactor", cost_factor) self.bbox_factor = json_param.get("bboxFactor", bbox_factor) self.class_num = json_param.get("classNum", class_num) self.overlap_threshold = json_param.get("overlapThreshold", overlap_threshold) #find sparse / instcount / compare layer self.sparse_layer = common.find_layers(layers, "denet-sparse", False) assert self.sparse_layer != None, "Error: Requires denet-sparse layer to be specified before denet-detect layer!" self.use_bbox_reg = (self.bbox_factor > 0.0) self.batch_size = self.sparse_layer.batch_size self.sample_num = self.sparse_layer.sample_num self.null_class = self.class_num # s0 = self.class_num + 1 s1 = 4 if self.use_bbox_reg else 0 self.layers = [ ConvLayer([InitialLayer(self.input, self.input_shape)], (s0 + s1, self.input_shape[1], 1, 1), (1, 1), True, "valid", 0.0) ] #class assignment log(Pr({c,null} | sample_j, sample_i)) self.det_shape = (self.batch_size, s0, self.sample_num, self.sample_num) self.det_lh = self.layers[-1].output[:, :s0, ...] self.det_pr = theano_util.log_softmax(self.det_lh, axis=[1]) #bbox regression if self.use_bbox_reg: self.bbox_shape = (self.batch_size, s1, self.sample_num, self.sample_num) self.bbox_reg = self.layers[-1].output[:, s0:(s0 + s1), ...] self.detect_func = None self.nms_func = None logging.verbose("Adding", self)
def __init__(self, layers, activation="relu", json_param={}): super().__init__(layer_index=len(layers)) self.input = layers[-1].output self.input_shape = layers[-1].output_shape self.activation=json_param.get("activation", activation) self.output_shape = self.input_shape self.output = ActivationLayer.apply(self.input, self.activation) #apply activation logging.verbose("Adding", self)
def __init__(self, layers, json_param={}): super().__init__(layer_index=len(layers)) self.enabled = json_param.get("enabled", True) self.has_split = self.enabled self.input = layers[-1].output self.output_shape = self.input_shape = layers[-1].output_shape if self.enabled: self.output = theano.shared(numpy.zeros(self.output_shape).astype(theano.config.floatX), str(self) + " - output") else: self.output = self.input logging.verbose("Adding", self, "layer - input:", self.input_shape, "enabled:", self.enabled)
def __init__(self, layers, skip_index=0, split=False, json_param={}): super().__init__(layer_index=len(layers)) self.skip_index = json_param.get("index", skip_index) self.has_split = json_param.get("split", split) self.input = layers[-1].output self.input_shape = layers[-1].output_shape self.output_shape = self.input_shape if self.has_split: self.output = theano.shared(numpy.zeros(self.output_shape).astype(theano.config.floatX), str(self) + " - output") self.skip = theano.shared(numpy.zeros(self.output_shape).astype(theano.config.floatX), str(self) + " - skip") else: self.skip = self.output = self.input logging.verbose("Adding", self, "split:", self.has_split)
def get_target(self, model, data_x, metas): sample_bboxs = self.get_samples(data_x, train=True) total_cover = 0 total_bbox = 0 for b, meta in enumerate(metas): #compute groundtruth coverage cover = 0 for meta_bbox in meta["bbox"]: for _, sample_bbox in sample_bboxs[b]: if common.overlap_iou(meta_bbox, sample_bbox) > 0.5: cover += 1 break logging.verbose( "%i: corner detector found %i samples (%i/%i coverage)" % (b, len(sample_bboxs[b]), cover, len(meta["bbox"]))) total_cover += cover total_bbox += len(meta["bbox"]) n = self.sample_count - math.floor( self.random_sample * self.sample_count) if len(sample_bboxs[b]) > n: # logging.verbose("%i: removing %i samples to make room for random samples"%(b, len(sample_bboxs[b]) - n)) sample_bboxs[b] = random.sample(sample_bboxs[b], n) #add random samples if bbox detector produces too few while len(sample_bboxs[b]) < self.sample_count: x0 = random.uniform(0.0, 1.0) y0 = random.uniform(0.0, 1.0) x1 = random.uniform(x0, 1.0) y1 = random.uniform(y0, 1.0) bbox = (x0, y0, x1, y1) sample_bboxs[b].append((0.0, bbox)) #insert groundtruth if self.sample_gt: for index, bbox in enumerate(meta["bbox"]): sample_bboxs[b][-(index + 1)] = (1.0, bbox) logging.verbose( "Overall %i/%i (%.2f%%) coverage" % (total_cover, total_bbox, 100.0 * total_cover / total_bbox)) self.set_samples(sample_bboxs) return None
def __init__(self, layers, size=(2, 2), json_param={}): super().__init__(layer_index=len(layers)) self.input = layers[-1].output self.input_shape = layers[-1].output_shape self.size=json_param.get("size", size) #output dim self.output_shape = (self.input_shape[0], self.input_shape[1], self.size[1]*self.input_shape[2], self.size[0]*self.input_shape[3]) self.use_optimized = theano.sandbox.cuda.cuda_enabled if self.use_optimized: self.output = PoolInvOp(self.size)(self.input) else: self.output = tensor.repeat(tensor.repeat(self.input, self.size[1], axis=2), self.size[0], axis=3) logging.verbose("Adding", self)
def build(self, model_desc, data_shape, activation="relu", border_mode="valid", weight_init="he-forward"): self.model_desc = " ".join(model_desc) self.data_shape = data_shape self.layers = [InitialLayer(self.input, self.get_input_shape())] for i, layer_desc in enumerate(model_desc): wb = weight_init[min(len(weight_init) - 1, i)] self.build_layer(layer_desc, self.layers, activation, border_mode, wb) logging.verbose("Number of parameters in model: %d" % self.get_parameter_num())
def get_samples(self, data_x, train=False, store_shared=False): global profile if self.corner_func is None: logging.verbose("Building corner function - store samples:", store_shared, "train:", train) updates = [(self.corner_layer.sample_shared, self.corner_layer.sample)] if store_shared else [] self.corner_func = theano.function( [self.model_input], self.corner_layer.corner_pr, updates=updates, profile=profile, givens=[(get_train(), tensor.cast(int(train), 'int8'))], on_unused_input='ignore') #find corners timer = common.Timer() logging.debug("Running corner function") corner_pr = self.corner_func(data_x) if profile: logging.debug("Profiling corner function") theano_util.profile(self.corner_func, 10, data_x) theano_util.export_graph("./corner.graph", self.corner_func) logging.debug("Done") exit(0) #build sampling bounding boxs timer.mark() logging.debug("Build samples (%i threads)" % self.thread_num) samples = c_code.build_samples(self.thread_num, corner_pr, self.corner_threshold, self.sample_num, self.corner_max, self.local_max, self.nms_threshold) timer.mark() logging.verbose( "Took %i ms to get_samples (%i model, %i build, %i max corners) " % (timer.current_ms(), timer.delta_ms(0), timer.delta_ms(1), self.corner_max)) return samples
def __init__(self, layers, dropout_rate=1.0, json_param={}): super().__init__(layer_index=len(layers)) self.input = layers[-1].output self.input_shape = layers[-1].output_shape #get parameters self.dropout_rate = json_param.get("dropoutRate", dropout_rate) scale = 1.0 / (1.0 - self.dropout_rate) mask = get_rng().binomial(n=1, p=1.0 - self.dropout_rate, size=self.input_shape) mask = tensor.cast(mask, theano.config.floatX) self.output = theano.ifelse.ifelse(get_train(), self.input * mask * scale, self.input) self.output_shape = self.input_shape logging.verbose("Adding", self, "layer - input:", self.input_shape, "dropout: %.0f%%" % (100.0 * self.dropout_rate))
def __init__(self, layers, use_center=True, valid=[], json_param={}): super().__init__(layer_index=len(layers)) self.input = layers[-1].output self.input_shape = layers[-1].output_shape #used for determining multiview map self.multiview_layers = layers if use_center: yc = self.input_shape[-2] // 2 xc = self.input_shape[-1] // 2 valid = [(0,yc,xc)] self.valid = json_param.get("valid", valid) #could be done faster with advanced indexing if len(self.valid) > 0: x_shape = (self.input_shape[0], self.input_shape[1], len(self.valid)) x = tensor.zeros(x_shape) for i, offset in enumerate(self.valid): x = tensor.set_subtensor(x[:,:,i], self.input[:,:,offset[1],offset[2]]) else: x = self.input x_shape = self.input_shape self.log_likelihood = x self.log_pr = self.log_softmax(x, axis=1) self.log_pr_shape = x_shape #return probabilities as output self.output_shape = (self.log_pr_shape[0], self.log_pr_shape[1]) if len(self.log_pr_shape) > 2: self.output = tensor.exp(self.log_pr).mean(axis=range(2, len(self.log_pr_shape))) else: self.output = tensor.exp(self.log_pr) logging.verbose("Adding", self, "layer - input:", self.input_shape, "valid:", self.valid)
def train_epoch(self, dataset, epoch, learning_rate, momentum=[0, 1, 0], decay=0.0, solver_mode="sgd"): #train over batches (assume dataset size is mulitple of batch_size!) logging.info("Evaluating training function") dataset_x, dataset_m, dataset_size = dataset.export(self.batch_size) index_num = math.ceil(dataset_size / self.batch_size) total_cost = 0 for index in range(index_num): #upload data to GPU and perform train step timer = common.Timer() data_x = dataset_x[index * self.batch_size:(index + 1) * self.batch_size] data_m = dataset_m[index * self.batch_size:(index + 1) * self.batch_size] cost, _ = self.train_step(data_x, data_m, epoch, self.iteration, learning_rate, momentum, decay) #watch out for GPU's randomly producing NaN! if math.isnan(cost): raise Exception("ERROR: Cost is NaN") logging.verbose( "Batch %i.%i - iteration: %i cost:" % (epoch, index * self.batch_size, self.iteration), cost, "took: %i ms" % timer.current_ms()) total_cost += cost self.iteration += 1 return total_cost
def __init__(self, layers, skip_index=0, combine_mode="proj-add", json_param={}): super().__init__(layer_index=len(layers)) self.combine_mode = json_param.get("combineMode", combine_mode) self.skip_index = json_param.get("index", skip_index) self.skip_layer=None for layer in layers: if layer.type_name == "skip-src" and layer.skip_index == self.skip_index: self.skip_layer=layer break assert self.skip_layer != None self.x = layers[-1].output self.x_shape = layers[-1].output_shape self.y = self.skip_layer.skip self.y_shape = self.skip_layer.output_shape if self.combine_mode == "proj-add": self.output_shape = self.x_shape if self.y_shape[1] != self.x_shape[1]: self.layers = [InitialLayer(self.y, self.y_shape)] self.layers.append(ConvLayer(self.layers, filter_shape = (self.x_shape[1], self.y_shape[1], 1, 1))) self.output = self.x + self.layers[-1].output else: self.output = self.x + self.y elif self.combine_mode == "concat": self.output_shape = (self.x_shape[0], self.x_shape[1] + self.y_shape[1], self.x_shape[2], self.x_shape[3]) self.output = tensor.concatenate([self.x, self.y], axis=1) else: raise Exception("Unknown combine mode: %s"%self.combine_mode) logging.verbose("Adding", self, "layer - x:", self.x_shape, "y:", self.y_shape, "skip index:", self.skip_index, "combine mode:", self.combine_mode)
def train_step(self, data_x, data_m, epoch, it, learning_rate, momentum, decay): #assert "train" in self.func, "Call build_train_func() before calling train_step()" denet.layer.get_iteration().set_value(it) momentum = numpy.array(momentum, dtype=numpy.float32) #for split mode explicitly perform bwd/fwd propagation if self.use_split_mode: #only get_targets after train_fwd has been performed targets = [] fwd_index = 0 for layer in self.layers: if layer.has_split: logging.verbose("Forward prop %i" % fwd_index) self.func["train_fwd"][fwd_index](data_x) fwd_index += 1 target = layer.get_target(self, data_x, data_m) if not target is None: print(layer.type_name) targets += target # print(targets) logging.verbose("Backward prop %i" % 0) costs = self.func["train_bwd"][0](epoch, it, learning_rate, momentum, decay, data_x, *targets) for i, f_bwd in enumerate(self.func["train_bwd"][1:]): logging.verbose("Backward prop %i" % (i + 1)) f_bwd(epoch, it, learning_rate, momentum, decay, data_x, *targets) else: targets = [] for layer in self.layers: target = layer.get_target(self, data_x, data_m) if not target is None: targets += target costs = self.func["train_step"](epoch, it, learning_rate, momentum, decay, data_x, *targets) return costs[0], costs[1:]
def run_sync(self): #connect to clients server_socket, client_sockets = self.connect_clients() #construct update object for each client / server client_updates = [ shared.ModelUpdate(self.model_dims) for _ in range(self.client_num) ] server_update = shared.ModelUpdate(self.model_dims) #perform synchronization while True: try: logging.info("Waiting for updates...") for i, sock in enumerate(client_sockets): update_json = network.recv_json(sock) client_updates[i].import_json(update_json["data"]) logging.info("Synchronising...") ts = time.time() server_update.set_mean(client_updates, self.thread_num) logging.verbose("mean calc took %.2f sec" % (time.time() - ts)) ts = time.time() server_json = server_update.export_json() logging.verbose("json export took %.2f sec" % (time.time() - ts)) #send mean update to clients ts = time.time() for sock in client_sockets: network.send_json(sock, server_json) logging.verbose("transferring data to clients took %.2f sec" % (time.time() - ts)) except (KeyboardInterrupt, SystemExit): logging.info("Done") sys.exit(0) except Exception as e: logging.error("Encounter exception: ", e)
def __init__(self, layers, grid_size=3, sample_num=16, corner_threshold=0.01, random_sample=0.0, local_max=0, nms_threshold=0.7, sample_gt=True, version="v2", json_param={}): super().__init__(layer_index=len(layers)) self.input = layers[-1].output self.input_shape = layers[-1].output_shape self.batch_size = self.input_shape[0] self.model_input = layers[0].output self.grid_size = json_param.get("gridSize", grid_size) self.sample_num = json_param.get("sampleNum", sample_num) self.sample_gt = json_param.get("sampleGT", sample_gt) self.corner_threshold = json_param.get("cornerThreshold", corner_threshold) self.nms_threshold = json_param.get("nmsThreshold", nms_threshold) self.random_sample = json_param.get("randomSample", random_sample) self.local_max = json_param.get("localMax", local_max) self.version = json_param.get("version", version) self.corner_max = 1024 self.thread_num = self.batch_size self.sample_count = self.sample_num * self.sample_num #find corner layer self.corner_func = None self.corner_layer = common.find_layers(layers, "denet-corner", True) assert not self.corner_layer is None, "denet-corner layer required before spare layer!" #sampling bounding boxs self.sample_bbox_list = [] self.sample_bbox = theano.shared( value=numpy.zeros((self.batch_size, self.sample_num, self.sample_num, 4), dtype=numpy.float32)) self.output_feat = self.grid_size * self.grid_size * self.corner_layer.sample_shape[ 1] + 2 self.output_shape = (self.batch_size, self.output_feat, self.sample_num, self.sample_num) #Use optimized op! sample_input = tensor.switch(get_train(), self.corner_layer.sample, self.corner_layer.sample_shared) self.use_optimized = theano.sandbox.cuda.cuda_enabled if self.use_optimized: self.output = DeNetSparseOp(self.grid_size)(sample_input, self.sample_bbox) else: sample_w = self.sample_bbox[:, :, :, 2] - self.sample_bbox[:, :, :, 0] sample_h = self.sample_bbox[:, :, :, 3] - self.sample_bbox[:, :, :, 1] sample_x = self.sample_bbox[:, :, :, 0, None] + tensor.arange( self.grid_size, dtype='float32')[ None, None, None, :] * sample_w[:, :, :, None] / (self.grid_size - 1) sample_y = self.sample_bbox[:, :, :, 1, None] + tensor.arange( self.grid_size, dtype='float32')[ None, None, None, :] * sample_h[:, :, :, None] / (self.grid_size - 1) batch_range = tensor.arange(self.batch_size, dtype='int64')[:, None, None, None, None] grid_range = tensor.arange(self.grid_size * self.grid_size, dtype='int64').reshape( (self.grid_size, self.grid_size))[None, None, None, :, :] #extract samples b, f, h, w = self.corner_layer.sample_shape #get sample position in feature map (b,sj,si,gy/gx) sample_xf = tensor.maximum(0, tensor.minimum(sample_x * w, w - 1)) sample_yf = tensor.maximum(0, tensor.minimum(sample_y * h, h - 1)) syf = tensor.cast(sample_yf.round(), 'int64') sxf = tensor.cast(sample_xf.round(), 'int64') byxc = batch_range * h * w + syf[:, :, :, :, None] * w + sxf[:, :, :, None, :] #extract sample sample = sample_input.dimshuffle((1, 0, 2, 3)).flatten(2) sample = sample[:, byxc.flatten()] sample = sample.reshape((f, b, self.sample_num, self.sample_num, self.grid_size * self.grid_size)) sample = sample.dimshuffle((1, 2, 3, 4, 0)) sample = sample.reshape((b, self.sample_num, self.sample_num, self.grid_size * self.grid_size * f)) sample = sample.dimshuffle((0, 3, 1, 2)) #add width / height self.output = tensor.concatenate( [sample, sample_h[:, None, :, :], sample_w[:, None, :, :]], axis=1) global c_code logging.verbose("Adding", self)
def load(self, input_dir, data_format, is_training, thread_num, class_labels=None): from .basic import DatasetFromDir self.input_dir = input_dir if self.input_dir[-1] == '/': self.input_dir = self.input_dir[:-1] self.data_format = data_format self.thread_num = thread_num #generate class labels self.class_labels = class_labels fname = os.path.join(os.path.dirname(self.input_dir), "class_labels.txt") if os.path.isfile(fname) and self.class_labels is None: logging.info("Loading class labels from:", fname) self.class_labels = {} with open(fname, "r") as f: for line in f.readlines(): tokens = line.rstrip('\n').split(" ") self.class_labels[tokens[1]] = int(tokens[0]) elif self.class_labels is None: self.class_labels = DatasetFromDir.find_class_labels(input_dir) #check to see if buffered file list is present list_fname = os.path.join(input_dir, "image_list.json") if os.path.isfile(list_fname): logging.info("Loading dataset metadata:", list_fname) json_data = common.json_from_file(list_fname) if json_data.get("version", 0) < 1: logging.warning( "Warning: image_list.json is old version, missing bounding boxs!" ) self.images = [{ "fname": fname, "bboxs": [] } for fname in json_data["images"]] else: self.images = json_data["images"] else: bbox_dir = os.path.join(os.path.dirname(input_dir), "bbox") if not os.path.isdir(bbox_dir): raise Exception("ERROR: cannot find bbox dir:" + bbox_dir) fnames = [] for i, c in enumerate(os.listdir(input_dir).sort()): images_cls = DatasetFromDir.find_paths( os.path.join(input_dir, c), "*.JPEG") logging.info("Found %i images for class" % len(images_cls), c) fnames += images_cls logging.info("Finding bboxs in:", bbox_dir) self.images = [] for i, fname in enumerate(fnames): logging.verbose("%i/%i" % (i, len(fnames))) cls_name = os.path.basename(os.path.dirname(fname)) obj_fname = os.path.join( bbox_dir, cls_name, os.path.splitext(os.path.basename(fname))[0] + ".xml") bboxs = [] if os.path.isfile(obj_fname): obj_tree = xml.parse(obj_fname).getroot() size = obj_tree.find("size") width = int(size.find("width").text) height = int(size.find("height").text) for obj in obj_tree.iter("object"): bndbox = obj.find("bndbox") min_x = int(bndbox.find("xmin").text) min_y = int(bndbox.find("ymin").text) max_x = int(bndbox.find("xmax").text) max_y = int(bndbox.find("ymax").text) bboxs.append({ "x0": min_x, "x1": max_x, "y0": min_y, "y1": max_y }) self.images.append({"fname": fname, "bboxs": bboxs}) try: logging.info("Saving dataset metadata:", list_fname) common.json_to_file(list_fname, { "images": self.images, "version": 1 }) except Exception as e: logging.warning( "Warning: failed to write buffered image list - ", e) #add/fix fields to fit new image_loader interface for image in self.images: fname = image["fname"] cls = self.class_labels[os.path.basename(os.path.dirname(fname))] image["class"] = cls image["bboxs"] = [(cls, (bb["x0"], bb["y0"], bb["x1"], bb["y1"])) for bb in image["bboxs"]] param_str = ",".join(data_format.split(",")[1:]) format_params = common.get_params_dict(param_str) self.image_loader = ImageLoader(thread_num, is_training, format_params) #from facebook resnet implementation self.image_loader.rgb_mean = numpy.array([0.485, 0.456, 0.406], dtype=numpy.float32) self.image_loader.rgb_std = numpy.array([0.229, 0.224, 0.225], dtype=numpy.float32) self.image_loader.rgb_eigen_val = numpy.array([0.2175, 0.0188, 0.0045], dtype=numpy.float32) self.image_loader.rgb_eigen_vec = numpy.array( [[-0.5675, 0.7192, 0.4009], [-0.5808, -0.0045, -0.8140], [-0.5836, -0.6948, 0.4203]], dtype=numpy.float32) #others self.subset_size = format_params.get("images_per_subset", 10000) self.use_null_class = format_params.get("null", False) self.subset_num = format_params.get("subset_num", sys.maxsize) self.bbox_only = format_params.get("bbox_only", False) #only use samples with bounding boxes if self.image_loader.is_training and self.bbox_only: images_bbox = [] for image in self.images: if len(image["bboxs"]) > 0: images_bbox.append(image) self.images = images_bbox #append null class if self.use_null_class and not "null" in self.class_labels: self.class_labels["null"] = len(self.class_labels) self.subset_index = -1 self.subset_total_size = len(self.images) self.subset_num = min( self.subset_num, int(math.ceil(self.subset_total_size / self.subset_size))) logging.info("Using Imagenet dataset - size:", self.subset_total_size, "subset_num", self.subset_num, "images per subset:", self.subset_size, self.image_loader)
def export_graph(fname, func): logging.verbose("Saving function graph: " + fname) with open(fname, "w") as f: theano.printing.debugprint(func, file=f, print_type=True)
def load(self, input_dir, data_format, is_training, thread_num, class_labels=None): self.data = [] self.thread_num = thread_num param_str = ",".join(data_format.split(",")[1:]) format_params = common.get_params_dict(param_str) self.data_types = [] if format_params.get("2014-train", False): self.data_types.append("train2014") if format_params.get("2014-val", False): self.data_types.append("val2014") if format_params.get("2014-test", False): self.data_types.append("test2014") if format_params.get("2015-test", False): self.data_types.append("test2015") if format_params.get("2015-test-dev", False): self.data_types.append("test-dev2015") if len(self.data_types) == 0: raise Exception("please specify mscoco subset") bbox_hist = [0 for _ in range(32)] self.images = [] self.class_labels = {} self.categories = None for data_type in self.data_types: if "test" in data_type: fname = os.path.join( input_dir, "annotations/image_info_%s.json" % data_type) else: fname = os.path.join( input_dir, "annotations/instances_%s.json" % data_type) json_data = common.json_from_file(fname) #get class labels data_categories = {} for i, json_cat in enumerate(json_data["categories"]): data_categories[json_cat["id"]] = json_cat["name"] if not json_cat["name"] in self.class_labels: self.class_labels[json_cat["name"]] = len( self.class_labels) assert (self.categories is None) or (self.categories == data_categories) self.categories = data_categories logging.verbose("Found %i labels:" % len(self.class_labels)) #collect bounding boxes bboxs = {} for json_ann in json_data.get("annotations", []): cls_id = self.class_labels[self.categories[ json_ann["category_id"]]] image_id = json_ann["image_id"] bbox = json_ann["bbox"] if not image_id in bboxs: bboxs[image_id] = [] bboxs[image_id].append( (cls_id, (bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3]))) logging.verbose("Found %i bboxs" % (sum([len(bbox) for bbox in bboxs.values()]))) #collect images if data_type == "test-dev2015": data_type = "test2015" for image in json_data["images"]: fname = image["file_name"] image_id = image["id"] bbox_list = bboxs.get(image_id, []) bbox_hist[min(len(bbox_list), 31)] += 1 self.images.append({ "fname": os.path.join(input_dir, data_type, fname), "bboxs": bbox_list, "id": image_id }) print("BBox histogram (%):", [round(100.0 * x / sum(bbox_hist), 1) for x in bbox_hist]) #setup image loader self.image_loader = ImageLoader(thread_num, is_training, format_params) #subset self.output_size = self.image_loader.crop self.images_per_subset = format_params.get("images_per_subset", 10000) self.subset_total_size = len(self.images) self.subset_num = format_params.get("subset_num", sys.maxsize) self.subset_num = min( self.subset_num, int(math.ceil(self.subset_total_size / self.images_per_subset))) self.subset_index = -1 self.subset_size = self.images_per_subset self.bbox_only = format_params.get("bbox_only", False) #only use samples with bounding boxes if self.image_loader.is_training and self.bbox_only: images_bbox = [] for image in self.images: if len(image["bboxs"]) > 0: images_bbox.append(image) logging.info("Removed %i images without bboxs" % (len(self.images) - len(images_bbox))) self.images = images_bbox logging.info("Using MSCOCO dataset - size:", self.subset_total_size, "subset_num", self.subset_num, "images per subset:", self.subset_size, self.image_loader)
def save_to_file(model, fname, compresslevel=9): logging.info("Saving model to %s" % fname) t = time.time() common.json_to_gz(fname, model.export_json(), compresslevel) logging.verbose("Saving model took %.2f sec" % (time.time() - t))
def main(): #load arguments: parser = argparse.ArgumentParser( description='Train a convolutional network using labelled data.') logging.add_arguments(parser) parser.add_argument("--model", required=False, default=None, help="Model to continue training.") parser.add_argument("--cost-factors", default=[], nargs="+", help="Multiplicative factors for model costs") parser.add_argument( "--thread-num", type=int, default=1, help= "Number of threads to use for supported opeartions (e.g. loading/distorting datasets)" ) parser.add_argument("--extension", default="ppm", help="Image file extension") parser.add_argument("--train", default=None, help="The folder with training / validation data") parser.add_argument("--test", default=None, help="The folder with testing data (optional)") parser.add_argument("--test-epochs", type=int, default=1, help="Epochs between each test evaluation") parser.add_argument("--test-mode", default="default", help="Mode to use for testing") parser.add_argument( "--border-mode", default="valid", help="Border mode for convolutional layers (full, valid)") parser.add_argument("--output-prefix", default="./model", help="Output prefix for model files") parser.add_argument( "--activation", default="relu", help= "Activation function used in convolution / hidden layers (tanh, relu, leaky-relu)" ) parser.add_argument("--solver", type=str, default="nesterov", help="") parser.add_argument("--weight-init", nargs="+", default=["he-backward"], help="Weight initialization scheme") parser.add_argument("--learn-rate", type=float, default=0.1, help="Learning rate for weights and biases.") parser.add_argument( "--learn-momentum", type=float, default=[0.0, 0.0], nargs="+", help="Learning momentum for weights and biases (0.0 - 1.0).") parser.add_argument( "--learn-anneal", type=float, default=1, help="Annealing factor per epoch for weight and bias learning rate") parser.add_argument( "--learn-anneal-epochs", nargs="+", type=int, default=[], help="Epochs to apply learning rate annealing (default every epoch)") parser.add_argument("--learn-decay", type=float, default=0.0, help="L2 weight decay (not applied to biases). ") parser.add_argument("--epochs", type=int, default=30, help="The number of training epochs") parser.add_argument("--max-samples", type=int, default=None, help="Maximum samples to load from training set") parser.add_argument("--batch-size", type=int, default=32, help="Size of processing batchs") parser.add_argument("--seed", type=int, default=23455, help="Random Seed for weights") parser.add_argument( "--distort-mode", default=[], nargs="+", help="Distortions to apply to training data (default, cifar10, disable)" ) parser.add_argument("--disable-intermediate", default=False, action="store_true", help="Disable outputting of intermediate model files") parser.add_argument( "--augment-mirror", default=False, action="store_true", help="Augment training data with horizontally mirrored copies") parser.add_argument("--skip-train", default=False, action="store_true", help="Skip training of model") parser.add_argument("--skip-layer-updates", type=int, nargs="+", default=[], help="Skip training updates to specified layers") parser.add_argument("--model-desc", default=[ "C[100,7]", "P[2]", "C[150,4]", "P[2]", "C[250,4]", "P[2]", "C[300,1]", "R" ], nargs="+", type=str, help="Network layer description") args = parser.parse_args() logging.init(args) #set random seeds random.seed(args.seed) numpy.random.seed(args.seed) #load training dataset logging.info("Loading training data:", args.train) train_data = dataset.load(args.train, args.extension, is_training=True, thread_num=args.thread_num) data_shape = train_data.get_data_shape() class_num = train_data.get_class_num() class_labels = train_data.class_labels logging.info("Found %i class labels:\n" % class_num, class_labels) #hack for reducing training data size if not args.max_samples is None: train_data.data = random.sample(train_data.data, args.max_samples) #mirror training data if args.augment_mirror: train_data.augment_mirror() logging.info("Training: %i samples" % len(train_data)) #load test dataset if args.test: logging.info("Loading test: " + args.test) test_data = dataset.load(args.test, args.extension, is_training=False, thread_num=args.thread_num, class_labels=class_labels) #initialize model model = model_cnn.initialize(args, data_shape, class_labels, class_num) model.build_train_func(args.solver, args.cost_factors) #Run training best_test_error = 100.0 learn_rate = args.learn_rate for epoch in range(args.epochs): logging.info("----- Training Epoch: %i -----" % epoch) #perform training if not args.skip_train: logging.info("Training with solver " + args.solver + ", learning rate " + str(learn_rate) + " and momentum " + str(args.learn_momentum)) #shuffle dataset: train_data.shuffle() for subset in range(train_data.subset_num): timer = common.Timer() train_data.load_from_subset(subset) logging.info("Performing Gradient Descent...") cost = model.train_epoch(train_data, epoch, learn_rate, args.learn_momentum, args.learn_decay) nbatch = math.ceil(len(train_data) / model.batch_size) logging.info("Training subset %i - Cost: %.3f, Took %.1f sec" % (subset, cost, timer.current())) if len(args.learn_anneal_epochs) == 0 or ( epoch + 1) in args.learn_anneal_epochs: logging.verbose("Annealing learning rate") learn_rate *= args.learn_anneal #perform testing test_error = 0 if not args.test is None and ((epoch % args.test_epochs) == 0 or epoch == (args.epochs - 1)): test_error, test_class_errors = compute_error(test_data, model) logging.info( "Epoch %i test error: %.2f%% (%i samples)" % (epoch, test_error, int(test_error * len(test_data) / 100.0))) save_results(args.output_prefix + "_epoch%03i.test" % epoch, test_error, test_class_errors) #save intermediate models if not args.disable_intermediate: model_cnn.save_to_file( model, args.output_prefix + "_epoch%03i.mdl.gz" % (epoch)) #save final model model_cnn.save_to_file( model, args.output_prefix + "_epoch%03i_final.mdl.gz" % epoch) logging.info("Finished Training")
def build_train_func(self, solver_mode="sgd", cost_factors=[], use_acc_mode=False, skip_build=False): #arguments to function logging.info( "Building training functions - solver: %s, use_acc_mode: %s" % (solver_mode, use_acc_mode)) iteration = tensor.fscalar() learn_rate = tensor.fscalar() momentum = tensor.fvector() decay = tensor.fscalar() #find costs self.yt = [] self.cost_list = [] self.cost_layers = [] self.cost_layer_names = [] for layer in self.layers: yt_index = tensor.lvector("target index %i" % len(self.cost_layers)) yt_value = tensor.fvector("target value %i" % len(self.cost_layers)) cost = layer.cost(yt_index, yt_value) if not cost is None: self.yt += [yt_index, yt_value] self.cost_list.append(cost) self.cost_layers.append(layer) self.cost_layer_names.append(layer.type_name) self.cost_factors = [1.0] * len(self.cost_list) if len( cost_factors) == 0 else cost_factors assert len(self.cost_factors) == len( self.cost_list ), "Different number of cost factors (%i) and cost layers (%i)" % (len( self.cost_factors), len(self.cost_layers)) logging.info("Found %i costs in model:" % len(self.cost_layers), list(zip(self.cost_layer_names, self.cost_factors))) self.train_cost = tensor.as_tensor_variable(0) for i, cost in enumerate(self.cost_list): self.train_cost += self.cost_factors[i] * cost if self.gradient_clip > 0.0: logging.info("Clipping gradient to [%f,%f]" % (-self.gradient_clip, self.gradient_clip)) self.train_cost = theano.gradient.grad_clip( self.train_cost, -self.gradient_clip, self.gradient_clip) #find split points split_points = [0] self.use_split_mode = False for index, layer in enumerate(self.layers): if layer.has_split: self.use_split_mode = True split_points.append(index) split_points.append(len(self.layers)) if self.use_split_mode: logging.verbose("Using split mode with split points:", split_points) self.func["train_fwd"] = [] self.func["train_bwd"] = [] self.updates = [] for sp in range(len(split_points) - 1): logging.info("Building training functions for layers %i-%i" % (split_points[sp], split_points[sp + 1])) split_start = self.layers[split_points[sp]] if sp > 0 else None split_end = self.layers[split_points[sp + 1]] if ( sp + 2) < len(split_points) else None split_cost = self.train_cost if split_end is None else None split_layers = [] for i, layer in enumerate(self.layers): if (i > split_points[sp]) and (i < split_points[sp + 1]): split_layers.append(layer) #determine known_grads provided by previous backward passes from collections import OrderedDict split_known_grads = OrderedDict() for i in range(sp + 1, len(split_points) - 1): split_known_grads.update( self.layers[split_points[i]].split_known_grads()) if len(split_known_grads) == 0: split_known_grads = None # print(split_known_grads) # print(split_known_grads) # print(sp+1, len(split_points)-1) # def get_sgd_updates(p, g): m = theano.shared(numpy.zeros(p.shape.eval(), dtype=theano.config.floatX), broadcastable=p.broadcastable, borrow=True) rho = tensor.switch(tensor.gt(iteration, 0), momentum[0], 0.0) m_update = rho * m + (1.0 - rho) * g p_update = p - learn_rate * m_update return [(p, p_update), (m, m_update)] def get_torch_updates(p, g): m = theano.shared(numpy.zeros(p.shape.eval(), dtype=theano.config.floatX), broadcastable=p.broadcastable, borrow=True) rho = tensor.switch(tensor.gt(iteration, 0), momentum[0], 0.0) m_update = rho * m + g p_update = p - learn_rate * (g + momentum[0] * m_update) return [(p, p_update), (m, m_update)] def get_adam_updates(p, g): eps = 1e-8 m = theano.shared(numpy.zeros(p.shape.eval(), dtype=theano.config.floatX), broadcastable=p.broadcastable, borrow=True) v = theano.shared(numpy.zeros(p.shape.eval(), dtype=theano.config.floatX), broadcastable=p.broadcastable, borrow=True) m_update = momentum[0] * m + (1.0 - momentum[0]) * g v_update = momentum[1] * v + (1.0 - momentum[1]) * (g * g) m_hat = m_update / (1.0 - tensor.pow(momentum[0], iteration + 1)) v_hat = v_update / (1.0 - tensor.pow(momentum[1], iteration + 1)) p_update = p - learn_rate * m_hat / (tensor.sqrt(v_hat) + eps) return [(p, p_update), (m, m_update), (v, v_update)] #append parameter updates params = [] params_decay = [] for layer in split_layers: params += layer.weights() params_decay += [True] * len(layer.weights()) params += layer.biases() params_decay += [False] * len(layer.biases()) #build updates print("known grads:", split_known_grads) grads = tensor.grad(split_cost, params, known_grads=split_known_grads) solver_updates = [] for p, g, p_decay in zip(params, grads, params_decay): #add L2 weight decay if needed if p_decay or self.bias_decay: g += decay * p if solver_mode == "adam": solver_updates += get_adam_updates(p, g) elif solver_mode == "torch" or solver_mode == "nesterov": solver_updates += get_torch_updates(p, g) else: solver_updates += get_sgd_updates(p, g) #append per layer updates local_updates = solver_updates + sum( [layer.updates(self.train_cost) for layer in split_layers], []) #all updates self.updates += local_updates #skipping actual theano function building (if you just want updates, etc) if skip_build: continue global debug_train if debug_train: logging.warning("WARNING: Debug mode is active!") from theano.compile.nanguardmode import NanGuardMode debug_mode = theano.compile.MonitorMode( post_func=debug_detect_errors) else: debug_mode = None if self.use_split_mode: if not split_end is None: updates = sum( [layer.split_forward() for layer in split_layers], []) updates += split_end.split_forward() print("fwd updates:", updates) f = theano.function([self.input], [], updates=updates, givens=[(denet.layer.get_train(), tensor.cast(1, 'int8'))], on_unused_input='ignore', mode=debug_mode) self.func["train_fwd"].append(f) outputs = ([self.train_cost] + self.cost_list) if split_end is None else [] updates = sum([ layer.split_backward(split_cost, split_known_grads) for layer in split_layers ], []) if not split_start is None: updates += split_start.split_backward( split_cost, split_known_grads) print("bwd updates:", updates) updates += local_updates f = theano.function([ denet.layer.get_epoch(), iteration, learn_rate, momentum, decay, self.input ] + self.yt, outputs, updates=updates, givens=[(denet.layer.get_train(), tensor.cast(1, 'int8'))], on_unused_input='ignore', mode=debug_mode) self.func["train_bwd"].insert(0, f) elif use_acc_mode: acc_counter = theano.shared( numpy.array(0, dtype=theano.config.floatX)) begin_updates = [(acc_counter, tensor.zeros_like(acc_counter))] step_updates = [(acc_counter, acc_counter + 1)] end_updates = [] self.acc_params = [] for p_dest, p_src in self.updates: p_acc = theano.shared(numpy.zeros( p_dest.shape.eval(), dtype=theano.config.floatX), broadcastable=p_dest.broadcastable, borrow=True) begin_updates.append((p_acc, tensor.zeros_like(p_acc))) step_updates.append((p_acc, p_acc + p_src)) end_updates.append((p_dest, p_acc / acc_counter)) self.acc_params.append(p_acc) logging.info( "Constructing parameter accumulate update functions (solver=%s)" % solver_mode) self.func["train_begin"] = theano.function( [], [], updates=begin_updates) self.func["train_step"] = theano.function( [ denet.layer.get_epoch(), iteration, learn_rate, momentum, decay, self.input ] + self.yt, [self.train_cost] + self.cost_list, updates=step_updates, givens=[(denet.layer.get_train(), tensor.cast(1, 'int8'))], on_unused_input='ignore', allow_input_downcast=True, mode=debug_mode) self.func["train_end"] = theano.function([], [], updates=end_updates) else: logging.info( "Constructing parameter update function (solver=%s)" % solver_mode) #making f_input = theano.In(self.input, borrow=True) f_yt = [theano.In(yt, borrow=True) for yt in self.yt] self.func["train_step"] = theano.function( [ denet.layer.get_epoch(), iteration, learn_rate, momentum, decay, f_input ] + f_yt, [self.train_cost] + self.cost_list, updates=self.updates, givens=[(denet.layer.get_train(), tensor.cast(1, 'int8'))], on_unused_input='ignore', allow_input_downcast=True, mode=debug_mode) logging.verbose("Exporting graph...") with open("graph.txt", "w") as f: theano.printing.debugprint(self.func["train_step"], file=f, print_type=True)
def run_worker(gpu, args, data_shape, class_labels, class_num, task_queue, active, epoch, learn_rate, cost, timer, data_x, data_y, data_m, model_update): #redirect output (unbuffered) sys.stdout = open(gpu + ".out", 'w') sys.stderr = open(gpu + ".err", 'w') logging.init(args, flush=True) sys.setrecursionlimit(10000) #create thread to flush stdout / stderr every 5 seconds flush_logs() logging.info(gpu + ": initializing") #remove all openmpi variables! for v in os.environ.keys(): if v[:5] == "OMPI_": del os.environ[v] #set compile dir and gpu (possible since theano hasn't been imported yet!) if not "THEANO_FLAGS" in os.environ: os.environ["THEANO_FLAGS"] = "" import socket os.environ["THEANO_FLAGS"] += "," + args.theano_flags + "," os.environ["THEANO_FLAGS"] += "device=" + gpu + "," os.environ["THEANO_FLAGS"] += "force_device=True," os.environ["THEANO_FLAGS"] += "compiledir=~/.theano/" + socket.gethostname( ) + "-" + gpu + "/," #os.environ["THEANO_FLAGS"] += "lib.cnmem=1,"; os.environ["THEANO_FLAGS"] += "nvcc.flags=-D_FORCE_INLINES," logging.info(gpu + ": Using THEANO_FLAGS:", os.environ["THEANO_FLAGS"]) #initialize local model import denet.model.model_cnn as model_cnn model = model_cnn.initialize(args, data_shape, class_labels, class_num) #pre-initialize training function use_acc_mode = args.batch_size_factor > 1 and args.use_acc_mode model.build_train_func(args.solver, args.cost_factors, use_acc_mode=use_acc_mode) if use_acc_mode: train_begin_func = model.func["train_begin"] train_end_func = model.func["train_end"] #begin processing loop iteration = 0 while (True): #try to start next task immediately otherwise wait for task wait_time = time.time() try: task = task_queue.get(block=False) except queue.Empty: logging.verbose(gpu + ": waiting for task") with active.get_lock(): active.value = 0 task = task_queue.get(block=True) wait_time = time.time() - wait_time logging.verbose(gpu + ": " + task + " (wait time=%i ms)" % (1000 * wait_time)) #calculate updates ts = time.time() if task == "predict": with data_x.lock, data_y.lock: data_y.get_array()[...] = model.predict_output_step( data_x.get_array()) elif task == "model-read": model_update.import_updates(model) elif task == "model-write": model_update.export_updates(model) elif task == "train-begin": if use_acc_mode: train_begin_func() with cost.get_lock(): cost.value = 0 elif task == "train-step": with cost.get_lock(), epoch.get_lock(), learn_rate.get_lock( ), data_x.lock: data_meta = data_m.get(block=True) c, _ = model.train_step(data_x.get_array(), data_meta, epoch.value, iteration, learn_rate.value, args.learn_momentum, args.learn_decay) if math.isnan(c): raise Exception("Encountered NaN cost for worker") cost.value += c iteration += 1 elif task == "train-end": if use_acc_mode: train_end_func() with cost.get_lock(): cost.value /= args.batch_size_factor elif task == "done": exit(0) with timer.get_lock(): timer.value = time.time() - ts logging.info(gpu + ": %s took %i ms" % (task, 1000 * timer.value))
def __init__(self, layers, filter_shape=None, stride = (1,1), bottleneck=0, activation = "relu", version="original", json_param={}): super().__init__(layer_index=len(layers)) self.input = layers[-1].output self.input_shape = layers[-1].output_shape #get parameters self.filter_shape = json_param.get("shape", filter_shape) self.stride = json_param.get("stride", stride) self.bottleneck = json_param.get("bottleneck", bottleneck) self.version = json_param.get("version", version) self.activation = json_param.get("activation", activation) self.bn_json_param = json_param.get("bnParam", { "enabled":json_param.get("enableBatchNorm", True), }) #determing convolution shapes if self.bottleneck > 0: self.size = (self.filter_shape[2], self.filter_shape[3]) shape0 = (self.bottleneck, self.filter_shape[1], 1, 1) shape1 = (self.bottleneck, self.bottleneck, self.filter_shape[2], self.filter_shape[3]) shape2 = (self.filter_shape[0], self.bottleneck, 1, 1) else: self.size = (self.filter_shape[2]*2 - 1, self.filter_shape[3]*2 - 1) shape0 = self.filter_shape shape1 = (self.filter_shape[0], self.filter_shape[0], self.filter_shape[2], self.filter_shape[3]) shape2 = None logging.verbose("Adding", self) # logging.verbose("Adding", self, "layer - input:", self.input_shape, "shape:", self.filter_shape, "stride:", self.stride, "bottleneck:", self.bottleneck, # "activation:", self.activation, "version:", self.version, "bn param:", self.bn_json_param) logging.verbose("--------RESNET BLOCK----------") self.layers = [InitialLayer(self.input, self.input_shape)] if "pre-activation" in self.version: if "bnrelu" in self.version and self.activation == "relu": self.layers.append(BatchNormReluLayer(self.layers, json_param = self.bn_json_param)) else: self.layers.append(BatchNormLayer(self.layers, json_param = self.bn_json_param)) self.layers.append(ActivationLayer(self.layers, self.activation)) self.layers.append(ConvLayer(self.layers, filter_shape = shape0, filter_stride=self.stride, border_mode="half", use_bias=False)) if "bnrelu" in self.version and self.activation == "relu": self.layers.append(BatchNormReluLayer(self.layers, json_param = self.bn_json_param)) else: self.layers.append(BatchNormLayer(self.layers, json_param = self.bn_json_param)) self.layers.append(ActivationLayer(self.layers, self.activation)) self.layers.append(ConvLayer(self.layers, filter_shape = shape1, border_mode="half", use_bias=False)) #for bottleneck design add additional conv if self.bottleneck > 0: if "bnrelu" in self.version and self.activation == "relu": self.layers.append(BatchNormReluLayer(self.layers, json_param = self.bn_json_param)) else: self.layers.append(BatchNormLayer(self.layers, json_param = self.bn_json_param)) self.layers.append(ActivationLayer(self.layers, self.activation)) self.layers.append(ConvLayer(self.layers, filter_shape = shape2, border_mode="half", use_bias=False)) if not "pre-activation" in self.version: self.layers.append(BatchNormLayer(self.layers, json_param = self.bn_json_param)) y = self.layers[-1].output y_shape = self.layers[-1].output_shape #project input shape to output shape dimensions if self.input_shape != y_shape: logging.verbose("---------SHORTCUT----------") #handle resnet models with batchnorm in shortcut route if "pre-activation" in self.version: input_layers = self.layers[0:2] else: input_layers = [InitialLayer(self.input, self.input_shape)] self.layers.append(ConvLayer(input_layers, filter_shape=(y_shape[1], self.input_shape[1], 1, 1), filter_stride=self.stride, use_bias=False, border_mode="half")) #original model has batch norm after shortcut if "original" in self.version: self.layers.append(BatchNormLayer(self.layers, json_param = self.bn_json_param)) x = self.layers[-1].output else: x = self.input logging.verbose("------------------------------") #add residual self.output_shape = y_shape if "pre-activation" in self.version: self.output = x + y else: self.output = ActivationLayer.apply(x + y, self.activation)
def __init__(self, layers, filter_shape=None, filter_stride=(1, 1), use_bias=True, border_mode="valid", wb="he-backward", json_param={}): super().__init__(layer_index=len(layers)) self.input = layers[-1].output self.input_shape = layers[-1].output_shape #get parameters self.border_mode = json_param.get("border", border_mode) self.filter_shape = tuple(json_param.get("shape", filter_shape)) self.stride = tuple(json_param.get("stride", filter_stride)) self.use_bias = json_param.get("useBias", use_bias) self.size = (self.filter_shape[2], self.filter_shape[3]) #use initialization if type(wb) is float: self.w_bound = float(wb) elif "he-forward" in wb: self.w_bound = math.sqrt( 2.0 / (self.filter_shape[2] * self.filter_shape[3] * self.filter_shape[1])) elif "he-backward" in wb: self.w_bound = math.sqrt( 2.0 / (self.filter_shape[2] * self.filter_shape[3] * self.filter_shape[0])) elif "xavier-forward" in wb: self.w_bound = math.sqrt( 1.0 / (self.filter_shape[2] * self.filter_shape[3] * self.filter_shape[1])) elif "xavier-backward" in wb: self.w_bound = math.sqrt( 1.0 / (self.filter_shape[2] * self.filter_shape[3] * self.filter_shape[0])) #initialize weights if self.w_bound > 0: if "uniform" in wb: w = numpy.random.uniform(-self.w_bound, self.w_bound, size=self.filter_shape) else: w = numpy.random.normal(0.0, self.w_bound, size=self.filter_shape) else: w = numpy.zeros(shape=self.filter_shape) self.omega = theano.shared(numpy.asarray(w, dtype=theano.config.floatX), name="deconv omega") #initialize bias if self.use_bias: self.beta = theano.shared(value=numpy.zeros( (self.filter_shape[0], ), dtype=theano.config.floatX), name="deconv beta") #calculate output shape if self.border_mode == "half": fh = self.filter_shape[2] // 2 fw = self.filter_shape[3] // 2 h = self.input_shape[2] * self.stride[ 0] - 2 * fh + self.filter_shape[2] - 1 w = self.input_shape[3] * self.stride[ 1] - 2 * fw + self.filter_shape[3] - 1 else: raise Exception("Unknown border mode: " + str(self.border_mode)) self.output_shape = (self.input_shape[0], self.filter_shape[0], h, w) self.output = conv2d_grad_wrt_inputs( self.input, self.omega.dimshuffle((1, 0, 2, 3)), self.output_shape, (self.filter_shape[1], self.filter_shape[0], self.filter_shape[2], self.filter_shape[3]), self.border_mode, self.stride) if self.use_bias: self.output += self.beta[None, :, None, None] logging.verbose("Adding", self)
def __init__(self, layers, crop_size=None, mirror_pr=0.0, flip_pr=0.0, json_param={}): super().__init__(layer_index=len(layers)) self.input = layers[-1].output self.input_shape = layers[-1].output_shape self.crop_size = json_param.get("crop", crop_size) self.mirror_pr = json_param.get("mirror", mirror_pr) self.flip_pr = json_param.get("flip", flip_pr) self.output_shape = (self.input_shape[0], self.input_shape[1], self.crop_size[0], self.crop_size[1]) self.output = [] zero = tensor.zeros((self.input_shape[0], ), dtype=numpy.int8) index_b = tensor.arange(self.input_shape[0]) index_c = tensor.arange(self.input_shape[1]) index_x = tensor.arange(self.crop_size[0])[None, :] index_y = tensor.arange(self.crop_size[1])[None, :] #randomly mirror (y-axis) data if self.mirror_pr > 0.0: mirror = tensor.gt(get_rng().uniform(size=(self.input_shape[0], )), 1.0 - self.mirror_pr) mirror = tensor.switch(get_train(), mirror, zero) index_y = tensor.switch(mirror[:, None], -index_y + self.crop_size[1] - 1, index_y) #randomly flip (x-axis) data if self.flip_pr > 0.0: flip = tensor.gt(get_rng().uniform(size=(self.input_shape[0], )), 1.0 - self.flip_pr) flip = tensor.switch(get_train(), flip, zero) index_x = tensor.switch(flip[:, None], -index_x + self.crop_size[0] - 1, index_x) #randomly offset crop dx = self.input_shape[2] - self.crop_size[0] dy = self.input_shape[3] - self.crop_size[1] if self.crop_size[0] != self.input_shape[2] or self.crop_size[ 1] != self.input_shape[3]: center_x = theano.shared(numpy.full(shape=(self.input_shape[0], ), fill_value=dx // 2, dtype=numpy.int32), borrow=False) center_y = theano.shared(numpy.full(shape=(self.input_shape[0], ), fill_value=dy // 2, dtype=numpy.int32), borrow=False) offset_x = get_rng().random_integers(size=(self.input_shape[0], ), low=0, high=dx) offset_y = get_rng().random_integers(size=(self.input_shape[0], ), low=0, high=dy) index_x += tensor.switch(get_train(), offset_x, center_x)[:, None] index_y += tensor.switch(get_train(), offset_y, center_y)[:, None] #perform advanced indexing self.output = self.input[index_b[:, None, None, None], index_c[None, :, None, None], index_x[:, None, :, None], index_y[:, None, None, :]] logging.verbose("Adding", self)
def __init__(self, layers, filter_shape=None, filter_stride=(1, 1), use_bias=False, border_mode="half", wb="he-backward", json_param={}): super().__init__(layer_index=len(layers)) self.input = layers[-1].output self.input_shape = layers[-1].output_shape #get parameters self.border_mode = json_param.get("border", border_mode) self.filter_shape = tuple(json_param.get("shape", filter_shape)) self.stride = tuple(json_param.get("stride", filter_stride)) self.use_bias = json_param.get("useBias", use_bias) self.size = (self.filter_shape[2], self.filter_shape[3]) self.enabled = json_param.get("enabled", True) #use initialization if type(wb) is float: self.w_bound = float(wb) elif "he-forward" in wb: self.w_bound = math.sqrt( 2.0 / (self.filter_shape[2] * self.filter_shape[3] * self.filter_shape[1])) elif "he-backward" in wb: self.w_bound = math.sqrt( 2.0 / (self.filter_shape[2] * self.filter_shape[3] * self.filter_shape[0])) elif "xavier-forward" in wb: self.w_bound = math.sqrt( 1.0 / (self.filter_shape[2] * self.filter_shape[3] * self.filter_shape[1])) elif "xavier-backward" in wb: self.w_bound = math.sqrt( 1.0 / (self.filter_shape[2] * self.filter_shape[3] * self.filter_shape[0])) #initialize weights if self.w_bound > 0: if "uniform" in wb: w = numpy.random.uniform(-self.w_bound, self.w_bound, size=self.filter_shape) else: w = numpy.random.normal(0.0, self.w_bound, size=self.filter_shape) else: w = numpy.zeros(shape=self.filter_shape) self.omega = theano.shared(numpy.asarray(w, dtype=theano.config.floatX), name="conv omega") #initialize bias if self.use_bias: self.beta = theano.shared(value=numpy.zeros( (self.filter_shape[0], ), dtype=theano.config.floatX), name="conv beta") #calculate output shape if self.border_mode == 'valid': w = math.ceil((self.input_shape[-2] - self.filter_shape[2] + 1) / self.stride[0]) h = math.ceil((self.input_shape[-1] - self.filter_shape[3] + 1) / self.stride[1]) elif self.border_mode == "full": w = math.ceil((self.input_shape[-2] + self.filter_shape[2] - 1) / self.stride[0]) h = math.ceil((self.input_shape[-1] + self.filter_shape[3] - 1) / self.stride[1]) elif self.border_mode == "half": fh = self.filter_shape[2] // 2 fw = self.filter_shape[3] // 2 w = math.ceil( (self.input_shape[-2] + 2 * fh - self.filter_shape[2] + 1) / self.stride[0]) h = math.ceil( (self.input_shape[-1] + 2 * fw - self.filter_shape[3] + 1) / self.stride[1]) elif self.border_mode == "same": assert self.stride == (1, 1) w = self.input_shape[-2] h = self.input_shape[-1] elif isinstance(self.border_mode, int): w = math.ceil((self.input_shape[-2] + 2 * self.border_mode - self.filter_shape[2] + 1) / self.stride[0]) h = math.ceil((self.input_shape[-1] + 2 * self.border_mode - self.filter_shape[3] + 1) / self.stride[1]) else: raise Exception("Unknown border mode: " + str(self.border_mode)) #handle "same" mode specially if self.border_mode == "same": y0 = (self.filter_shape[2] - 1) // 2 x0 = (self.filter_shape[3] - 1) // 2 y = tensor.nnet.conv2d(input=self.input, filters=self.omega, filter_shape=self.filter_shape, subsample=self.stride, input_shape=self.input_shape, border_mode="full")[:, :, y0:(y0 + h), x0:(x0 + w)] else: # y = tensor.nnet.conv2d(input=self.input, filters=self.omega, filter_shape=self.filter_shape, subsample=self.stride, input_shape=self.input_shape, border_mode=self.border_mode) y = tensor.nnet.conv2d(input=self.input, filters=self.omega, filter_shape=self.filter_shape, subsample=self.stride, border_mode=self.border_mode) y_shape = (self.input_shape[0], self.filter_shape[0], w, h) self.output_unbiased = y if self.use_bias: y += self.beta[None, :, None, None] self.output_shape = y_shape self.output = y logging.verbose("Adding", self)