def __init__(self, recurrent=[], recurrent_dim=[], skip_list=[], use_fast_fprop=0, self_recurrent=1, init_state_cons=0., init_U=InitCell('ortho'), **kwargs): super(RecurrentLayer, self).__init__(**kwargs) self.recurrent = OrderedDict() if self_recurrent: self.recurrent[self.name] = self.nout recurrent_dim = tolist(recurrent_dim) for i, rec in enumerate(tolist(recurrent)): if len(recurrent_dim) != 0: self.recurrent[rec] = recurrent_dim[i] else: self.recurrent[rec] = None self.init_U = init_U self.init_states = OrderedDict() self.init_state_cons = init_state_cons self.use_fast_fprop = use_fast_fprop self.skip_list = tolist(skip_list) if len(self.skip_list) > 0: if len(self.skip_list) != len(self.parent): raise ValueError("length of parents and skip list should match")
def __init__(self, recurrent=[], recurrent_dim=[], self_recurrent=1, clip_gradient = True, clip_bound = 5, init_U=InitCell('ortho'), **kwargs): super(RecurrentLayer, self).__init__(**kwargs) self.recurrent = OrderedDict() if self_recurrent: self.recurrent[self.name] = self.nout recurrent_dim = tolist(recurrent_dim) for i, rec in enumerate(tolist(recurrent)): if len(recurrent_dim) != 0: self.recurrent[rec] = recurrent_dim[i] else: self.recurrent[rec] = None self.clip_gradient = clip_gradient self.clip_bound = clip_bound self.init_U = init_U
def __init__(self, parent=[], parent_dim=[], nout=None, init_W=InitCell('randn'), init_b=InitCell('zeros'), cons=0., name=None, lr_scaler=None, **kwargs): super(StemCell, self).__init__(**kwargs) if name is None: name = self.__class__.name__.lower() self.name = name self.nout = nout self.init_W = init_W self.init_b = init_b self.cons = cons self.parent = OrderedDict() parent_dim = tolist(parent_dim) for i, par in enumerate(tolist(parent)): if len(parent_dim) != 0 and len(parent) != 0: if len(parent) != len(parent_dim): raise AssertionError("You probably had a mistake providing,\ write number of values. It will end,\ up with a model containing a bug.") self.parent[par] = parent_dim[i] else: self.parent[par] = None self.params = OrderedDict() self.lr_scaler = lr_scaler
def __init__(self, parent=[], parent_dim=[], nout=None, init_W=InitCell('randn'), init_b=InitCell('zeros'), cons=0., name=None, lr_scaler=None, **kwargs): super(StemCell, self).__init__(**kwargs) if name is None: name = self.__class__.name__.lower() self.name = name self.nout = nout self.init_W = init_W self.init_b = init_b self.cons = cons self.parent = OrderedDict() parent_dim = tolist(parent_dim) for i, par in enumerate(tolist(parent)): if len(parent_dim) != 0 and len(parent) != 0: if len(parent) != len(parent_dim): raise AssertionError( "You probably had a mistake providing,\ write number of values. It will end,\ up with a model containing a bug.") self.parent[par] = parent_dim[i] else: self.parent[par] = None self.params = OrderedDict() self.lr_scaler = lr_scaler
def __init__(self, is_vector=1, weight_norm=1.9365, keys=['W'], waivers=[]): """ .. todo:: WRITEME """ self.name = 'ext_regularize_post_grad' self.weight_norm = weight_norm self.keys = tolist(keys) self.waivers = tolist(waivers) self.is_vector = is_vector
def __init__(self, name, data, model, optimizer, cost, outputs, debug_print=0, trainlog=None, extension=None): self.name = name self.data = data self.model = model self.optimizer = optimizer self.inputs = model.inputs self.params = model.params self.cost = cost self.outputs = tolist(outputs) self.updates = model.updates self.extension = extension self.debug_print = debug_print t0 = time.time() self.cost_fn = self.build_training_graph() print "Elapsed compilation time: %f" % (time.time() - t0) if self.debug_print: from theano.printing import debugprint debugprint(self.cost_fn) if trainlog is None: self.trainlog = TrainLog() else: self.trainlog = trainlog self.endloop = 0
def __init__(self, lambd=0.0002, param_name=['W']): """ .. todo:: WRITEME """ self.name = 'ext_regularize_pre_grad' self.lambd = lambd self.param_name = tolist(param_name)
def build_recurrent_graph(self, n_steps=None, reverse=False, **kwargs): self.nonseq_args = kwargs.pop('nonseq_args', None) self.output_args = kwargs.pop('output_args', None) self.context_args = kwargs.pop('context_args', None) self.iterators = kwargs.pop('iterators', None) self.nonseq_inputs = kwargs.pop('nonseq_inputs', None) self.nNone = 0 inputs = self.inputs.values() seqs = [] outputs = [] nonseqs = [] if self.nonseq_inputs is not None: for i in self.nonseq_inputs: nonseqs.append(inputs.pop(i)) self.input_args = OrderedDict() self.recur_args = OrderedDict() for name, node in self.nodes.items(): if hasattr(node, 'get_init_state'): self.recur_args[name] = node state = node.get_init_state() outputs.append(state) self.nrecur = len(self.recur_args) # Substitutes initial hidden state into a context if self.context_args is not None: for i, (nname, node) in enumerate(self.output_args.items()): for aname, arg in self.context_args.items(): if nname == aname: outputs[i] = arg if self.iterators is None: seqs += inputs elif self.iterators is not None: seqs += inputs[len(self.iterators):] outputs += inputs[:len(self.iterators)] if self.output_args is not None: self.nNone = len(self.output_args) outputs = flatten(outputs + [None] * self.nNone) if self.nonseq_args is not None: for arg in self.nonseq_args: nonseqs.append(arg) self.nseqs = len(seqs) self.noutputs = len(outputs) self.nnonseqs = len(nonseqs) result, updates = theano.scan( fn=self.scan_fn, sequences=seqs, outputs_info=outputs, non_sequences=nonseqs, n_steps=n_steps, go_backwards=reverse) result = tolist(result) if self.output_args is None and self.iterators is None: return result if len(updates) == 0: return result[-self.nNone:] for k, v in updates.iteritems(): k.default_update = v return result[-self.nNone:], updates
def __init__(self, lambd=0.0002, keys=['W']): """ .. todo:: WRITEME """ self.name = 'ext_regularize_pre_grad' self.lambd = lambd self.keys = tolist(keys)
def __init__(self, is_vector=1, weight_norm=1.9365, param_name=['W']): """ .. todo:: WRITEME """ self.name = 'ext_regularize_post_grad' self.weight_norm = weight_norm self.param_name = tolist(param_name) self.is_vector = is_vector
def set_graph(self): self.graph = {} for nname, node in self.nodes.items(): parent = node.parent for par in tolist(parent.keys()): if par in self.inputs.keys(): continue if par in self.graph.keys(): self.graph[par] =\ tolist(self.graph[par]) + [node.name] else: self.graph[par] = node.name sorted_nodes = topological_sort(self.graph) if len(self.graph) > 0: for i in xrange(len(self.nodes)): self.sorted_nodes.append(sorted_nodes.popleft()) for node in self.nodes: parent = self.nodes[node].parent for par in tolist(parent.keys()): try: self.nodes[node].parent[par] = self.inputs_dim[par] except: if self.nodes[par].nout is not None: # Assume this is FullyConnectedLayer self.nodes[node].parent[par] = self.nodes[par].nout else: # Assume this is ConvLayer try: self.nodes[node].parent[par] = self.nodes[ par].outshape except: # Assume this is MaxPool2D self.nodes[par].initialize() self.nodes[node].parent[par] = self.nodes[ par].outshape if hasattr(node, 'recurrent'): recurrent = self.nodes[node].recurrent for rec in tolist(recurrent.keys()): self.nodes[node].recurrent[rec] = self.nodes[rec].nout
def __init__(self, recurrent=[], recurrent_dim=[], self_recurrent=1, init_U=InitCell('ortho'), **kwargs): super(RecurrentLayer, self).__init__(**kwargs) self.recurrent = OrderedDict() if self_recurrent: self.recurrent[self.name] = self.nout recurrent_dim = tolist(recurrent_dim) for i, rec in enumerate(tolist(recurrent)): if len(recurrent_dim) != 0: self.recurrent[rec] = recurrent_dim[i] else: self.recurrent[rec] = None self.init_U = init_U
def __init__(self, batch_size, recurrent=[], recurrent_dim=[], self_recurrent=1, init_state_cons=0., init_U=InitCell('ortho'), **kwargs): super(RecurrentLayer, self).__init__(**kwargs) self.recurrent = OrderedDict() if self_recurrent: self.recurrent[self.name] = self.nout recurrent_dim = tolist(recurrent_dim) for i, rec in enumerate(tolist(recurrent)): if len(recurrent_dim) != 0: self.recurrent[rec] = recurrent_dim[i] else: self.recurrent[rec] = None self.batch_size = batch_size self.init_U = init_U self.init_states = OrderedDict() self.init_state_cons = init_state_cons
def set_graph(self): self.graph = {} for nname, node in self.nodes.items(): parent = node.parent for par in tolist(parent.keys()): if par in self.inputs.keys(): continue if par in self.graph.keys(): self.graph[par] =\ tolist(self.graph[par]) + [node.name] else: self.graph[par] = node.name sorted_nodes = topological_sort(self.graph) if len(self.graph) > 0: for i in xrange(len(self.nodes)): self.sorted_nodes.append(sorted_nodes.popleft()) for node in self.nodes: parent = self.nodes[node].parent for par in tolist(parent.keys()): try: self.nodes[node].parent[par] = self.inputs_dim[par] except: if self.nodes[par].nout is not None: # Assume this is FullyConnectedLayer self.nodes[node].parent[par] = self.nodes[par].nout else: # Assume this is ConvLayer try: self.nodes[node].parent[par] = self.nodes[par].outshape except: # Assume this is MaxPool2D self.nodes[par].initialize() self.nodes[node].parent[par] = self.nodes[par].outshape if hasattr(node, 'recurrent'): recurrent = self.nodes[node].recurrent for rec in tolist(recurrent.keys()): self.nodes[node].recurrent[rec] = self.nodes[rec].nout
def __init__(self, name, data, model, optimizer, cost, outputs, n_steps, debug_print=0, trainlog=None, extension=None, lr_iterations=None, decay_schedule=2, k_speedOfconvergence=40): #picklelized? self.name = name # yes self.data = data # no self.model = model #yes self.optimizer = optimizer #no self.inputs = model.inputs #no self.cost = cost #yes self.outputs = tolist(outputs) #no self.updates = OrderedDict() # no self.updates.update(model.updates) #??? self.extension = extension #no self.debug_print = debug_print #no lr_scalers = OrderedDict() #yes for node in self.model.nodes: #should lr_scalers[node.name] = node.lr_scaler self.optimizer.lr_scalers = lr_scalers #should self.nBernoulli = np.ones((n_steps, )) #yes t0 = time.time() self.cost_fn = self.build_training_graph() # no but should print "Elapsed compilation time: %f" % (time.time() - t0) if self.debug_print: #no from theano.printing import debugprint debugprint(self.cost_fn) if trainlog is None: #yes self.trainlog = TrainLog() else: self.trainlog = trainlog self.endloop = 0 #no self.lr_iterations = lr_iterations #yes self.lastBatchlastPoch = 0 #yes self.decay_schedule = decay_schedule #yes self.k = k_speedOfconvergence #yes self.schedRate = 1 #yes self.n_steps = n_steps #yes
def restore(self, data, optimizer, cost, outputs, n_steps, debug_print=0, trainlog=None, extension=None, lr_iterations=None, decay_schedule=2, k_speedOfconvergence=40): self.data = data self.optimizer = optimizer self.inputs = self.model.inputs self.cost = cost self.outputs = tolist(outputs) #self.updates = OrderedDict() #self.updates.update(self.model.updates) self.updates = self.model.updates self.extension = extension self.debug_print = debug_print lr_scalers = OrderedDict() for node in self.model.nodes: lr_scalers[node.name] = node.lr_scaler self.optimizer.lr_scalers = lr_scalers self.nBernoulli = np.ones((n_steps, )) t0 = time.time() self.cost_fn = self.build_training_graph() print "Elapsed compilation time: %f" % (time.time() - t0) if self.debug_print: from theano.printing import debugprint debugprint(self.cost_fn) if trainlog is None: self.trainlog = TrainLog() else: self.trainlog = trainlog self.endloop = 0 self.lr_iterations = lr_iterations self.lastBatchlastPoch = 0 self.decay_schedule = decay_schedule self.k = k_speedOfconvergence self.schedRate = 1 self.n_steps = n_steps
def __init__(self, name, data, model, optimizer, cost, outputs, extension=None): self.name = name self.data = data self.model = model self.optimizer = optimizer self.inputs = model.inputs self.params = model.params self.cost = cost self.outputs = tolist(outputs) self.updates = model.updates self.extension = extension self.cost_fn = self.build_training_graph() self.trainlog = TrainLog() self.endloop = 0
def __init__(self, parshape=[], outshape=None, filtershape=None, tied_bias=True, step_size=(1, 1), border_mode='valid', **kwargs): super(Conv2DLayer, self).__init__(**kwargs) # Shape should be (batch_size, num_channels, x, y) if (outshape is None and filtershape is None) or\ (outshape is not None and filtershape is not None): raise ValueError("Either outshape or filtershape should be given,\ but don't provide both of them.") self.outshape = outshape self.filtershape = filtershape self.tied_bias = tied_bias self.step_size = totuple(step_size) self.border_mode = border_mode for i, par in enumerate(tolist(self.parent.keys())): if len(parshape) != 0: self.parent[par] = parshape[i]
def get_params(self): params = [] for graph in tolist(self.graphs): params += graph.params return params
def slices(self, start, end): batches = [mat[start:end] for mat in self.data] mask = tolist(self.create_mask(batches[0].swapaxes(0, 1))) batches = [self.zero_pad(batch) for batch in batches] return totuple(batches + mask)
def add_node(self, nodes): for node in tolist(nodes): self.nodes[node.name] = node self.params = self.get_params()
def scan_fn(self, *args): next_recurrence = [] sorted_nodes = topological_sort(self.graph) inputs = tolist(args[:self.nseqs]) recurrence = tolist(args[self.nseqs:self.nseqs+self.nrecur]) inputs += tolist(args[self.nseqs+self.nrecur:self.nseqs+self.noutputs]) nonseqs = tolist(args[self.nseqs+self.noutputs:]) for nname, node in self.nodes.items(): for i, (aname, arg) in enumerate(self.recur_args.items()): if node is arg: node.rec_out = recurrence[i] if len(sorted_nodes) != 0: for node in self.sorted_nodes: inp = [] parent = self.nodes[node].parent for par in parent: tok = 1 for inp2 in inputs: if par in inp2.name: inp.append(inp2) tok = 0 break if tok: inp.append(self.nodes[par].out) if self.nodes[node] in self.recur_args.values(): rec_inp = [] recurrent = self.nodes[node].recurrent for rec in recurrent: rec_inp.append(self.nodes[rec].rec_out) inp = [inp, rec_inp] self.nodes[node].out = self.nodes[node].fprop(inp) next_recurrence.append(self.nodes[node].out) else: self.nodes[node].out = self.nodes[node].fprop(inp) else: # Assume that you have only single depth (parallel) graph # Instead of Queue use for-loop to forcibly run the operation for node in self.nodes: inp = [] parent = self.nodes[node].parent for par in parent: tok = 1 for inp2 in inputs: if par in inp2.name: inp.append(inp2) tok = 0 break if tok: inp.append(self.nodes[par].out) if self.nodes[node] in self.recur_args.values(): rec_inp = [] recurrent = self.nodes[node].recurrent for rec in recurrent: rec_inp.append(self.nodes[rec].rec_out) inp = [inp, rec_inp] self.nodes[node].out = self.nodes[node].fprop(inp) next_recurrence.append(self.nodes[node].out) else: self.nodes[node].out = self.nodes[node].fprop(inp) required_outputs = [] if self.iterators is not None: for arg in self.iterators: for node in self.nodes.values(): if node is arg: required_outputs.append(node.out) if self.output_args is not None: for arg in self.output_args: for node in self.nodes.values(): if node is arg: required_outputs.append(node.out) return next_recurrence + required_outputs
def load(self, data_path): dataset = 'audio.tar.gz' datafile = os.path.join(data_path, dataset) if not os.path.isfile(datafile): try: import urllib urllib.urlretrieve('http://google.com') url =\ 'https://dl.dropboxusercontent.com/u/15378192/audio.tar.gz' except AttributeError: import urllib.request as urllib url =\ 'https://dl.dropboxusercontent.com/u/15378192/audio.tar.gz' print("Downloading data from %s" % url) urllib.urlretrieve(url, datafile) if not os.path.exists(os.path.join(data_path, "audio")): tar = tarfile.open(datafile) os.chdir(data_path) tar.extractall() tar.close() h5_file_path = os.path.join(data_path, "saved_fruit.h5") if not os.path.exists(h5_file_path): data_path = os.path.join(data_path, "audio") audio_matches = [] for root, dirnames, filenames in os.walk(data_path): for filename in fnmatch.filter(filenames, '*.wav'): audio_matches.append(os.path.join(root, filename)) random.seed(1999) random.shuffle(audio_matches) # http://mail.scipy.org/pipermail/numpy-discussion/2011-March/055219.html h5_file = tables.openFile(h5_file_path, mode='w') data_x = h5_file.createVLArray(h5_file.root, 'data_x', tables.Float32Atom(shape=()), filters=tables.Filters(1)) data_y = h5_file.createVLArray(h5_file.root, 'data_y', tables.Int32Atom(shape=()), filters=tables.Filters(1)) for wav_path in audio_matches: # Convert chars to int classes word = wav_path.split(os.sep)[-1][:6] chars = [ord(c) - 97 for c in word] data_y.append(np.array(chars, dtype='int32')) fs, d = wavfile.read(wav_path) data_x.append(d.astype(theano.config.floatX)) h5_file.close() h5_file = tables.openFile(h5_file_path, mode='r') raw_X = np.array([np.asarray(x) for x in h5_file.root.data_x]) cls = np.array([''.join([chr(y+97) for y in Y]) for Y in h5_file.root.data_y]) if self.name != 'all': fruit_list = [] if len(self.name) > 1: for i, fruit_name in enumerate(cls): for name in self.name: if name in fruit_name: fruit_list.append(i) else: for i, fruit_name in enumerate(cls): if self.name in fruit_name: fruit_list.append(i) else: fruit_list = tolist(np.arange(len(raw_X))) raw_X = raw_X[fruit_list] if self.prep == 'normalize': pre_X, self.X_mean, self.X_std = self.global_normalize(raw_X) elif self.prep == 'standardize': pre_X, self.X_max, self.X_min = self.standardize(raw_X) X = np.array([segment_axis(x, self.frame_size, 0) for x in pre_X]) return [X]