def loss(self, loss_object): if isinstance(loss_object, str): self.__loss = Loss(loss_object).loss_layer() elif isinstance(loss_object, LossLayer): self.__loss = loss_object elif isinstance(loss_object, Loss): self.__loss = loss_object.loss_layer() else: raise ValueError('Invalid loss type: {}'.format(type(loss_object)))
class Network: def __init__(self): self.epoch = None self.iteration = None self.epochs = None self.batch_size = None self.engine = Engine() self.__layer = [] self.__layer_name_map = collections.OrderedDict() self.__layer_number_map = collections.OrderedDict() self.__layer_number = 0 self.__input_symbol = Placeholder(name='InputSymbol') self.__current_symbol = self.__input_symbol self.__current_output = None self.__variables = [] self.__data = None self.__optimizer = None self.__loss = None self.__predict_engine = Engine() self.__plugin = collections.OrderedDict() self.load_default_plugin() def __valid_current_output(self): if self.__current_output is None: raise ValueError('Current output is None.') else: return self.__current_output def get_layer(self, name: str): if name in self.__layer_name_map: return self.__layer_name_map[name] else: raise ValueError('No such get_layer in Network: {}'.format(name)) def layer_name_map(self): return self.__layer_name_map def layer_number_map(self): return self.__layer_number_map def add(self, layer, name=None): if isinstance(layer, collections.Iterable): for i, l in enumerate(layer): if name is not None and i < len(name): self.__add(l, name[i]) else: self.__add(l) else: self.__add(layer, name) def __add(self, layer, name: str = None): self.__layer.append(layer) if name is None: name = 'layer_{}'.format(self.__layer_number) if name in self.__layer_name_map: raise ValueError( 'Layer name has contained in Network: {}'.format(name)) else: self.__layer_name_map[name] = layer self.__layer_number_map[self.__layer_number] = layer self.__layer_number += 1 if isinstance(layer, Operator): self.__add_operator(layer) elif isinstance(layer, ConnectionLayer): self.__add_connection(layer) elif isinstance(layer, Connection): self.__add_connection(layer.connection_layer()) elif isinstance(layer, ActivationLayer): self.__add_activation(layer) elif isinstance(layer, Activation): self.__add_activation(layer.activation_layer()) elif isinstance(layer, ConvolutionLayer): self.__add_convolution(layer) elif isinstance(layer, Convolution): self.__add_convolution(layer.convolution_layer()) elif isinstance(layer, PoolingLayer): self.__add_pooling(layer) elif isinstance(layer, Pooling): self.__add_pooling(layer.pooling_layer()) elif isinstance(layer, UnpoolingLayer): self.__add_unpooling(layer) elif isinstance(layer, Unpooling): self.__add_unpooling(layer.unpooling_layer()) else: raise ValueError('Invalid get_layer type: {}'.format(type(layer))) def __add_operator(self, layer: Operator): self.__current_symbol = Symbol(operator=layer, inputs=[self.__current_symbol], category=SymbolCategory.operator) def __add_connection(self, layer: ConnectionLayer): if layer.input_dimension is None: current_output = self.__valid_current_output() if not isinstance(current_output, int): self.__current_symbol = spread(self.__current_symbol, 1 - len(current_output)) current_output = reduce(lambda a, b: a * b, current_output[1:]) layer.input_dimension = current_output weight, bias = layer.weight_bias() self.__variables.append(weight) self.__variables.append(bias) self.__current_symbol = self.__current_symbol @ weight + bias self.__current_output = layer.output_dimension def __add_activation(self, layer: ActivationLayer): self.__current_symbol = layer.activation_function( self.__current_symbol) previous_layer = self.__layer_number_map[self.__layer_number - 2] if isinstance(previous_layer, Connection) or isinstance( previous_layer, ConnectionLayer): previous_layer = previous_layer.connection_layer() weight, bias = previous_layer.weight_bias() weight.value = layer.weight_initialization(weight.value.shape) bias.value = layer.bias_initialization(bias.value.shape) def __add_convolution(self, layer: ConvolutionLayer): kernel = layer.kernel() self.__variables.append(kernel) self.__current_symbol = layer.convolution_function()( self.__current_symbol, kernel, layer.mode) if layer.input_shape is None: layer.input_shape = self.__valid_current_output() self.__current_output = layer.get_output_shape() def __add_pooling(self, layer: PoolingLayer): self.__current_symbol = layer.pooling_function()(self.__current_symbol, layer.size, layer.step) if layer.input_shape is None: layer.input_shape = self.__valid_current_output() self.__current_output = layer.get_output_shape() def __add_unpooling(self, layer: UnpoolingLayer): self.__current_symbol = layer.unpooling_function()( self.__current_symbol, layer.size, layer.step) if layer.input_shape is None: layer.input_shape = self.__valid_current_output() self.__current_output = layer.get_output_shape() def get_symbol(self): return self.__current_symbol def optimizer(self, optimizer_object, *args, **kwargs): if isinstance(optimizer_object, str): name = optimizer_object.lower() if name in optimizer_map: self.__optimizer = optimizer_map[name](*args, **kwargs) else: raise ValueError('No such optimizer: {}'.format(name)) elif isinstance(optimizer_object, Optimizer): self.__optimizer = optimizer_object else: raise ValueError('Invalid optimizer type: {}'.format( type(optimizer_object))) def loss(self, loss_object): if isinstance(loss_object, str): self.__loss = Loss(loss_object).loss_layer() elif isinstance(loss_object, LossLayer): self.__loss = loss_object elif isinstance(loss_object, Loss): self.__loss = loss_object.loss_layer() else: raise ValueError('Invalid loss type: {}'.format(type(loss_object))) def train(self, data, target, epochs: int = 10000, batch_size: int = 0): data = numpy.array(data) target = numpy.array(target) self.epochs = epochs if data.shape[0] != target.shape[0]: raise ValueError( 'Data dimension not match target dimension: {} {}'.format( data.shape[0], target.shape[0])) data_scale = data.shape[0] target_symbol = None if batch_size != 0: loss, target_symbol = self.__loss.loss_function( self.__current_symbol, target[:batch_size], True) else: loss = self.__loss.loss_function(self.__current_symbol, target) self.engine.bind = {self.__input_symbol: data} self.engine.symbol = loss self.engine.variables = self.__variables try: self.iteration = 0 self.run_plugin('begin_training') for epoch in range(self.epochs): self.epoch = epoch + 1 self.run_plugin('begin_epoch') for i in ([0] if batch_size == 0 else range( 0, data_scale, batch_size)): if batch_size != 0: self.engine.bind = { self.__input_symbol: data[i:min([i + batch_size, data_scale])], target_symbol: target[i:min([i + batch_size, data_scale])] } self.iteration += 1 self.run_plugin('begin_iteration') self.__optimizer.minimize(self.engine) self.run_plugin('end_iteration') self.run_plugin('end_epoch') except KeyboardInterrupt: print('Keyboard Interrupt') self.run_plugin('end_training') def predict(self, data): self.__predict_engine.symbol = self.__current_symbol self.__predict_engine.bind = {self.__input_symbol: data} predict_data = self.__predict_engine.value() return predict_data def load_default_plugin(self): for name, plugin, enable in default_plugin: plugin.enable = enable self.add_plugin(name, plugin) def add_plugin(self, name: str, plugin: Plugin): self.__plugin[name.lower()] = plugin plugin.bind_network(self) def run_plugin(self, stage: str): for _, plugin in self.__plugin.items(): if plugin.enable: getattr(plugin, stage)() def plugin(self, name: str): if name.lower() in self.__plugin: return self.__plugin[name.lower()] else: raise ValueError('No such plugin: {}'.format(name))
class Network: def __init__(self): self.epoch = None self.iteration = None self.epochs = None self.batch_size = None self.engine = Engine() self.predict_engine = Engine() self.__layer = [] self.__layer_name_map = collections.OrderedDict() self.__layer_stack = [] self.__input_symbol = Placeholder(name='InputSymbol') self.__current_symbol = self.__input_symbol self.__current_output = None self.__variables = [] self.__layer_weights = {} self.__data = None self.__optimizer = None self.__loss = None self.__regularization_term = None self.__plugin = collections.OrderedDict() self.load_default_plugin() def __valid_current_output(self): if self.__current_output is None: raise ValueError('Current output is None.') else: return self.__current_output def get_layer(self, name: str): if name in self.__layer_name_map: return self.__layer_name_map[name] else: raise ValueError('No such layer in Network named: {}'.format(name)) def layer_name_map(self): return self.__layer_name_map def layer_stack(self): return self.__layer_stack def add(self, layer, name=None): if isinstance(layer, collections.Iterable): for i, l in enumerate(layer): if name is not None and i < len(name): self.__add(l, name[i]) else: self.__add(l) else: self.__add(layer, name) def __add(self, layer, name: str = None): self.__layer.append(layer) if name is None: name = 'layer_{}'.format(len(self.__layer_stack)) if name in self.__layer_name_map: raise ValueError( 'Layer name has contained in Network: {}'.format(name)) else: self.__layer_name_map[name] = layer self.__layer_stack.append(layer) if isinstance(layer, Operator): self.__add_operator(layer, name) elif isinstance(layer, ConnectionLayer): self.__add_connection(layer, name) elif isinstance(layer, Connection): self.__add_connection(layer.connection_layer(), name) elif isinstance(layer, Convolution): self.__add_connection(layer.convolution_layer(), name) elif isinstance(layer, Pooling): self.__add_connection(layer.pooling_layer(), name) elif isinstance(layer, Unpooling): self.__add_connection(layer.unpooling_layer(), name) elif isinstance(layer, ActivationLayer): self.__add_activation(layer, name) elif isinstance(layer, Activation): self.__add_activation(layer.activation_layer(), name) else: raise ValueError('Invalid get_layer type: {}'.format(type(layer))) def __add_operator(self, layer: Operator, name: str = None): self.__current_symbol = Symbol(operator=layer, inputs=[self.__current_symbol], category=SymbolCategory.operator) self.__current_output = layer.shape(self.__current_output) self.__layer_weights[name] = [] def __add_connection(self, layer: ConnectionLayer, name: str = None): if layer.input_shape is None: layer.input_shape = self.__valid_current_output() self.__current_symbol = layer.connection(self.__current_symbol) self.__current_output = layer.output_shape for v in layer.variables(): self.__variables.append(v) self.__layer_weights[name] = layer.weights() def __add_activation(self, layer: ActivationLayer, name: str = None): self.__current_symbol = layer.activation_function( self.__current_symbol) previous_layer = self.__layer_stack[-2] if isinstance(previous_layer, Connection) or isinstance( previous_layer, ConnectionLayer): previous_layer = previous_layer.connection_layer() for weight in previous_layer.weights(): weight.value = layer.weight_initialization(weight.value.shape) for bias in previous_layer.biases(): bias.value = layer.bias_initialization(bias.value.shape) self.__layer_weights[name] = [] def get_symbol(self): return self.__current_symbol def optimizer(self, optimizer_object, *args, **kwargs): if isinstance(optimizer_object, str): name = optimizer_object.lower() if name in optimizer_map: self.__optimizer = optimizer_map[name](*args, **kwargs) else: raise ValueError('No such optimizer: {}'.format(name)) elif isinstance(optimizer_object, Optimizer): self.__optimizer = optimizer_object else: raise ValueError('Invalid optimizer type: {}'.format( type(optimizer_object))) def loss(self, loss_object, *args, **kwargs): if isinstance(loss_object, str): self.__loss = Loss(loss_object, *args, **kwargs).loss_layer() elif isinstance(loss_object, LossLayer): self.__loss = loss_object elif isinstance(loss_object, Loss): self.__loss = loss_object.loss_layer() else: raise ValueError('Invalid loss type: {}'.format(type(loss_object))) def regularization(self, regularization_object, decay: float, name=None, *args, **kwargs): regularization_weights = set() if name is None: for _, weights in self.__layer_weights.items(): regularization_weights |= set(weights) else: if isinstance(name, str): name = [name] if isinstance(name, collections.Iterable): for each_name in name: if each_name in self.__layer_weights: regularization_weights |= set( self.__layer_weights[each_name]) else: raise ValueError( 'No such layer in Network named: {}'.format( each_name)) else: ValueError('Invalid name type: {}'.format(type(name))) if isinstance(regularization_object, str): regularization_function = Regularization( regularization_object, *args, **kwargs).regularization_layer().regularization_term elif isinstance(regularization_object, RegularizationLayer): regularization_function = regularization_object.regularization_term elif isinstance(regularization_object, Regularization): regularization_function = regularization_object.regularization_layer( ).regularization_term else: raise ValueError('Invalid regularization type: {}'.format( type(regularization_object))) for weight in regularization_weights: self.__add_regularization_term( regularization_function(weight, decay)) def __add_regularization_term(self, regularization_term): if self.__regularization_term is None: self.__regularization_term = regularization_term else: self.__regularization_term += regularization_term def train(self, data, target, epochs: int = 10000, batch_size: int = 0): data = numpy.array(data) target = numpy.array(target) self.epochs = epochs if data.shape[0] != target.shape[0]: raise ValueError( 'Data dimension not match target dimension: {} {}'.format( data.shape[0], target.shape[0])) data_scale = data.shape[0] target_symbol = None if batch_size != 0: target_symbol = Placeholder() loss = self.__loss.loss_function(self.__current_symbol, target_symbol) else: loss = self.__loss.loss_function(self.__current_symbol, Constant(target)) self.engine.bind = {self.__input_symbol: data} if self.__regularization_term is not None: loss += self.__regularization_term self.engine.symbol = loss self.engine.variables = self.__variables try: self.iteration = 0 self.run_plugin('begin_training') for epoch in range(self.epochs): self.epoch = epoch + 1 self.run_plugin('begin_epoch') for i in ([0] if batch_size == 0 else range( 0, data_scale, batch_size)): if batch_size != 0: self.engine.bind = { self.__input_symbol: data[i:min([i + batch_size, data_scale])], target_symbol: target[i:min([i + batch_size, data_scale])] } self.run_plugin('begin_iteration') self.__optimizer.minimize(self.engine) self.iteration += 1 self.run_plugin('end_iteration') self.run_plugin('end_epoch') except KeyboardInterrupt: print('Keyboard Interrupt') self.run_plugin('end_training') def predict(self, data): self.predict_engine.symbol = self.__current_symbol self.predict_engine.bind = {self.__input_symbol: data} self.run_plugin('begin_predict') predict_data = self.predict_engine.value() self.run_plugin('end_predict') return predict_data def load_default_plugin(self): for name, plugin, enable in default_plugin: plugin.enable = enable self.add_plugin(name, plugin) def add_plugin(self, name: str, plugin: Plugin): self.__plugin[name.lower()] = plugin plugin.bind_network(self) def run_plugin(self, stage: str): for _, plugin in self.__plugin.items(): if plugin.enable: getattr(plugin, stage)() def plugin(self, name: str): if name.lower() in self.__plugin: return self.__plugin[name.lower()] else: raise ValueError('No such plugin: {}'.format(name))
class Network: def __init__(self): self.__layer = [] self.__input_symbol = Variable(name='InputSymbol') self.__current_symbol = self.__input_symbol self.__current_output = None self.__current_weight = None self.__current_bias = None self.__variables = [] self.__data = None self.__optimizer = None self.__loss = None self.__train_engine = Engine() self.__predict_engine = Engine() def add(self, layers): if isinstance(layers, collections.Iterable): for layer in layers: self.__add(layer) else: self.__add(layers) def __add(self, layer): if isinstance(layer, ConnectionLayer): self.__add_connection(layer) elif isinstance(layer, Connection): self.__add_connection(layer.connection_layer()) elif isinstance(layer, ActivationLayer): self.__add_activation(layer) elif isinstance(layer, Activation): self.__add_activation(layer.activation_layer()) else: raise ValueError('Invalid layer type: {}'.format(type(layer))) def __add_connection(self, layer: ConnectionLayer): weight, bias = layer.weight_bias(self.__current_output) self.__variables.append(weight) self.__variables.append(bias) self.__current_weight = weight self.__current_bias = bias self.__current_symbol = weight @ self.__current_symbol + bias self.__current_output = layer.output_dimension() def __add_activation(self, layer: ActivationLayer): self.__current_symbol = layer.activation_function( self.__current_symbol) self.__current_weight.value = layer.weight_initialization( self.__current_weight.value.shape) self.__current_bias.value = numpy.random.normal( 0, 1, self.__current_bias.value.shape) def get_symbol(self): return self.__current_symbol def optimizer(self, optimizer_object, *args, **kwargs): if isinstance(optimizer_object, str): name = optimizer_object.lower() if name in optimizer_map: self.__optimizer = optimizer_map[name](*args, **kwargs) else: raise ValueError('No such optimizer: {}'.format(name)) elif isinstance(optimizer_object, Optimizer): self.__optimizer = optimizer_object else: raise ValueError('Invalid optimizer type: {}'.format( type(optimizer_object))) def loss(self, loss_object): if isinstance(loss_object, str): self.__loss = Loss(loss_object).loss_layer() elif isinstance(loss_object, LossLayer): self.__loss = loss_object elif isinstance(loss_object, Loss): self.__loss = loss_object.loss_layer() else: raise ValueError('Invalid loss type: {}'.format(type(loss_object))) def train(self, data, target, epochs: int = 10000, loss_threshold: float = 0.001, state_cycle: int = 100): loss = self.__loss.loss_function(self.__current_symbol, target) self.__train_engine.symbol = loss self.__train_engine.variables = self.__variables self.__train_engine.bind = {self.__input_symbol: data} start_time = time.time() cycle_start_time = time.time() for epoch in range(epochs): self.__optimizer.minimize(self.__train_engine) if (epoch + 1) % state_cycle == 0: speed = state_cycle / (time.time() - cycle_start_time) cycle_start_time = time.time() loss_value = self.__train_engine.value() print( 'Training State [epoch = {}/{}, loss = {:.8f}, speed = {:.2f}(epochs/s)' .format(epoch + 1, epochs, loss_value, speed)) if loss_value < loss_threshold: print('Touch loss threshold: {} < {}'.format( loss_value, loss_threshold)) break print('Training Complete [{}]'.format( time.strftime('%H:%M:%S', time.gmtime(time.time() - start_time)))) def predict(self, data): self.__predict_engine.symbol = self.__current_symbol self.__predict_engine.bind = {self.__input_symbol: data} predict_data = self.__predict_engine.value() return predict_data