def get_extensions(self, ext_array, config_id): if ext_array is None: return [] extensions = [] for ext_id in ext_array: row = self.db.executeSQL( """ SELECT ext_class FROM hps3.extension WHERE ext_id = %s """, (ext_id, ), self.db.FETCH_ONE) if not row or row is None: raise HPSData("No extension for ext_id=" + str(ext_id)) ext_class = row[0] fn = getattr(self, 'get_ext_' + ext_class) extensions.append(fn(ext_id)) # monitor based save best if self.mbsb_channel_name is not None: save_path = self.save_prefix + str(config_id) + "_optimum.pkl" extensions.append( MonitorBasedSaveBest(channel_name=self.mbsb_channel_name, save_path=save_path)) # HPS Logger extensions.append(HPSLog(self.log_channel_names, self.db, config_id)) return extensions
def setup(self, trainers): """ Add tracking to all trainers. Parameters ---------- trainers : list List of Train objects belonging to the parent TrainCV object. """ for k, trainer in enumerate(trainers): if self.save_path is not None and self.save_folds: path, ext = os.path.splitext(self.save_path) save_path = path + '-{}'.format(k) + ext else: save_path = None if self.tag_key is not None: tag_key = '{}-{}'.format(self.tag_key, k) else: tag_key = None extension = MonitorBasedSaveBest( self.channel_name, save_path=save_path, store_best_model=True, higher_is_better=self.higher_is_better, tag_key=tag_key) trainer.extensions.append(extension)
def create_training_problem(self, save_best_path): ext1 = MonitorBasedSaveBest(channel_name=self.optimize_for, save_path=save_best_path) trainer = Train(dataset=self.alg_datasets['train'], model=self.model, algorithm=self.algorithm, extensions=[ext1]) self.trainer = trainer
def main(job_id, requested_params, cache): # Fix sub directory problems sys.path.append(os.path.dirname(os.getcwd())) os.chdir(os.path.dirname(os.path.realpath(__file__))) # Add parameters that are not currently being tuned but could potentially be tuned. params = additional_args params.update(requested_params) if params.get('rate', None) is not None: params['log_init_learning_rate'][0] = numpy.array([params['rate']]) train_params = { 'train_start': params['start'], 'train_stop': params['stop'], 'valid_start': 20000, 'valid_stop': 24000, 'test_stop': 4000, 'batch_size': 100, 'max_epochs': 20, 'max_batches': 10, 'sgd_seed': sgd_seed_str, 'mlp_seed': mlp_seed_str, 'weight_decay_y': math.pow(10, params['l_wdecay_y'][0]), 'max_col_norm_y': params['max_norm_y'][0], 'irange_y': math.pow(10, params['l_ir_y'][0]), 'init_momentum': 0.5, 'init_learning_rate': math.pow(10, params['log_init_learning_rate'][0]), } with open('slp_fooddata.yaml', 'r') as f: trainer = f.read() yaml_string = trainer % train_params train_obj = yaml_parse.load(yaml_string) pretrained_model_path = params.get('model', None) if pretrained_model_path is not None: print 'loading pre trained model' pretrained_model = serial.load(pretrained_model_path) print 'loading done' train_obj.model.set_param_values(pretrained_model.get_param_values()) if 'converge' in params: train_obj.algorithm.termination_criterion._criteria[ 0]._max_epochs = 100 train_obj.extensions.append( MonitorBasedSaveBest('valid_y_misclass', params['save'])) train_obj.setup() train_obj.model.monitor.on_channel_conflict = 'ignore' if 'converge' not in params: train_obj.algorithm.termination_criterion._criteria[0].initialize( train_obj.model) train_obj.main_loop(do_setup=False) original_misclass = read_channel(train_obj.model, misclass_channel) return float(original_misclass)
def create_algorithm(self, data, save_best_path=None): self.set_dataset(data) self.create_adjustors() term = EpochCounter(max_epochs=self.max_epochs) if self.valid_stop: cost_crit = MonitorBased(channel_name='valid_objective', prop_decrease=.0, N=3) term = And(criteria=[cost_crit, term]) #(layers, A_weight_decay) coeffs = None if self.reg_factors: rf = self.reg_factors lhdims = len(self.tagger.hdims) l_inputlayer = len(self.tagger.layers[0].layers) coeffs = ([[rf] * l_inputlayer] + ([rf] * lhdims) + [rf], rf) cost = SeqTaggerCost(coeffs, self.dropout) self.cost = cost self.mbsb = MonitorBasedSaveBest(channel_name='valid_objective', save_path=save_best_path) mon_dataset = dict(self.dataset) if not self.monitor_train: del mon_dataset['train'] _learning_rule = (self.momentum_rule if self.use_momentum else None) self.algorithm = SGD( batch_size=1, learning_rate=self.lr, termination_criterion=term, monitoring_dataset=mon_dataset, cost=cost, learning_rule=_learning_rule, ) self.algorithm.setup(self, self.dataset['train']) if self.plot_monitor: cn = ["valid_objective", "test_objective"] if self.monitor_train: cn.append("train_objective") plots = Plots(channel_names=cn, save_path=self.plot_monitor) self.pm = PlotManager([plots], freq=1) self.pm.setup(self, None, self.algorithm)
def get_extensions(self): if 'ext_array' not in self.state: return [] extensions = [] for ext_obj in self.state.ext_array.values(): fn = getattr(self, 'get_ext_' + ext_obj.ext_class) extensions.append(fn(ext_obj)) # monitor based save best print 'save best channel', self.mbsb_channel_name if self.mbsb_channel_name is not None: self.save_path = self.save_prefix + str(self.state.config_id) + "_optimum.pkl" extensions.append(MonitorBasedSaveBest( channel_name = self.mbsb_channel_name, save_path = self.save_path ) ) return extensions
def create_algorithm(self, data, save_best_path=None): self.set_dataset(data) self.create_adjustors() term = EpochCounter(max_epochs=self.max_epochs) if self.valid_stop: cost_crit = MonitorBased(channel_name='valid_objective', prop_decrease=.0, N=3) term = And(criteria=[cost_crit, term]) #(layers, A_weight_decay) coeffs = None if self.reg_factors: rf = self.reg_factors lhdims = len(self.tagger.hdims) l_inputlayer = len(self.tagger.layers[0].layers) coeffs = ([[rf] * l_inputlayer] + ([rf] * lhdims) + [rf], rf) cost = SeqTaggerCost(coeffs, self.dropout) self.cost = cost self.mbsb = MonitorBasedSaveBest(channel_name='valid_objective', save_path=save_best_path) mon_dataset = dict(self.dataset) if not self.monitor_train: del mon_dataset['train'] _learning_rule = (self.momentum_rule if self.use_momentum else None) self.algorithm = SGD(batch_size=1, learning_rate=self.lr, termination_criterion=term, monitoring_dataset=mon_dataset, cost=cost, learning_rule=_learning_rule, ) self.algorithm.setup(self, self.dataset['train']) if self.plot_monitor: cn = ["valid_objective", "test_objective"] if self.monitor_train: cn.append("train_objective") plots = Plots(channel_names=cn, save_path=self.plot_monitor) self.pm = PlotManager([plots], freq=1) self.pm.setup(self, None, self.algorithm)
def get_layer_trainer_sgd(model, trainset): drop_cost = Dropout(input_include_probs={'h0': .4}, input_scales={'h0': 1.}) # configs on sgd train_algo = SGD(train_iteration_mode='batchwise_shuffled_equential', learning_rate=0.2, cost=drop_cost, monitoring_dataset=trainset, termination_criterion=EpochCounter(max_epochs=MAX_EPOCHS), update_callbacks=None) extensions = [ MonitorBasedSaveBest(channel_name="y_kl", save_path="./convnet_test_best.pkl") ] return Train(model=model, algorithm=train_algo, extensions=extensions, dataset=trainset)
def get_trainer(model, trainset, validset, save_path): monitoring = dict(valid=validset, train=trainset) termination = MonitorBased(channel_name='valid_y_misclass', prop_decrease=.001, N=100) extensions = [MonitorBasedSaveBest(channel_name='valid_y_misclass', save_path=save_path), #MomentumAdjustor(start=1, saturate=100, final_momentum=.9), LinearDecayOverEpoch(start=1, saturate=200, decay_factor=0.01)] config = { 'learning_rate': .01, #'learning_rule': Momentum(0.5), 'learning_rule': RMSProp(), 'train_iteration_mode': 'shuffled_sequential', 'batch_size': 1200,#250, #'batches_per_iter' : 100, 'monitoring_dataset': monitoring, 'monitor_iteration_mode' : 'shuffled_sequential', 'termination_criterion' : termination, } return Train(model=model, algorithm=SGD(**config), dataset=trainset, extensions=extensions)
def main(): #creating layers #2 convolutional rectified layers, border mode valid batch_size = 48 lr = 1.0 #0.1/4 finMomentum = 0.9 maxout_units = 2000 num_pcs = 4 lay1_reg = lay2_reg = maxout_reg = None #save_path = './models/no_maxout/titan_lr_0.1_btch_64_momFinal_0.9_maxout_2000_4.joblib' #best_path = '/models/no_maxout/titan_bart10_gpu2_best.joblib' #save_path = './models/'+params.host+'_'+params.device+'_'+sys.argv[1]+'.joblib' #best_path = './models/'+params.host+'_'+params.device+'_'+sys.argv[1]+'best.joblib' save_path = '/Tmp/zumerjer/bart10_sumcost_adadelta_drop_perturb.joblib' best_path = '/Tmp/zumerjer/bart10_sumcost_adadelta_drop_perturb_best.joblib' #numBatches = 400000/batch_size ''' print 'Applying preprocessing' ddmTrain = EmotiwKeypoints(start=0, stop =40000) ddmValid = EmotiwKeypoints(start=40000, stop = 44000) ddmTest = EmotiwKeypoints(start=44000) stndrdz = preprocessing.Standardize() stndrdz.applyLazily(ddmTrain, can_fit=True, name = 'train') stndrdz.applyLazily(ddmValid, can_fit=False, name = 'val') stndrdz.applyLazily(ddmTest, can_fit=False, name = 'test') GCN = preprocessing.GlobalContrastNormalization(batch_size = 1000) GCN.apply(ddmTrain, can_fit =True, name = 'train') GCN.apply(ddmValid, can_fit =False, name = 'val') GCN.apply(ddmTest, can_fit = False, name = 'test') return ''' ddmTrain = ComboDatasetPyTable('/Tmp/zumerjer/perturbed_', which_set='train') ddmValid = ComboDatasetPyTable('/Tmp/zumerjer/perturbed_', which_set='valid') #ddmSmallTrain = ComboDatasetPyTable('/Tmp/zumerjer/all_', which_set='small_train') layer1 = ConvRectifiedLinear(layer_name = 'convRect1', output_channels = 64, irange = .05, kernel_shape = [5, 5], pool_shape = [4, 4], pool_stride = [2, 2], W_lr_scale = 0.1, max_kernel_norm = lay1_reg) layer2 = ConvRectifiedLinear(layer_name = 'convRect2', output_channels = 128, irange = .05, kernel_shape = [5, 5], pool_shape = [3, 3], pool_stride = [2, 2], W_lr_scale = 0.1, max_kernel_norm = lay2_reg) # Rectified linear units #layer3 = RectifiedLinear(dim = 3000, # sparse_init = 15, # layer_name = 'RectLin3') #Maxout layer maxout = Maxout(layer_name= 'maxout', irange= .005, num_units= maxout_units, num_pieces= num_pcs, W_lr_scale = 0.1, max_col_norm= maxout_reg) #multisoftmax n_groups = 196 n_classes = 96 layer_name = 'multisoftmax' layerMS = MultiSoftmax(n_groups=n_groups,irange = 0.05, n_classes=n_classes, layer_name= layer_name) #setting up MLP MLPerc = MLP(batch_size = batch_size, input_space = Conv2DSpace(shape = [96, 96], num_channels = 3, axes=('b', 0, 1, 'c')), layers = [ layer1, layer2, maxout, layerMS]) #mlp_cost missing_target_value = -1 mlp_cost = MLPCost(cost_type='default', missing_target_value=missing_target_value ) mlp_cost.setup_dropout(input_include_probs= { 'convRect1' : 1.0 }, input_scales= { 'convRect1': 1. }) #dropout_cost = Dropout(input_include_probs= { 'convRect1' : .8 }, # input_scales= { 'convRect1': 1. }) #algorithm monitoring_dataset = {'validation':ddmValid}#, 'mini-train':ddmSmallTrain} term_crit = MonitorBased(prop_decrease = 1e-7, N = 100, channel_name = 'validation_objective') kp_ada = KeypointADADELTA(decay_factor = 0.95, #init_momentum = 0.5, monitoring_dataset = monitoring_dataset, batch_size = batch_size, termination_criterion = term_crit, cost = mlp_cost) #train extension #train_ext = ExponentialDecayOverEpoch(decay_factor = 0.998, min_lr_scale = 0.001) #train_ext = LinearDecayOverEpoch(start= 1,saturate= 250,decay_factor= .01) #train_ext = ADADELTA(0.95) #train object train = Train(dataset = ddmTrain, save_path= save_path, save_freq=10, model = MLPerc, algorithm= kp_ada, extensions = [#train_ext, MonitorBasedSaveBest(channel_name='validation_objective', save_path= best_path)#, # MomentumAdjustor(start = 1,# # saturate = 25, # final_momentum = finMomentum) ] ) train.main_loop() train.save()
def test_tagging(): """Test the tagging functionality of this extension.""" try: # TODO: serial.save should be able to take an open file-like object so # we can direct its output to a StringIO or something and not need to # screw around like this in tests that don't actually need to touch # the filesystem. /dev/null would work but the test would fail on # Windows. fd, fn = tempfile.mkstemp(suffix='.pkl') os.close(fd) # Test that the default key gets created. def_model = MockModel() def_model.monitor = Monitor(def_model) def_ext = MonitorBasedSaveBest(channel_name='foobar', save_path=fn) def_ext.setup(def_model, None, None) assert 'MonitorBasedSaveBest' in def_model.tag # Test with a custom key. model = MockModel() model.monitor = Monitor(model) model.monitor.channels['foobar'] = MockChannel() ext = MonitorBasedSaveBest(channel_name='foobar', tag_key='test123', save_path=fn) # Best cost is initially infinity. ext.setup(model, None, None) assert model.tag['test123']['best_cost'] == float("inf") # Best cost after one iteration. model.monitor.channels['foobar'].val_record.append(5.0) model.monitor.report_epoch() ext.on_monitor(model, None, None) assert model.tag['test123']['best_cost'] == 5.0 # Best cost after a second, worse iteration. model.monitor.channels['foobar'].val_record.append(7.0) model.monitor.report_epoch() ext.on_monitor(model, None, None) assert model.tag['test123']['best_cost'] == 5.0 # Best cost after a third iteration better than 2 but worse than 1. model.monitor.channels['foobar'].val_record.append(6.0) model.monitor.report_epoch() ext.on_monitor(model, None, None) assert model.tag['test123']['best_cost'] == 5.0 # Best cost after a fourth, better iteration. model.monitor.channels['foobar'].val_record.append(3.0) model.monitor.report_epoch() ext.on_monitor(model, None, None) assert model.tag['test123']['best_cost'] == 3.0 # setting the starting epoch of saving models. model = MockModel() model.monitor = Monitor(model) model.monitor.channels['foobar'] = MockChannel() ext = MonitorBasedSaveBest(channel_name='foobar', tag_key='test12', start_epoch=4, save_path=fn) ext.setup(model, None, None) assert model.tag['test12']['best_cost'] == float("inf") # Best cost after one iteration. model.monitor.channels['foobar'].val_record.append(5.0) model.monitor.report_epoch() ext.on_monitor(model, None, None) assert model.tag['test12']['best_cost'] == float("inf") # Best cost after a second, better iteration. model.monitor.channels['foobar'].val_record.append(3.0) model.monitor.report_epoch() ext.on_monitor(model, None, None) assert model.tag['test12']['best_cost'] == float("inf") # Best cost after a third, worse iteration. model.monitor.channels['foobar'].val_record.append(7.0) model.monitor.report_epoch() ext.on_monitor(model, None, None) assert model.tag['test12']['best_cost'] == float("inf") # Best cost after a fourth, worse iteration. model.monitor.channels['foobar'].val_record.append(7.0) model.monitor.report_epoch() ext.on_monitor(model, None, None) assert model.tag['test12']['best_cost'] == 7.0 # Best cost after a fifth, worse iteration. model.monitor.channels['foobar'].val_record.append(10.0) model.monitor.report_epoch() ext.on_monitor(model, None, None) assert model.tag['test12']['best_cost'] == 7.0 # Best cost after a fifth, better iteration. model.monitor.channels['foobar'].val_record.append(1.0) model.monitor.report_epoch() ext.on_monitor(model, None, None) assert model.tag['test12']['best_cost'] == 1.0 finally: os.remove(fn)
algorithm = SGD(learning_rate=0.01, cost=MethodCost("cost_from_X"), batch_size=batchSize, monitoring_batch_size=batchSize, monitoring_dataset={ 'train': train, 'valid': valid }, monitor_iteration_mode="even_batchwise_shuffled_sequential", termination_criterion=EpochCounter(max_epochs=200), learning_rule=Momentum(init_momentum=0.99), train_iteration_mode="even_batchwise_shuffled_sequential") train = Train(dataset=train, model=model, algorithm=algorithm, save_path="ConvNet4.pkl", save_freq=1, extensions=[ MonitorBasedSaveBest(channel_name="valid_y_misclass", save_path="ConvNet4_best.pkl"), MomentumAdjustor(final_momentum=0, start=0, saturate=100) ]) print("Starting training session") train.main_loop() print("Done!")
def set_extensions(self, extensions=None): self.extensions = [MonitorBasedSaveBest(channel_name='test_objective', save_path = './training/training_monitor_best.pkl')]
class SequenceTaggerNetwork(Model): def __init__(self, dataset, w2i, t2i, featurizer, edim=None, hdims=None, fedim=None, max_epochs=100, use_momentum=False, lr=.01, lr_lin_decay=None, lr_scale=False, lr_monitor_decay=False, valid_stop=False, reg_factors=None, dropout=False, dropout_params=None, embedding_init=None, embedded_model=None, monitor_train=True, plot_monitor=None, num=False): super(SequenceTaggerNetwork, self).__init__() self.vocab_size = dataset.vocab_size self.window_size = dataset.window_size self.total_feats = dataset.total_feats self.feat_num = dataset.feat_num self.n_classes = dataset.n_classes self.max_epochs = max_epochs if edim is None: edim = 50 if hdims is None: hdims = [100] if fedim is None: fedim = 5 self.edim = edim self.fedim = fedim self.hdims = hdims self.w2i = w2i self.t2i = t2i self.featurizer = featurizer self._create_tagger() A_value = numpy.random.uniform(low=-.1, high=.1, size=(self.n_classes + 2, self.n_classes)) self.A = sharedX(A_value, name='A') self.use_momentum = use_momentum self.lr = lr self.lr_lin_decay = lr_lin_decay self.lr_monitor_decay = lr_monitor_decay self.lr_scale = lr_scale self.valid_stop = valid_stop self.reg_factors = reg_factors self.close_cache = {} self.dropout_params = dropout_params self.dropout = dropout or self.dropout_params is not None self.hdims = hdims self.monitor_train = monitor_train self.num = num self.plot_monitor = plot_monitor if embedding_init is not None: self.set_embedding_weights(embedding_init) def _create_tagger(self): self.tagger = WordTaggerNetwork(self.vocab_size, self.window_size, self.total_feats, self.feat_num, self.hdims, self.edim, self.fedim, self.n_classes) def _create_data_specs(self, dataset): self.input_space = CompositeSpace([ dataset.data_specs[0].components[i] for i in xrange(len(dataset.data_specs[0].components) - 1) ]) self.output_space = dataset.data_specs[0].components[-1] self.input_source = dataset.data_specs[1][:-1] self.target_source = dataset.data_specs[1][-1] def __getstate__(self): d = {} d['vocab_size'] = self.vocab_size d['window_size'] = self.window_size d['feat_num'] = self.feat_num d['total_feats'] = self.total_feats d['n_classes'] = self.n_classes d['input_space'] = self.input_space d['output_space'] = self.output_space d['input_source'] = self.input_source d['target_source'] = self.target_source d['A'] = self.A d['tagger'] = self.tagger d['w2i'] = self.w2i d['t2i'] = self.t2i d['featurizer'] = self.featurizer d['max_epochs'] = self.max_epochs d['use_momentum'] = self.use_momentum d['lr'] = self.lr d['lr_lin_decay'] = self.lr_lin_decay d['lr_monitor_decay'] = self.lr_monitor_decay d['lr_scale'] = self.lr_scale d['valid_stop'] = self.valid_stop d['reg_factors'] = self.reg_factors d['dropout'] = self.dropout d['dropout_params'] = self.dropout_params d['monitor_train'] = self.monitor_train d['num'] = self.num d['plot_monitor'] = self.plot_monitor return d def fprop(self, data): tagger_out = self.tagger.fprop(data) probs = T.concatenate([self.A, tagger_out]) return probs def dropout_fprop(self, data, default_input_include_prob=0.5, input_include_probs=None, default_input_scale=2.0, input_scales=None, per_example=True): if input_scales is None: input_scales = {'input': 1.0} if input_include_probs is None: input_include_probs = {'input': 1.0} if self.dropout_params is not None: if len(self.dropout_params) == len(self.tagger.layers) - 1: input_include_probs['tagger_out'] = self.dropout_params[-1] input_scales['tagger_out'] = 1.0 / self.dropout_params[-1] for i, p in enumerate(self.dropout_params[:-1]): input_include_probs['h{0}'.format(i)] = p input_scales['h{0}'.format(i)] = 1.0 / p tagger_out = self.tagger.dropout_fprop(data, default_input_include_prob, input_include_probs, default_input_scale, input_scales, per_example) probs = T.concatenate([self.A, tagger_out]) return probs @functools.wraps(Model.get_lr_scalers) def get_lr_scalers(self): if not self.lr_scale: return {} d = self.tagger.get_lr_scalers() d[self.A] = 1. / self.n_classes return d @functools.wraps(Model.get_params) def get_params(self): return self.tagger.get_params() + [self.A] def create_adjustors(self): initial_momentum = .5 final_momentum = .99 start = 1 saturate = self.max_epochs self.momentum_adjustor = learning_rule.MomentumAdjustor( final_momentum, start, saturate) self.momentum_rule = learning_rule.Momentum(initial_momentum, nesterov_momentum=True) if self.lr_monitor_decay: self.learning_rate_adjustor = MonitorBasedLRAdjuster( high_trigger=1., shrink_amt=0.9, low_trigger=.95, grow_amt=1.1, channel_name='train_objective') elif self.lr_lin_decay: self.learning_rate_adjustor = LinearDecayOverEpoch( start, saturate, self.lr_lin_decay) def compute_used_inputs(self): seen = {'words': set(), 'feats': set()} for sen_w in self.dataset['train'].X1: seen['words'] |= reduce(lambda x, y: set(x) | set(y), sen_w, set()) for sen_f in self.dataset['train'].X2: seen['feats'] |= reduce(lambda x, y: set(x) | set(y), sen_f, set()) words = set(xrange(len(self.w2i))) feats = set(xrange(self.total_feats)) self.notseen = { 'words': numpy.array(sorted(words - seen['words'])), 'feats': numpy.array(sorted(feats - seen['feats'])) } def set_dataset(self, data): self._create_data_specs(data['train']) self.dataset = data self.compute_used_inputs() self.tagger.notseen = self.notseen def create_algorithm(self, data, save_best_path=None): self.set_dataset(data) self.create_adjustors() term = EpochCounter(max_epochs=self.max_epochs) if self.valid_stop: cost_crit = MonitorBased(channel_name='valid_objective', prop_decrease=.0, N=3) term = And(criteria=[cost_crit, term]) #(layers, A_weight_decay) coeffs = None if self.reg_factors: rf = self.reg_factors lhdims = len(self.tagger.hdims) l_inputlayer = len(self.tagger.layers[0].layers) coeffs = ([[rf] * l_inputlayer] + ([rf] * lhdims) + [rf], rf) cost = SeqTaggerCost(coeffs, self.dropout) self.cost = cost self.mbsb = MonitorBasedSaveBest(channel_name='valid_objective', save_path=save_best_path) mon_dataset = dict(self.dataset) if not self.monitor_train: del mon_dataset['train'] _learning_rule = (self.momentum_rule if self.use_momentum else None) self.algorithm = SGD( batch_size=1, learning_rate=self.lr, termination_criterion=term, monitoring_dataset=mon_dataset, cost=cost, learning_rule=_learning_rule, ) self.algorithm.setup(self, self.dataset['train']) if self.plot_monitor: cn = ["valid_objective", "test_objective"] if self.monitor_train: cn.append("train_objective") plots = Plots(channel_names=cn, save_path=self.plot_monitor) self.pm = PlotManager([plots], freq=1) self.pm.setup(self, None, self.algorithm) def train(self): while True: if not self.algorithm.continue_learning(self): break self.algorithm.train(dataset=self.dataset['train']) self.monitor.report_epoch() self.monitor() self.mbsb.on_monitor(self, self.dataset['valid'], self.algorithm) if self.use_momentum: self.momentum_adjustor.on_monitor(self, self.dataset['valid'], self.algorithm) if hasattr(self, 'learning_rate_adjustor'): self.learning_rate_adjustor.on_monitor(self, self.dataset['valid'], self.algorithm) if hasattr(self, 'pm'): self.pm.on_monitor(self, self.dataset['valid'], self.algorithm) def prepare_tagging(self): X = self.get_input_space().make_theano_batch(batch_size=1) Y = self.fprop(X) self.f = theano.function([X[0], X[1]], Y) self.start = self.A.get_value()[0] self.end = self.A.get_value()[1] self.A_value = self.A.get_value()[2:] def process_input(self, words, feats): return self.f(words, feats) def tag_sen(self, words, feats, debug=False, return_probs=False): if not hasattr(self, 'f'): self.prepare_tagging() y = self.process_input(words, feats) tagger_out = y[2 + self.n_classes:] res = viterbi(self.start, self.A_value, self.end, tagger_out, self.n_classes, return_probs) if return_probs: return res / res.sum(axis=1)[:, numpy.newaxis] #return res.reshape((1, len(res))) if debug: return numpy.array([[e] for e in res[1]]), tagger_out return numpy.array([[e] for e in res[1]]) def get_score(self, dataset, mode='pwp'): self.prepare_tagging() tagged = (self.tag_sen(w, f) for w, f in izip(dataset.X1, dataset.X2)) gold = dataset.y good, bad = 0., 0. if mode == 'pwp': for t, g in izip(tagged, gold): g = g.argmax(axis=1) t = t.flatten() good += sum(t == g) bad += sum(t != g) return [good / (good + bad)] elif mode == 'f1': i2t = [t for t, i in sorted(self.t2i.items(), key=lambda x: x[1])] f1c = FScCounter(i2t, binary_input=False) gold = map(lambda x: x.argmax(axis=1), gold) tagged = map(lambda x: x.flatten(), tagged) return f1c.count_score(gold, tagged) def set_embedding_weights(self, embedding_init): # load embedding with gensim from gensim.models import Word2Vec try: m = Word2Vec.load_word2vec_format(embedding_init, binary=False) edim = m.layer1_size except UnicodeDecodeError: try: m = Word2Vec.load_word2vec_format(embedding_init, binary=True) edim = m.layer1_size except UnicodeDecodeError: # not in word2vec format m = Word2Vec.load(embedding_init) edim = m.layer1_size except ValueError: # glove model m = {} if embedding_init.endswith('gz'): fp = gzip.open(embedding_init) else: fp = open(embedding_init) for l in fp: le = l.split() m[le[0].decode('utf-8')] = numpy.array( [float(e) for e in le[1:]], dtype=theano.config.floatX) edim = len(le) - 1 if edim != self.edim: raise Exception("Embedding dim and edim doesn't match") m_lower = {} vocab = (m.vocab if hasattr(m, 'vocab') else m) for k in vocab: if k in ['UNKNOWN', 'PADDING']: continue if self.num: m_lower[replace_numerals(k.lower())] = m[k] else: m_lower[k.lower()] = m[k] # transform weight matrix with using self.w2i params = numpy.zeros( self.tagger.layers[0].layers[0].get_param_vector().shape, dtype=theano.config.floatX) e = self.edim for w in self.w2i: if w in m_lower: v = m_lower[w] i = self.w2i[w] params[i * e:(i + 1) * e] = v if 'UNKNOWN' in vocab: params[-1 * e:] = vocab['UNKNOWN'] if 'PADDING' in vocab: params[-2 * e:-1 * e] = vocab['PADDING'] self.tagger.layers[0].layers[0].set_param_vector(params)
def main(job_id, params, cache): # Fix sub directory problems sys.path.append(os.path.dirname(os.getcwd())) os.chdir(os.path.dirname(os.path.realpath(__file__))) # Add parameters that are not currently being tuned but could potentially be tuned. params.update(additional_args) fixed_params = (params['kernel_size_h2'][0], params['kernel_size_h3'][0]) if 'cached_trainer' + str(fixed_params) not in cache: train_params = { 'train_stop': 20000, 'valid_stop': 24000, 'test_stop': 4000, 'batch_size': 100, 'max_epochs': 1, 'max_batches': 10, 'sgd_seed': sgd_seed_str, 'mlp_seed': mlp_seed_str, 'save_file': 'result', 'kernel_size_h2': int(params['kernel_size_h2'][0]), 'output_channels_h2': 1 * k, 'irange_h2': math.pow(10, params['l_ir_h2'][0]), 'max_kernel_norm_h2': params['max_norm_h2'][0], 'kernel_size_h3': int(params['kernel_size_h3'][0]), 'output_channels_h3': int(1.7 * k), 'irange_h3': math.pow(10, params['l_ir_h3'][0]), 'max_kernel_norm_h3': params['max_norm_h3'][0], 'kernel_size_h4': int(params['kernel_size_h4'][0]), 'output_channels_h4': int(2.5 * k), 'irange_h4': math.pow(10, params['l_ir_h4'][0]), 'max_kernel_norm_h4': params['max_norm_h4'][0], 'weight_decay_h2': math.pow(10, params['l_wdecay_h2'][0]), 'weight_decay_h3': math.pow(10, params['l_wdecay_h3'][0]), 'weight_decay_h4': math.pow(10, params['l_wdecay_h4'][0]), 'weight_decay_y': math.pow(10, params['l_wdecay_y'][0]), 'max_col_norm_y': params['max_norm_y'][0], 'irange_y': math.pow(10, params['l_ir_y'][0]), 'init_learning_rate': math.pow(10, params['log_init_learning_rate'][0]), 'init_momentum': params['init_momentum'][0], 'rectifier_left_slope': 0.2 } with open('conv_fooddata_spearmint.yaml', 'r') as f: trainer = f.read() yaml_string = trainer % train_params train_obj = yaml_parse.load(yaml_string) if 'converge' in params: del train_obj.algorithm.termination_criterion._criteria[:] train_obj.extensions.append(MonitorBasedSaveBest('valid_y_misclass', 'best_model.pkl')) train_obj.setup() train_obj.model.monitor.on_channel_conflict = 'ignore' cache['cached_trainer' + str(fixed_params)] = train_obj else: train_obj = cache['cached_trainer' + str(fixed_params)] train_obj.model.monitor.set_state([0, 0, 0]) train_obj.model.training_succeeded = False # train_obj.algorithm.update_callbacks[0].reinit_from_monitor() model = train_obj.model model_params = dict([(param.name, param) for param in model.get_params()]) rng = model.rng update_conv_layer(model.layers[0], params['l_ir_h2'][0], params['max_norm_h2'][0], model_params, rng) update_conv_layer(model.layers[1], params['l_ir_h3'][0], params['max_norm_h3'][0], model_params, rng) update_conv_layer(model.layers[2], params['l_ir_h4'][0], params['max_norm_h4'][0], model_params, rng) update_softmax_layer(model.layers[3], params['l_ir_y'][0], params['max_norm_y'][0], model_params, rng) train_obj.algorithm.learning_rate.set_value( math.pow(10, params['log_init_learning_rate'][0].astype(numpy.float32))) train_obj.algorithm.learning_rule.momentum.set_value(params['init_momentum'][0].astype(numpy.float32)) pass if 'converge' not in params: train_obj.algorithm.termination_criterion._criteria[0].initialize(train_obj.model) train_obj.main_loop(do_setup=False) original_misclass = read_channel(train_obj.model, misclass_channel) return float(original_misclass) * 50
def on_monitor(self, model, dataset, algorithm): if self.epoch > self.k: return MonitorBasedSaveBest.on_monitor(self, model, dataset, algorithm) self.epoch += 1
def __init__(self, k, channel_name, save_path): MonitorBasedSaveBest.__init__(self, channel_name, save_path) self.k = k self.epoch = 0
class SequenceTaggerNetwork(Model): def __init__(self, dataset, w2i, t2i, featurizer, edim=None, hdims=None, fedim=None, max_epochs=100, use_momentum=False, lr=.01, lr_lin_decay=None, lr_scale=False, lr_monitor_decay=False, valid_stop=False, reg_factors=None, dropout=False, dropout_params=None, embedding_init=None, embedded_model=None, monitor_train=True, plot_monitor=None, num=False): super(SequenceTaggerNetwork, self).__init__() self.vocab_size = dataset.vocab_size self.window_size = dataset.window_size self.total_feats = dataset.total_feats self.feat_num = dataset.feat_num self.n_classes = dataset.n_classes self.max_epochs = max_epochs if edim is None: edim = 50 if hdims is None: hdims = [100] if fedim is None: fedim = 5 self.edim = edim self.fedim = fedim self.hdims = hdims self.w2i = w2i self.t2i = t2i self.featurizer = featurizer self._create_tagger() A_value = numpy.random.uniform(low=-.1, high=.1, size=(self.n_classes + 2, self.n_classes)) self.A = sharedX(A_value, name='A') self.use_momentum = use_momentum self.lr = lr self.lr_lin_decay = lr_lin_decay self.lr_monitor_decay = lr_monitor_decay self.lr_scale = lr_scale self.valid_stop = valid_stop self.reg_factors = reg_factors self.close_cache = {} self.dropout_params = dropout_params self.dropout = dropout or self.dropout_params is not None self.hdims = hdims self.monitor_train = monitor_train self.num = num self.plot_monitor = plot_monitor if embedding_init is not None: self.set_embedding_weights(embedding_init) def _create_tagger(self): self.tagger = WordTaggerNetwork( self.vocab_size, self.window_size, self.total_feats, self.feat_num, self.hdims, self.edim, self.fedim, self.n_classes) def _create_data_specs(self, dataset): self.input_space = CompositeSpace([ dataset.data_specs[0].components[i] for i in xrange(len(dataset.data_specs[0].components) - 1)]) self.output_space = dataset.data_specs[0].components[-1] self.input_source = dataset.data_specs[1][:-1] self.target_source = dataset.data_specs[1][-1] def __getstate__(self): d = {} d['vocab_size'] = self.vocab_size d['window_size'] = self.window_size d['feat_num'] = self.feat_num d['total_feats'] = self.total_feats d['n_classes'] = self.n_classes d['input_space'] = self.input_space d['output_space'] = self.output_space d['input_source'] = self.input_source d['target_source'] = self.target_source d['A'] = self.A d['tagger'] = self.tagger d['w2i'] = self.w2i d['t2i'] = self.t2i d['featurizer'] = self.featurizer d['max_epochs'] = self.max_epochs d['use_momentum'] = self.use_momentum d['lr'] = self.lr d['lr_lin_decay'] = self.lr_lin_decay d['lr_monitor_decay'] = self.lr_monitor_decay d['lr_scale'] = self.lr_scale d['valid_stop'] = self.valid_stop d['reg_factors'] = self.reg_factors d['dropout'] = self.dropout d['dropout_params'] = self.dropout_params d['monitor_train'] = self.monitor_train d['num'] = self.num d['plot_monitor'] = self.plot_monitor return d def fprop(self, data): tagger_out = self.tagger.fprop(data) probs = T.concatenate([self.A, tagger_out]) return probs def dropout_fprop(self, data, default_input_include_prob=0.5, input_include_probs=None, default_input_scale=2.0, input_scales=None, per_example=True): if input_scales is None: input_scales = {'input': 1.0} if input_include_probs is None: input_include_probs = {'input': 1.0} if self.dropout_params is not None: if len(self.dropout_params) == len(self.tagger.layers) - 1: input_include_probs['tagger_out'] = self.dropout_params[-1] input_scales['tagger_out'] = 1.0/self.dropout_params[-1] for i, p in enumerate(self.dropout_params[:-1]): input_include_probs['h{0}'.format(i)] = p input_scales['h{0}'.format(i)] = 1.0/p tagger_out = self.tagger.dropout_fprop( data, default_input_include_prob, input_include_probs, default_input_scale, input_scales, per_example) probs = T.concatenate([self.A, tagger_out]) return probs @functools.wraps(Model.get_lr_scalers) def get_lr_scalers(self): if not self.lr_scale: return {} d = self.tagger.get_lr_scalers() d[self.A] = 1. / self.n_classes return d @functools.wraps(Model.get_params) def get_params(self): return self.tagger.get_params() + [self.A] def create_adjustors(self): initial_momentum = .5 final_momentum = .99 start = 1 saturate = self.max_epochs self.momentum_adjustor = learning_rule.MomentumAdjustor( final_momentum, start, saturate) self.momentum_rule = learning_rule.Momentum(initial_momentum, nesterov_momentum=True) if self.lr_monitor_decay: self.learning_rate_adjustor = MonitorBasedLRAdjuster( high_trigger=1., shrink_amt=0.9, low_trigger=.95, grow_amt=1.1, channel_name='train_objective') elif self.lr_lin_decay: self.learning_rate_adjustor = LinearDecayOverEpoch( start, saturate, self.lr_lin_decay) def compute_used_inputs(self): seen = {'words': set(), 'feats': set()} for sen_w in self.dataset['train'].X1: seen['words'] |= reduce( lambda x, y: set(x) | set(y), sen_w, set()) for sen_f in self.dataset['train'].X2: seen['feats'] |= reduce( lambda x, y: set(x) | set(y), sen_f, set()) words = set(xrange(len(self.w2i))) feats = set(xrange(self.total_feats)) self.notseen = { 'words': numpy.array(sorted(words - seen['words'])), 'feats': numpy.array(sorted(feats - seen['feats'])) } def set_dataset(self, data): self._create_data_specs(data['train']) self.dataset = data self.compute_used_inputs() self.tagger.notseen = self.notseen def create_algorithm(self, data, save_best_path=None): self.set_dataset(data) self.create_adjustors() term = EpochCounter(max_epochs=self.max_epochs) if self.valid_stop: cost_crit = MonitorBased(channel_name='valid_objective', prop_decrease=.0, N=3) term = And(criteria=[cost_crit, term]) #(layers, A_weight_decay) coeffs = None if self.reg_factors: rf = self.reg_factors lhdims = len(self.tagger.hdims) l_inputlayer = len(self.tagger.layers[0].layers) coeffs = ([[rf] * l_inputlayer] + ([rf] * lhdims) + [rf], rf) cost = SeqTaggerCost(coeffs, self.dropout) self.cost = cost self.mbsb = MonitorBasedSaveBest(channel_name='valid_objective', save_path=save_best_path) mon_dataset = dict(self.dataset) if not self.monitor_train: del mon_dataset['train'] _learning_rule = (self.momentum_rule if self.use_momentum else None) self.algorithm = SGD(batch_size=1, learning_rate=self.lr, termination_criterion=term, monitoring_dataset=mon_dataset, cost=cost, learning_rule=_learning_rule, ) self.algorithm.setup(self, self.dataset['train']) if self.plot_monitor: cn = ["valid_objective", "test_objective"] if self.monitor_train: cn.append("train_objective") plots = Plots(channel_names=cn, save_path=self.plot_monitor) self.pm = PlotManager([plots], freq=1) self.pm.setup(self, None, self.algorithm) def train(self): while True: if not self.algorithm.continue_learning(self): break self.algorithm.train(dataset=self.dataset['train']) self.monitor.report_epoch() self.monitor() self.mbsb.on_monitor(self, self.dataset['valid'], self.algorithm) if self.use_momentum: self.momentum_adjustor.on_monitor(self, self.dataset['valid'], self.algorithm) if hasattr(self, 'learning_rate_adjustor'): self.learning_rate_adjustor.on_monitor( self, self.dataset['valid'], self.algorithm) if hasattr(self, 'pm'): self.pm.on_monitor( self, self.dataset['valid'], self.algorithm) def prepare_tagging(self): X = self.get_input_space().make_theano_batch(batch_size=1) Y = self.fprop(X) self.f = theano.function([X[0], X[1]], Y) self.start = self.A.get_value()[0] self.end = self.A.get_value()[1] self.A_value = self.A.get_value()[2:] def process_input(self, words, feats): return self.f(words, feats) def tag_sen(self, words, feats, debug=False, return_probs=False): if not hasattr(self, 'f'): self.prepare_tagging() y = self.process_input(words, feats) tagger_out = y[2 + self.n_classes:] res = viterbi(self.start, self.A_value, self.end, tagger_out, self.n_classes, return_probs) if return_probs: return res / res.sum(axis=1)[:,numpy.newaxis] #return res.reshape((1, len(res))) if debug: return numpy.array([[e] for e in res[1]]), tagger_out return numpy.array([[e] for e in res[1]]) def get_score(self, dataset, mode='pwp'): self.prepare_tagging() tagged = (self.tag_sen(w, f) for w, f in izip(dataset.X1, dataset.X2)) gold = dataset.y good, bad = 0., 0. if mode == 'pwp': for t, g in izip(tagged, gold): g = g.argmax(axis=1) t = t.flatten() good += sum(t == g) bad += sum(t != g) return [good / (good + bad)] elif mode == 'f1': i2t = [t for t, i in sorted(self.t2i.items(), key=lambda x: x[1])] f1c = FScCounter(i2t, binary_input=False) gold = map(lambda x:x.argmax(axis=1), gold) tagged = map(lambda x:x.flatten(), tagged) return f1c.count_score(gold, tagged) def set_embedding_weights(self, embedding_init): # load embedding with gensim from gensim.models import Word2Vec try: m = Word2Vec.load_word2vec_format(embedding_init, binary=False) edim = m.layer1_size except UnicodeDecodeError: try: m = Word2Vec.load_word2vec_format(embedding_init, binary=True) edim = m.layer1_size except UnicodeDecodeError: # not in word2vec format m = Word2Vec.load(embedding_init) edim = m.layer1_size except ValueError: # glove model m = {} if embedding_init.endswith('gz'): fp = gzip.open(embedding_init) else: fp = open(embedding_init) for l in fp: le = l.split() m[le[0].decode('utf-8')] = numpy.array( [float(e) for e in le[1:]], dtype=theano.config.floatX) edim = len(le) - 1 if edim != self.edim: raise Exception("Embedding dim and edim doesn't match") m_lower = {} vocab = (m.vocab if hasattr(m, 'vocab') else m) for k in vocab: if k in ['UNKNOWN', 'PADDING']: continue if self.num: m_lower[replace_numerals(k.lower())] = m[k] else: m_lower[k.lower()] = m[k] # transform weight matrix with using self.w2i params = numpy.zeros( self.tagger.layers[0].layers[0].get_param_vector().shape, dtype=theano.config.floatX) e = self.edim for w in self.w2i: if w in m_lower: v = m_lower[w] i = self.w2i[w] params[i*e:(i+1)*e] = v if 'UNKNOWN' in vocab: params[-1*e:] = vocab['UNKNOWN'] if 'PADDING' in vocab: params[-2*e:-1*e] = vocab['PADDING'] self.tagger.layers[0].layers[0].set_param_vector(params)
model = MLP(layers = [h0, h1, y], batch_size = batchSize, input_space = inputSpace) algorithm = SGD(learning_rate = 1e-3, cost = MethodCost("cost_from_X"), batch_size = batchSize, monitoring_batch_size = batchSize, monitoring_dataset = {'train': train, 'valid':valid}, monitor_iteration_mode = "even_batchwise_shuffled_sequential", termination_criterion = EpochCounter(max_epochs = 100), learning_rule = Momentum(init_momentum = 0.0), train_iteration_mode = "even_batchwise_shuffled_sequential") train = Train(dataset = train, model = model, algorithm = algorithm, save_path = "2_layer_conv.pkl", save_freq = 1, extensions = [ MonitorBasedSaveBest(channel_name = "valid_y_misclass", save_path = "2_layer_conv_best.pkl") ]) print("Starting training session") train.main_loop() print("Done!")
def main(): #creating layers #2 convolutional rectified layers, border mode valid batch_size = params.batch_size lr = params.lr finMomentum = params.momentum maxout_units = params.units num_pcs = params.pieces lay1_reg = lay2_reg = maxout_reg = params.norm_reg #save_path = './models/no_maxout/titan_lr_0.1_btch_64_momFinal_0.9_maxout_2000_4.joblib' #best_path = '/models/no_maxout/titan_bart10_gpu2_best.joblib' save_path = './models/' + params.host + '_' + params.device + '_' + sys.argv[ 1] + '.joblib' best_path = './models/' + params.host + '_' + params.device + '_' + sys.argv[ 1] + 'best.joblib' numBatches = 400000 / batch_size from emotiw.common.datasets.faces.EmotiwKeypoints import EmotiwKeypoints ''' print 'Applying preprocessing' ddmTrain = EmotiwKeypoints(start=0, stop =40000) ddmValid = EmotiwKeypoints(start=40000, stop = 44000) ddmTest = EmotiwKeypoints(start=44000) stndrdz = preprocessing.Standardize() stndrdz.applyLazily(ddmTrain, can_fit=True, name = 'train') stndrdz.applyLazily(ddmValid, can_fit=False, name = 'val') stndrdz.applyLazily(ddmTest, can_fit=False, name = 'test') GCN = preprocessing.GlobalContrastNormalization(batch_size = 1000) GCN.apply(ddmTrain, can_fit =True, name = 'train') GCN.apply(ddmValid, can_fit =False, name = 'val') GCN.apply(ddmTest, can_fit = False, name = 'test') return ''' ddmTrain = EmotiwKeypoints(hack='train', preproc='STD') ddmValid = EmotiwKeypoints(hack='val', preproc='STD') layer1 = ConvRectifiedLinear(layer_name='convRect1', output_channels=64, irange=.05, kernel_shape=[5, 5], pool_shape=[4, 4], pool_stride=[2, 2], W_lr_scale=0.1, max_kernel_norm=lay1_reg) layer2 = ConvRectifiedLinear(layer_name='convRect2', output_channels=128, irange=.05, kernel_shape=[5, 5], pool_shape=[3, 3], pool_stride=[2, 2], W_lr_scale=0.1, max_kernel_norm=lay2_reg) # Rectified linear units #layer3 = RectifiedLinear(dim = 3000, # sparse_init = 15, # layer_name = 'RectLin3') #Maxout layer maxout = Maxout(layer_name='maxout', irange=.005, num_units=maxout_units, num_pieces=num_pcs, W_lr_scale=0.1, max_col_norm=maxout_reg) #multisoftmax n_groups = 196 n_classes = 96 irange = 0 layer_name = 'multisoftmax' layerMS = MultiSoftmax(n_groups=n_groups, irange=0.05, n_classes=n_classes, layer_name=layer_name) #setting up MLP MLPerc = MLP(batch_size=batch_size, input_space=Conv2DSpace(shape=[96, 96], num_channels=3), layers=[layer1, layer2, maxout, layerMS]) #mlp_cost missing_target_value = -1 mlp_cost = MLPCost(cost_type='default', missing_target_value=missing_target_value) mlp_cost.setup_dropout(input_include_probs={'convRect1': 1.0}, input_scales={'convRect1': 1.}) #dropout_cost = Dropout(input_include_probs= { 'convRect1' : .8 }, # input_scales= { 'convRect1': 1. }) #algorithm monitoring_dataset = {'validation': ddmValid} term_crit = MonitorBased(prop_decrease=1e-7, N=100, channel_name='validation_objective') kpSGD = KeypointSGD(learning_rate=lr, init_momentum=0.5, monitoring_dataset=monitoring_dataset, batch_size=batch_size, termination_criterion=term_crit, cost=mlp_cost) #train extension #train_ext = ExponentialDecayOverEpoch(decay_factor = 0.998, min_lr_scale = 0.001) train_ext = LinearDecayOverEpoch(start=1, saturate=250, decay_factor=.01) #train object train = Train(dataset=ddmTrain, save_path=save_path, save_freq=10, model=MLPerc, algorithm=kpSGD, extensions=[ train_ext, MonitorBasedSaveBest(channel_name='validation_objective', save_path=best_path), MomentumAdjustor(start=1, saturate=25, final_momentum=finMomentum) ]) train.main_loop() train.save()
train = Train(dataset, mlp, SGD(0.1, batch_size=128, monitoring_dataset={ 'train': dataset, 'valid': dataset_valid, 'test': dataset_test }, termination_criterion=EpochCounter(100), train_iteration_mode='even_shuffled_sequential', monitor_iteration_mode='even_sequential'), save_path="pkl/multicolumn.pkl", save_freq=5, extensions=[ MonitorBasedSaveBest(channel_name='test_y_misclass', save_path="pkl/multicolumn_best.pkl") ]) # # Load the saved model # model = serial.load(saved_model_path) # # # Remove last layer # del model.layers[-1] # # # Add new layer # new_output_layer = <make your new layer here> # model.add_layers([new_output_layer]) # mlp.layers.extend(pretrained_layers[start_layer:]) # , cost=Dropout(input_include_probs={'composite':1.})))
pdb.set_trace() algorithm = SGD( batch_size=batch_size, learning_rate=learning_rate, init_momentum=.5, monitoring_dataset={'valid': val_ds}, cost=Dropout(input_include_probs={'h0': .8}, input_scales={'h0': 1.}), termination_criterion=MonitorBased(channel_name="valid_y_misclass", prop_decrease=0., N=100), #termination_criterion: !obj:pylearn2.termination_criteria.EpochCounter {max_epochs: 1}, update_callbacks=ExponentialDecay(decay_factor=1.00004, min_lr=.000001)) extensions = [ MonitorBasedSaveBest(channel_name='valid_y_misclass', save_path=save_best_path), MomentumAdjustor(start=1, saturate=250, final_momentum=.7) ] model = MLP(batch_size=batch_size, input_space=Conv2DSpace(shape=[48, 48], num_channels=num_chan, axes=['c', 0, 1, 'b']), layers=[ MaxoutConvC01B(layer_name='h0', pad=0, num_channels=64, num_pieces=2, kernel_shape=[8, 8], pool_shape=[4, 4], pool_stride=[2, 2],
def test_works(): load = True if load == False: ddmTrain = FacialKeypoint(which_set='train', start=0, stop=6000) ddmValid = FacialKeypoint(which_set='train', start=6000, stop=7049) # valid can_fit = false pipeline = preprocessing.Pipeline() stndrdz = preprocessing.Standardize() stndrdz.apply(ddmTrain, can_fit=True) #doubt, how about can_fit = False? stndrdz.apply(ddmValid, can_fit=False) GCN = preprocessing.GlobalContrastNormalization() GCN.apply(ddmTrain, can_fit=True) GCN.apply(ddmValid, can_fit=False) pcklFile = open('kpd.pkl', 'wb') obj = (ddmTrain, ddmValid) pickle.dump(obj, pcklFile) pcklFile.close() return else: pcklFile = open('kpd.pkl', 'rb') (ddmTrain, ddmValid) = pickle.load(pcklFile) pcklFile.close() #creating layers #2 convolutional rectified layers, border mode valid layer1 = ConvRectifiedLinear(layer_name='convRect1', output_channels=64, irange=.05, kernel_shape=[5, 5], pool_shape=[3, 3], pool_stride=[2, 2], max_kernel_norm=1.9365) layer2 = ConvRectifiedLinear(layer_name='convRect2', output_channels=64, irange=.05, kernel_shape=[5, 5], pool_shape=[3, 3], pool_stride=[2, 2], max_kernel_norm=1.9365) # Rectified linear units layer3 = RectifiedLinear(dim=3000, sparse_init=15, layer_name='RectLin3') #multisoftmax n_groups = 30 n_classes = 98 irange = 0 layer_name = 'multisoftmax' layerMS = MultiSoftmax(n_groups=n_groups, irange=0.05, n_classes=n_classes, layer_name=layer_name) #setting up MLP MLPerc = MLP(batch_size=8, input_space=Conv2DSpace(shape=[96, 96], num_channels=1), layers=[layer1, layer2, layer3, layerMS]) #mlp_cost missing_target_value = -1 mlp_cost = MLPCost(cost_type='default', missing_target_value=missing_target_value) #algorithm # learning rate, momentum, batch size, monitoring dataset, cost, termination criteria term_crit = MonitorBased(prop_decrease=0.00001, N=30, channel_name='validation_objective') kpSGD = KeypointSGD(learning_rate=0.001, init_momentum=0.5, monitoring_dataset={ 'validation': ddmValid, 'training': ddmTrain }, batch_size=8, batches_per_iter=750, termination_criterion=term_crit, train_iteration_mode='random_uniform', cost=mlp_cost) #train extension train_ext = ExponentialDecayOverEpoch(decay_factor=0.998, min_lr_scale=0.01) #train object train = Train(dataset=ddmTrain, save_path='kpd_model2.pkl', save_freq=1, model=MLPerc, algorithm=kpSGD, extensions=[ train_ext, MonitorBasedSaveBest(channel_name='validation_objective', save_path='kpd_best.pkl'), MomentumAdjustor(start=1, saturate=20, final_momentum=.9) ]) train.main_loop() train.save()
def main(job_id, requested_params, cache): # Fix sub directory problems sys.path.append(os.path.dirname(os.getcwd())) os.chdir(os.path.dirname(os.path.realpath(__file__))) # Add parameters that are not currently being tuned but could potentially be tuned. params = additional_args params.update(requested_params) output_channels_h2 = int(1.00 * 50) output_channels_h3 = int(3.42 * 50) output_channels_h4 = int(11.67 * 50) dropout_h2 = float(params['dropout_h2'][0]) / 10 dropout_h3 = float(params['dropout_h3'][0]) / 10 dropout_h4 = float(params['dropout_h4'][0]) / 10 dropout_y = float(params['dropout_y'][0]) / 10 if params.get('rate', None) is not None: params['log_init_learning_rate'][0] += numpy.array([params['rate']]) fixed_params = (params['kernel_size_h2'][0], params['kernel_size_h3'][0], params['dropout_h2'][0], params['dropout_h3'][0], params['dropout_h4'][0], params['dropout_y'][0]) if 'cached_trainer' + str(fixed_params) not in cache: train_params = { 'train_start': params['start'], 'train_stop': params['stop'], 'valid_start': 20000, 'valid_stop': 24000, 'test_stop': 4000, 'batch_size': 100, 'max_epochs': params.get('epochs', 1), 'max_batches': 50, 'sgd_seed': sgd_seed_str, 'mlp_seed': mlp_seed_str, 'kernel_size_h2': int(params['kernel_size_h2'][0]), 'output_channels_h2': output_channels_h2, 'irange_h2': math.pow(10, params['l_ir_h2'][0]), 'max_kernel_norm_h2': params['max_norm_h2'][0], 'dropout_h2': dropout_h2, 'dscale_h2': params['dfac_h2'][0] * 1.0 / dropout_h2, 'w_lr_sc_h2': math.pow(dropout_h2, 2), 'weight_decay_h2': math.pow(10, params['l_wdecay_h2'][0]), 'kernel_size_h3': int(params['kernel_size_h3'][0]), 'output_channels_h3': output_channels_h3, 'irange_h3': math.pow(10, params['l_ir_h3'][0]), 'max_kernel_norm_h3': params['max_norm_h3'][0], 'dropout_h3': dropout_h3, 'dscale_h3': params['dfac_h3'][0] * 1.0 / dropout_h3, 'w_lr_sc_h3': math.pow(dropout_h3, 2), 'weight_decay_h3': math.pow(10, params['l_wdecay_h3'][0]), 'kernel_size_h4': int(params['kernel_size_h4'][0]), 'output_channels_h4': output_channels_h4, 'irange_h4': math.pow(10, params['l_ir_h4'][0]), 'max_kernel_norm_h4': params['max_norm_h4'][0], 'dropout_h4': dropout_h4, 'dscale_h4': params['dfac_h4'][0] * 1.0 / dropout_h4, 'w_lr_sc_h4': math.pow(dropout_h4, 2), 'weight_decay_h4': math.pow(10, params['l_wdecay_h4'][0]), 'weight_decay_y': math.pow(10, params['l_wdecay_y'][0]), 'max_col_norm_y': params['max_norm_y'][0], 'irange_y': math.pow(10, params['l_ir_y'][0]), 'dropout_y': dropout_y, 'dscale_y': 1.0 / dropout_y, 'w_lr_sc_y': math.pow(dropout_y, 2), 'init_learning_rate': math.pow(10, params['log_init_learning_rate'][0]), 'init_momentum': params['init_momentum'][0], 'rectifier_left_slope': 0.2 } with open('conv_fooddata_spearmint.yaml', 'r') as f: trainer = f.read() yaml_string = trainer % train_params train_obj = yaml_parse.load(yaml_string) if 'converge' in params: train_obj.algorithm.termination_criterion._criteria[0]._max_epochs = params.get('epochs', 100) train_obj.extensions.append(MonitorBasedSaveBest('valid_y_misclass', params['save'])) train_obj.setup() train_obj.model.monitor.on_channel_conflict = 'ignore' # cache['cached_trainer' + str(fixed_params)] = train_obj else: train_obj = cache['cached_trainer' + str(fixed_params)] train_obj.model.monitor.set_state([0, 0, 0]) train_obj.model.training_succeeded = False # train_obj.algorithm.update_callbacks[0].reinit_from_monitor() model = train_obj.model model_params = dict([(param.name, param) for param in model.get_params()]) rng = model.rng update_conv_layer(model.layers[0], params['l_ir_h2'][0], params['max_norm_h2'][0], model_params, rng) update_conv_layer(model.layers[1], params['l_ir_h3'][0], params['max_norm_h3'][0], model_params, rng) update_conv_layer(model.layers[2], params['l_ir_h4'][0], params['max_norm_h4'][0], model_params, rng) update_softmax_layer(model.layers[3], params['l_ir_y'][0], params['max_norm_y'][0], model_params, rng) train_obj.algorithm.learning_rate.set_value( math.pow(10, params['log_init_learning_rate'][0].astype(numpy.float32))) train_obj.algorithm.learning_rule.momentum.set_value(params['init_momentum'][0].astype(numpy.float32)) pass pretrained_model_path = params.get('model', None) if pretrained_model_path is not None: print 'loading pre trained model' pretrained_model = serial.load(pretrained_model_path) print 'loading done' train_obj.model.set_param_values(pretrained_model.get_param_values()) if 'converge' not in params: train_obj.algorithm.termination_criterion._criteria[0].initialize(train_obj.model) train_obj.main_loop(do_setup=False) if params.get('savelast', False): serial.save(params['save'] + 'f', train_obj.model, on_overwrite='backup') original_misclass = read_channel(train_obj.model, misclass_channel) return float(original_misclass) * 50