def train(self, training_instances, validation_instances=None, metrics=None): id_tag = (self.id + ': ') if self.id else '' if self.options.verbosity >= 2: print(id_tag + 'Training priors') self.train_priors(training_instances, listener_data=self.options.listener) self.dataset = training_instances xs, ys = self._data_to_arrays(training_instances, init_vectorizer=True) self._build_model() if self.options.verbosity >= 2: print(id_tag + 'Training conditional model') summary_path = config.get_file_path('losses.tfevents') if summary_path: writer = summary.SummaryWriter(summary_path) else: writer = None progress.start_task('Iteration', self.options.train_iters) for iteration in range(self.options.train_iters): progress.progress(iteration) self.model.fit(xs, ys, batch_size=self.options.batch_size, num_epochs=self.options.train_epochs, summary_writer=writer, step=iteration * self.options.train_epochs) validation_results = self.validate(validation_instances, metrics, iteration=iteration) if writer is not None: step = (iteration + 1) * self.options.train_epochs self.on_iter_end(step, writer) for key, value in validation_results.iteritems(): tag = 'val/' + key.split('.', 1)[1].replace('.', '/') writer.log_scalar(step, tag, value) writer.flush() progress.end_task()
def train(self, training_instances, validation_instances=None, metrics=None, keep_params=False): id_tag = (self.id + ': ') if self.id else '' if self.options.verbosity >= 2: print(id_tag + 'Training priors') self.train_priors(training_instances, listener_data=self.options.listener) self.dataset = training_instances xs, ys = self._data_to_arrays(training_instances, init_vectorizer=not hasattr(self, 'model')) if not hasattr(self, 'model') or not keep_params: if self.options.verbosity >= 2: print(id_tag + 'Building model') if keep_params: warnings.warn("keep_params was passed, but the model hasn't been built; " "initializing all parameters.") self._build_model() else: if not hasattr(self.options, 'reset_optimizer_vars') or \ self.options.reset_optimizer_vars: if self.options.verbosity >= 2: print(id_tag + 'Resetting optimizer') self.model.reset_optimizer() if self.options.verbosity >= 2: print(id_tag + 'Training conditional model') if hasattr(self, 'writer'): writer = self.writer else: summary_path = config.get_file_path('losses.tfevents') if summary_path: writer = summary.SummaryWriter(summary_path) else: writer = None self.writer = writer if not hasattr(self, 'step_base'): self.step_base = 0 progress.start_task('Iteration', self.options.train_iters) for iteration in range(self.options.train_iters): progress.progress(iteration) self.model.fit(xs, ys, batch_size=self.options.batch_size, num_epochs=self.options.train_epochs, summary_writer=writer, step=self.step_base + iteration * self.options.train_epochs) validation_results = self.validate(validation_instances, metrics, iteration=iteration) if writer is not None: step = self.step_base + (iteration + 1) * self.options.train_epochs self.on_iter_end(step, writer) for key, value in validation_results.iteritems(): tag = 'val/' + key.split('.', 1)[1].replace('.', '/') writer.log_scalar(step, tag, value) self.step_base += self.options.train_iters * self.options.train_epochs writer.flush() progress.end_task()
def __init__(self, module, loss, optimizer, optimizer_params, vectorizer): self.get_options() self.module = cu(module) self.loss = cu(loss) self.optimizer_class = optimizer self.optimizer_params = optimizer_params self.build_optimizer() self.vectorizer = vectorizer summary_path = config.get_file_path('monitoring.tfevents') if summary_path: self.summary_writer = summary.SummaryWriter(summary_path) else: self.summary_writer = None self.step = 0 self.last_timestamp = datetime.datetime.now()
def test_large_scalar(self): fs = patcher('stanza.monitoring.summary', '/test') open = fs.start() writer = summary.SummaryWriter('/test/large_scalar.tfevents') writer.log_scalar(1, 'bigvalue', 1.0e39) writer.flush() with open('/test/large_scalar.tfevents', 'r') as infile: events = list(summary.read_events(infile)) self.assertEqual(len(events), 1) self.assertEqual(len(events[0].summary.value), 1) self.assertTrue(np.isinf(events[0].summary.value[0].simple_value)) fs.stop()
def test_large_hist(self): fs = patcher('stanza.monitoring.summary', '/test') open = fs.start() writer = summary.SummaryWriter('/test/large_hist.tfevents') writer.log_histogram(1, 'bighist', np.array(1.0e39)) writer.flush() with open('/test/large_hist.tfevents', 'r') as infile: events = list(summary.read_events(infile)) self.assertEqual(len(events), 1) self.assertEqual(len(events[0].summary.value), 1) self.assertTrue(events[0].summary.value[0].HasField('histo'), events[0].summary.value[0]) fs.stop()