def __init__(self, batch_size=256, seed=-1, shuffle_level=2, allow_rollback=True, labels=None, log_path=None, verbose=3): super(MainLoop, self).__init__() self._labels = labels self._main_task = None self._task = [] self._subtask = [] self._evaltask = [] self._task_when = {} # mapping from `Task` to `Timer` self._task_freq = {} # mapping from `Task` to `Timer` self._allow_rollback = bool(allow_rollback) self._verbose = int(verbose) # create default RNG (no randomization) self._rng = struct() self._rng.randint = lambda *args, **kwargs: None # set batch self.set_batch(batch_size=batch_size, seed=seed, shuffle_level=shuffle_level) self._callback = CallbackList() # ====== for the checkpoint ====== # self._save_path = None self._save_obj = None self._save_variables = [] self._best_object = None self._save_history = True # ====== maximum stored checkpoint ====== # self._checkpoint_increasing = True self._checkpoint_max = -1 self._current_checkpoint_count = 0
def __init__(self, batch_size=256, seed=-1, shuffle_level=0, allow_rollback=True, labels=None, log_path=None, verbose=3): super(MainLoop, self).__init__() self._labels = labels self._main_task = None self._task = [] self._subtask = [] self._evaltask = [] self._task_when = {} # mapping from `Task` to `Timer` self._task_freq = {} # mapping from `Task` to `Timer` self._allow_rollback = bool(allow_rollback) self._verbose = int(verbose) # create default RNG (no randomization) self._rng = struct() self._rng.randint = lambda *args, **kwargs: None # set batch self.set_batch(batch_size=batch_size, seed=seed, shuffle_level=shuffle_level) self._callback = CallbackList() # ====== for the checkpoint ====== # self._save_path = None self._save_obj = None self._save_variables = [] self._best_object = None self._save_history = True # ====== maximum stored checkpoint ====== # self._checkpoint_increasing = True self._checkpoint_max = -1 self._current_checkpoint_count = 0
def set_batch(self, batch_size=None, seed=-1, shuffle_level=None): if batch_size is not None: self._batch_size = batch_size if seed is None or seed >= 0: if seed is not None: self._rng = np.random.RandomState(seed) else: self._rng = struct() self._rng.randint = lambda x: None self._rng.rand = get_rng().rand if shuffle_level is not None: self._shuffle_level = min(max(int(shuffle_level), 0), 2) return self
def set_batch(self, batch_size=None, seed=-1, shuffle_level=None): if batch_size is not None: self._batch_size = batch_size if seed is None or seed >= 0: if seed is not None: self._rng = np.random.RandomState(seed) else: self._rng = struct() self._rng.randint = lambda x: None self._rng.rand = get_rng().rand if shuffle_level is not None: self._shuffle_level = min(max(int(shuffle_level), 0), 2) return self
def set_batch(self, batch_size=None, seed=-1, shuffle_level=None): if batch_size is not None: self._batch_size = batch_size self._nb_samples_per_epoch = min([len(i) for i in self._data]) if seed is None or seed >= 0: if seed is not None: self._rng = np.random.RandomState(seed) else: self._rng = struct() self._rng.randint = lambda x: None self._rng.rand = RNG_GENERATOR.rand if shuffle_level is not None: self._shuffle_level = min(max(int(shuffle_level), 0), 2) return self
def set_batch(self, batch_size=None, seed=-1, shuffle_level=None): """ Parameters ---------- batch_size: int size of each batch return when iterate this Data seed: None, int if None, no shuffling is performed while iterating, if < 0, do not change the current seed if >= 0, enable randomization with given seed start: int, float if int, start indicates the index of starting data points to iterate. If float, start is the percentage of data to start. end: int, float ending point of the interation shuffle_level: int 0: only shuffle the order of each batch 1: shuffle the order of batches and inside each batch as well. 2: includes level 0 and 1, and custom shuffling (strongest form) """ if batch_size is not None: self._batch_size = batch_size if seed >= 0 or seed is None: if seed is not None: self._rng = np.random.RandomState(seed) else: self._rng = struct() self._rng.randint = lambda *args, **kwargs: None if shuffle_level is not None: shuffle_level = min(max(int(shuffle_level), 0), 2) self._shuffle_level = shuffle_level # ====== set_batch for Tasks ====== # if self._task is not None: self._task.set_batch(batch_size=batch_size, seed=seed, shuffle_level=shuffle_level) for i in self._subtask.itervalues(): i.set_batch(batch_size=batch_size, seed=seed, shuffle_level=shuffle_level) for i in self._crosstask.itervalues(): i.set_batch(batch_size=batch_size, seed=seed, shuffle_level=shuffle_level) return self
def __init__(self, batch_size=256, seed=-1, shuffle_level=0): super(MainLoop, self).__init__() self._task = None self._subtask = {} # run 1 epoch after given frequence self._crosstask = {} # randomly run 1 iter given probability # create default RNG (no randomization) self._rng = struct() self._rng.randint = lambda *args, **kwargs: None # set batch self.set_batch(batch_size=batch_size, seed=seed, shuffle_level=shuffle_level) self._callback = CallbackList() self._save_path = None self._save_hist = None self._save_obj = None
def set_batch(self, batch_size=None, seed=-1, shuffle_level=None): """ Parameters ---------- batch_size: int size of each batch return when iterate this Data seed: None, int if None, no shuffling is performed while iterating, if < 0, do not change the current seed if >= 0, enable randomization with given seed start: int, float if int, start indicates the index of starting data points to iterate. If float, start is the percentage of data to start. end: int, float ending point of the interation shuffle_level: int 0: only shuffle the order of each batch 1: shuffle the order of batches and inside each batch as well. 2: includes level 0 and 1, and custom shuffling (strongest form) """ if batch_size is not None: self._batch_size = batch_size if seed is None or seed >= 0: if seed is not None: self._rng = np.random.RandomState(seed) else: self._rng = struct() self._rng.randint = lambda *args, **kwargs: None if shuffle_level is not None: shuffle_level = min(max(int(shuffle_level), 0), 2) self._shuffle_level = shuffle_level # ====== set_batch for Tasks ====== # for i in self._task: i.set_batch(batch_size=batch_size, seed=seed, shuffle_level=shuffle_level) for i in self._subtask: i.set_batch(batch_size=batch_size, seed=seed, shuffle_level=shuffle_level) return self
def create_iteration(): seed = self._seed; self._seed = None if seed is not None: rng = np.random.RandomState(seed) else: # deterministic RandomState rng = struct() rng.randint = lambda x: None rng.permutation = lambda x: slice(None, None) # ====== easy access many private variables ====== # sequential = self._sequential start, end = self._start, self._end batch_size = self._batch_size distribution = np.asarray(self._distribution) # shuffle order of data (good for sequential mode) idx = rng.permutation(len(self._data)) data = self._data[idx] if isinstance(idx, slice) else [self._data[i] for i in idx] distribution = distribution[idx] shape = [i.shape[0] for i in data] # ====== prepare distribution information ====== # # number of sample should be traversed n = np.asarray([i * (_apply_approx(j, end) - _apply_approx(j, start)) for i, j in zip(distribution, shape)]) n = np.round(n).astype(int) # normalize the distribution (base on new sample n of each data) distribution = n / n.sum() distribution = _approximate_continuos_by_discrete(distribution) # somehow heuristic, rescale distribution to get more benifit from cache if distribution.sum() <= len(data): distribution = distribution * 3 # distribution now the actual batch size of each data distribution = (batch_size * distribution).astype(int) assert distribution.sum() % batch_size == 0, 'wrong distribution size!' # predefined (start,end) pair of each batch (e.g (0,256), (256,512)) idx = list(range(0, batch_size + distribution.sum(), batch_size)) idx = list(zip(idx, idx[1:])) # Dummy return to initialize everything yield None ##################################### # 1. optimized parallel code. if not sequential: # first iterators it = [iter(dat.set_batch(bs, seed=rng.randint(10e8), start=start, end=end, shuffle_level=self._shuffle_level)) for bs, dat in zip(distribution, data)] # iterator while sum(n) > 0: batch = [] for i, x in enumerate(it): if n[i] <= 0: continue try: x = next(x)[:n[i]] n[i] -= x.shape[0] batch.append(x) except StopIteration: # one iterator stopped it[i] = iter(data[i].set_batch(distribution[i], seed=rng.randint(10e8), start=start, end=end, shuffle_level=self._shuffle_level)) x = next(it[i])[:n[i]] n[i] -= x.shape[0] batch.append(x) # got final batch batch = np.vstack(batch) # no idea why random permutation is much faster than shuffle if self._shuffle_level > 0: batch = batch[rng.permutation(batch.shape[0])] # return the iterations for i, j in idx[:int(ceil(batch.shape[0] / batch_size))]: yield batch[i:j] ##################################### # 2. optimized sequential code. else: # first iterators batch_size = distribution.sum() it = [iter(dat.set_batch(batch_size, seed=rng.randint(10e8), start=start, end=end, shuffle_level=self._shuffle_level)) for dat in data] current_data = 0 # iterator while sum(n) > 0: if n[current_data] <= 0: current_data += 1 try: x = next(it[current_data])[:n[current_data]] n[current_data] -= x.shape[0] except StopIteration: # one iterator stopped it[current_data] = iter(data[current_data].set_batch(batch_size, seed=rng.randint(10e8), start=start, end=end, shuffle_level=self._shuffle_level)) x = next(it[current_data])[:n[current_data]] n[current_data] -= x.shape[0] # shuffle x if self._shuffle_level > 0: x = x[rng.permutation(x.shape[0])] for i, j in idx[:int(ceil(x.shape[0] / self._batch_size))]: yield x[i:j]
def _check_tag(var): if not hasattr(var, 'tag'): var.tag = struct()
from six.moves import range, zip, cPickle import numpy as np from odin import (SIG_TRAIN_ROLLBACK, SIG_TRAIN_SAVE, SIG_TRAIN_STOP) from odin.config import RNG_GENERATOR from odin import fuel from odin.fuel.dataset import Dataset from odin.utils import struct, as_tuple, is_number from .callbacks import * # =========================================================================== # Helper # =========================================================================== _SAVE_TASK = struct() _SAVE_TASK.name = "save" def __format_string(nb_of_float): x = ["{:.4f}"] * int(nb_of_float) return ";".join(x) def standard_trainer(train_data, valid_data, X, y_train, y_score, y_target, parameters,