def __init__(self,
num_units,
W_init=init_ops.glorot_uniform_initializer(seed=randint()),
b_init=init_ops.constant_initializer(0.),
rnn_mode='lstm',
num_layers=1,
skip_input=False,
is_bidirectional=False,
return_states=False,
dropout=0.,
**kwargs):
super(CudnnRNN, self).__init__(**kwargs)
# ====== defaults recurrent control ====== #
self.num_units = int(num_units)
self.num_layers = int(num_layers)
self.rnn_mode = str(rnn_mode)
self.skip_input = bool(skip_input)
self.is_bidirectional = bool(is_bidirectional)
self.return_states = bool(return_states)
self.dropout = dropout
self.W_init = W_init
self.b_init = b_init
if skip_input:
wprint("`skip_input` is not supported in Tensorflow.")
def _preprocessing_losses(losses,
y_true,
y_pred,
inherit_losses=None,
sample_weights=None):
""" Can be used for both objectives and metrics """
from odin import backend as K
# ====== special cases, only one inputs outputs, and multiple loss ====== #
nb_losses = len(losses)
if len(y_true) == 0:
y_true = [None] * nb_losses
elif len(y_true) == 1:
y_true = y_true * nb_losses
if len(y_pred) == 0:
y_pred = [None] * nb_losses
elif len(y_pred) == 1:
y_pred = y_pred * nb_losses
# ====== applying ====== #
cost = []
for idx, fn in enumerate(as_tuple(losses)):
weight = 1
kwargs = {}
# preprocess
if isinstance(fn, (tuple, list)):
if len(fn) == 1:
fn = fn[0]
else:
weight = [i for i in fn if is_number(i)]
weight = 1 if len(weight) == 0 else weight[0]
kwargs = [i for i in fn if isinstance(i, Mapping)]
kwargs = {} if len(kwargs) == 0 else kwargs[0]
fn = [i for i in fn if i != weight and i != kwargs][0]
# apply the loss
if is_number(fn):
if inherit_losses is None or fn >= len(inherit_losses):
raise ValueError("Cannot find losses at index: '%d'" % fn)
obj = inherit_losses[fn]
elif K.is_tensor(fn):
obj = fn
elif hasattr(fn, '__call__'):
try:
sign = inspect.signature(fn)
if 'weights' in sign.parameters and sample_weights is not None:
kwargs['weights'] = sample_weights
except ValueError:
pass
finally:
obj = fn(y_true[idx], y_pred[idx], **kwargs)
if isinstance(obj, (tuple, list)):
wprint(
"function: '%s' return %d outputs (%s), only pick the first one"
% (fn.__name__, len(obj), '; '.join([str(i)
for i in obj])))
obj = obj[0]
cost.append((weight, obj))
# ====== reduce ====== #
return [c if w == 1 else w * c for w, c in cost]
def _preprocessing_losses(losses, y_true, y_pred, inherit_losses=None,
sample_weights=None):
""" Can be used for both objectives and metrics """
from odin import backend as K
# ====== special cases, only one inputs outputs, and multiple loss ====== #
nb_losses = len(losses)
if len(y_true) == 0:
y_true = [None] * nb_losses
elif len(y_true) == 1:
y_true = y_true * nb_losses
if len(y_pred) == 0:
y_pred = [None] * nb_losses
elif len(y_pred) == 1:
y_pred = y_pred * nb_losses
# ====== applying ====== #
cost = []
for idx, fn in enumerate(as_tuple(losses)):
weight = 1
kwargs = {}
# preprocess
if isinstance(fn, (tuple, list)):
if len(fn) == 1:
fn = fn[0]
else:
weight = [i for i in fn if is_number(i)]
weight = 1 if len(weight) == 0 else weight[0]
kwargs = [i for i in fn if isinstance(i, Mapping)]
kwargs = {} if len(kwargs) == 0 else kwargs[0]
fn = [i for i in fn if i != weight and i != kwargs][0]
# apply the loss
if is_number(fn):
if inherit_losses is None or fn >= len(inherit_losses):
raise ValueError("Cannot find losses at index: '%d'" % fn)
obj = inherit_losses[fn]
elif K.is_tensor(fn):
obj = fn
elif hasattr(fn, '__call__'):
try:
sign = inspect.signature(fn)
if 'weights' in sign.parameters and sample_weights is not None:
kwargs['weights'] = sample_weights
except ValueError:
pass
finally:
obj = fn(y_true[idx], y_pred[idx], **kwargs)
if isinstance(obj, (tuple, list)):
wprint("function: '%s' return %d outputs (%s), only pick the first one"
% (fn.__name__,
len(obj),
'; '.join([str(i) for i in obj])))
obj = obj[0]
cost.append((weight, obj))
# ====== reduce ====== #
return [c if w == 1 else w * c for w, c in cost]
def save_mean_std(sum1, sum2, name):
N = dataset[name.split('_')[0]].shape[0]
mean = sum1 / N
std = np.sqrt(sum2 / N - np.power(mean, 2))
if np.any(np.isnan(mean)):
wprint('Mean contains NaN, name: %s' % name)
if np.any(np.isnan(std)):
wprint('Std contains NaN, name: %s' % name)
dataset[name + 'sum1'] = sum1
dataset[name + 'sum2'] = sum2
dataset[name + 'mean'] = mean
dataset[name + 'std'] = std
def save_mean_std(sum1, sum2, name):
N = dataset[name.split('_')[0]].shape[0]
mean = sum1 / N
std = np.sqrt(sum2 / N - np.power(mean, 2))
if np.any(np.isnan(mean)):
wprint('Mean contains NaN, name: %s' % name)
if np.any(np.isnan(std)):
wprint('Std contains NaN, name: %s' % name)
dataset[name + 'sum1'] = sum1
dataset[name + 'sum2'] = sum2
dataset[name + 'mean'] = mean
dataset[name + 'std'] = std
def _restore_variables(self):
""" This method can be called anywhere to make sure
the variable related to this NNOp is restored after
pickling.
"""
if hasattr(self, '_restore_vars_path') and \
self._restore_vars_path is not None:
folder_path = os.path.dirname(self._restore_vars_path)
if os.path.exists(folder_path):
K.restore_variables(self._restore_vars_path)
# delete cached folder if necessary
if self._delete_vars_folder:
shutil.rmtree(folder_path)
else:
wprint("NNOp: '%s' cannot restore variables from path: '%s'"
(self.name, folder_path))
# reset info
self._set_restore_info(None, False)
def __init__(self, num_units,
W_init=init_ops.glorot_uniform_initializer(seed=randint()),
b_init=init_ops.constant_initializer(0.),
rnn_mode='lstm', num_layers=1,
skip_input=False, is_bidirectional=False,
return_states=False, dropout=0., **kwargs):
super(CudnnRNN, self).__init__(**kwargs)
# ====== defaults recurrent control ====== #
self.num_units = int(num_units)
self.num_layers = int(num_layers)
self.rnn_mode = str(rnn_mode)
self.skip_input = bool(skip_input)
self.is_bidirectional = bool(is_bidirectional)
self.return_states = bool(return_states)
self.dropout = dropout
self.W_init = W_init
self.b_init = b_init
if skip_input:
wprint("`skip_input` is not supported in Tensorflow.")
def fast_tsne(*X,
n_components=2,
n_samples=None,
perplexity=30.0,
early_exaggeration=8.0,
learning_rate=200.0,
n_iter=1000,
n_iter_without_progress=300,
min_grad_norm=1e-7,
metric="euclidean",
init="random",
verbose=0,
random_state=1234,
method='barnes_hut',
angle=0.5,
n_jobs=4):
"""
Parameters
----------
n_components : int, optional (default: 2)
Dimension of the embedded space.
n_samples : {int, None}
if given, downsampling the data to given number of sample
perplexity : float, optional (default: 30)
The perplexity is related to the number of nearest neighbors that
is used in other manifold learning algorithms. Larger datasets
usually require a larger perplexity. Consider selecting a value
between 5 and 50. The choice is not extremely critical since t-SNE
is quite insensitive to this parameter.
early_exaggeration : float, optional (default: 8.0)
Controls how tight natural clusters in the original space are in
the embedded space and how much space will be between them. For
larger values, the space between natural clusters will be larger
in the embedded space. Again, the choice of this parameter is not
very critical. If the cost function increases during initial
optimization, the early exaggeration factor or the learning rate
might be too high.
learning_rate : float, optional (default: 200.0)
The learning rate for t-SNE is usually in the range [10.0, 1000.0]. If
the learning rate is too high, the data may look like a 'ball' with any
point approximately equidistant from its nearest neighbours. If the
learning rate is too low, most points may look compressed in a dense
cloud with few outliers. If the cost function gets stuck in a bad local
minimum increasing the learning rate may help.
n_iter : int, optional (default: 1000)
Maximum number of iterations for the optimization. Should be at
least 250.
n_iter_without_progress : int, optional (default: 300)
Maximum number of iterations without progress before we abort the
optimization, used after 250 initial iterations with early
exaggeration. Note that progress is only checked every 50 iterations so
this value is rounded to the next multiple of 50.
min_grad_norm : float, optional (default: 1e-7)
If the gradient norm is below this threshold, the optimization will
be stopped.
metric : string or callable, optional
The metric to use when calculating distance between instances in a
feature array. If metric is a string, it must be one of the options
allowed by scipy.spatial.distance.pdist for its metric parameter, or
a metric listed in pairwise.PAIRWISE_DISTANCE_FUNCTIONS.
If metric is "precomputed", X is assumed to be a distance matrix.
Alternatively, if metric is a callable function, it is called on each
pair of instances (rows) and the resulting value recorded. The callable
should take two arrays from X as input and return a value indicating
the distance between them. The default is "euclidean" which is
interpreted as squared euclidean distance.
init : string or numpy array, optional (default: "random")
Initialization of embedding. Possible options are 'random', 'pca',
and a numpy array of shape (n_samples, n_components).
PCA initialization cannot be used with precomputed distances and is
usually more globally stable than random initialization.
verbose : int, optional (default: 0)
Verbosity level.
random_state : int, RandomState instance or None, optional (default: None)
If int, random_state is the seed used by the random number generator;
If RandomState instance, random_state is the random number generator;
If None, the random number generator is the RandomState instance used
by `np.random`. Note that different initializations might result in
different local minima of the cost function.
method : string (default: 'barnes_hut')
By default the gradient calculation algorithm uses Barnes-Hut
approximation running in O(NlogN) time. method='exact'
will run on the slower, but exact, algorithm in O(N^2) time. The
exact algorithm should be used when nearest-neighbor errors need
to be better than 3%. However, the exact method cannot scale to
millions of examples.
angle : float (default: 0.5)
Only used if method='barnes_hut'
This is the trade-off between speed and accuracy for Barnes-Hut T-SNE.
'angle' is the angular size (referred to as theta in [3]) of a distant
node as measured from a point. If this size is below 'angle' then it is
used as a summary node of all points contained within it.
This method is not very sensitive to changes in this parameter
in the range of 0.2 - 0.8. Angle less than 0.2 has quickly increasing
computation time and angle greater 0.8 has quickly increasing error.
"""
assert len(X) > 0, "No input is given!"
if isinstance(X[0], (tuple, list)):
X = X[0]
if not all(isinstance(x, np.ndarray) for x in X):
raise ValueError(
"`X` can only be list of numpy.ndarray or numpy.ndarray")
# ====== kwarg for creating T-SNE class ====== #
kwargs = dict(locals())
del kwargs['X']
n_samples = kwargs.pop('n_samples', None)
# ====== downsampling ====== #
if n_samples is not None:
n_samples = int(n_samples)
assert n_samples > 0
new_X = []
rand = random_state if isinstance(random_state, np.random.RandomState) else \
np.random.RandomState(seed=random_state)
for x in X:
if x.shape[0] > n_samples:
ids = rand.permutation(x.shape[0])[:n_samples]
x = x[ids]
new_X.append(x)
X = new_X
# ====== import proper T-SNE ====== #
tsne_version = None
try:
from tsnecuda import TSNE
from tsnecuda.NaiveTSNE import NaiveTSNE as _exact_TSNE
tsne_version = 'cuda'
except ImportError:
# wprint("Install CUDA-TSNE from `https://github.com/CannyLab/tsne-cuda` "
# "for significant speed up.")
try:
from MulticoreTSNE import MulticoreTSNE as TSNE
tsne_version = 'multicore'
except ImportError:
wprint(
"Install MulticoreTSNE from `pip install git+https://github.com/DmitryUlyanov/Multicore-TSNE.git`"
' to accelerate the T-SNE on multiple CPU cores.')
try:
from sklearn.manifold import TSNE
tsne_version = 'sklearn'
except Exception as e:
raise e
# ====== modify kwargs ====== #
if tsne_version == 'cuda':
kwargs['random_seed'] = kwargs['random_state']
kwargs['theta'] = angle
if method == 'exact':
TSNE = _exact_TSNE
del kwargs['theta']
del kwargs['random_state']
del kwargs['n_jobs']
del kwargs['angle']
del kwargs['method']
elif tsne_version == 'multicore':
pass
else:
del kwargs['n_jobs']
# ====== getting cached values ====== #
results = []
X_new = []
for i, x in enumerate(X):
md5 = md5_checksum(x)
key = _create_key(kwargs, md5)
if key in _cached_values:
results.append((i, _cached_values[key]))
else:
X_new.append((i, md5, x))
# ====== perform T-SNE ====== #
def apply_tsne(j):
idx, md5, x = j
tsne = TSNE(**kwargs)
return (idx, md5, tsne.fit_transform(x))
# only 1 X, no need for MPI
if len(X_new) == 1:
idx, md5, x = apply_tsne(X_new[0])
results.append((idx, x))
_cached_values[_create_key(kwargs, md5)] = x
else:
mpi = MPI(jobs=X_new,
func=apply_tsne,
batch=1,
ncpu=min(len(X_new),
cpu_count() - 1))
for idx, md5, x in mpi:
results.append((idx, x))
_cached_values[_create_key(kwargs, md5)] = x
# ====== return and clean ====== #
results = sorted(results, key=lambda a: a[0])
results = [r[1] for r in results]
return results[0] if len(results) == 1 else results
def __init__(self,
jobs,
path,
extractor,
n_cache=0.12,
ncpu=1,
override=False,
identifier='name',
log_path=None,
stop_on_failure=False):
super(FeatureProcessor, self).__init__()
# ====== check outpath ====== #
path = os.path.abspath(str(path))
if os.path.isfile(path):
raise ValueError("`path` must be path to a directory, but found a "
"path to file.")
# check override
if os.path.exists(path) and override:
wprint("Remove existed Dataset at path: %s" % path)
for i in os.listdir(path):
i = os.path.join(path, i)
if os.path.isdir(i): # remove folder
shutil.rmtree(i)
else: # remove file
os.remove(i)
# set path and name
self.path = path
# ====== check jobs ====== #
if not isinstance(jobs, (tuple, list, np.ndarray)):
raise ValueError(
"Provided `jobs` must be instance of tuple, list or ndarray.")
if isinstance(jobs, np.ndarray):
jobs = jobs.tolist()
self.jobs = tuple(jobs)
# ====== check multiprocessing ====== #
if ncpu is None: # auto select number of CPU
ncpu = min(len(jobs), cpu_count() - 1)
ncpu = int(ncpu)
if ncpu <= 0 or n_cache <= 0:
raise ValueError(
'`ncpu` and `n_cache` must be greater than 0, but '
'given values ncpu=%d n_cache=%f' % (ncpu, n_cache))
self.n_cpu = ncpu
self.n_cache = n_cache
# ====== internal control for feature processor ====== #
if isinstance(extractor, Pipeline):
pass
elif isinstance(extractor, (tuple, list)):
steps = [('%s_%d' % (e.__class__.__name__, i), e)
for i, e in enumerate(extractor)]
extractor = Pipeline(steps=steps)
elif isinstance(extractor, Mapping):
steps = [(str(n), e) for n, e in extractor.items()]
extractor = Pipeline(steps=steps)
elif isinstance(extractor, Extractor):
extractor = Pipeline(steps=[(extractor.__class__.__name__,
extractor)])
self.extractor = extractor
# ====== check identifier and log path ====== #
self._identifier = str(identifier)
if log_path is None:
log_path = os.path.join(self.path, 'log.txt')
else:
log_path = str(log_path)
self._log_path = _check_logpath(log_path)
# ====== others ====== #
self.config = {}
self._error_log = []
self.stop_on_failure = bool(stop_on_failure)
def fit(self, X, y=None, cv=None):
self._initialize(X)
if not hasattr(X, 'shape') or not hasattr(X, '__iter__') or \
not hasattr(X, '__len__'):
raise ValueError(
"`X` must has 'shape', '__len__' and '__iter__' attributes")
nb_train_samples = len(X)
# convert to odin.fuel.Data if possible
if isinstance(X, (np.ndarray, list, tuple)):
X = F.as_data(X)
if isinstance(y, (np.ndarray, list, tuple)):
y = F.as_data(y)
start_tr = 0
end_tr = nb_train_samples
# ====== check if cross validating ====== #
create_it_cv = None
if is_number(cv):
cv = int(float(cv) * nb_train_samples) if cv < 1. else int(cv)
end_tr = nb_train_samples - cv
start_cv = end_tr
end_cv = nb_train_samples
nb_cv_samples = end_cv - start_cv
create_it_cv = _create_it_func(X=X,
y=y,
batch_size=self.batch_size,
start=start_cv,
end=end_cv)
elif isinstance(cv, (tuple, list)):
X_cv, y_cv = cv
nb_cv_samples = X_cv.shape[0]
create_it_cv = _create_it_func(X=X_cv,
y=y_cv,
batch_size=self.batch_size,
start=0,
end=X_cv.shape[0])
elif hasattr(cv, 'set_batch'):
nb_cv_samples = cv.shape[0]
create_it_cv = _create_it_func(X=cv,
y=None,
batch_size=self.batch_size,
start=0,
end=cv.shape[0])
elif cv is not None:
raise ValueError(
'`cv` can be float (0-1), tuple or list of X and y, '
'any object that have "shape" and "__iter__" attributes, '
'or None')
# ====== preprocessing ====== #
create_it = _create_it_func(X=X,
y=y,
batch_size=self.batch_size,
start=start_tr,
end=end_tr)
# ====== prepare ====== #
curr_niter = sum(epoch[0] for epoch in self._train_history)
curr_nepoch = len(self._train_history)
curr_patience = int(self.patience)
last_losses = None
last_checkpoint = None
best_epoch = None
is_converged = False
# ====== fitting ====== #
while not is_converged:
curr_nepoch += 1
seed = self._rand_state.randint(0, 10e8)
# ====== training ====== #
nb_iter, duration, results = _fitting_helper(
create_it(seed),
fn=self._f_train,
nb_samples=nb_train_samples,
nb_classes=self.nb_classes,
title='Epoch %d' % curr_nepoch)
curr_niter += nb_iter
self._train_history.append(
(nb_train_samples, nb_iter, duration, results))
# ====== cross validation ====== #
if create_it_cv is not None:
nb_iter, duration_valid, results = _fitting_helper(
create_it_cv(seed),
fn=self._f_score,
nb_samples=nb_cv_samples,
nb_classes=self.nb_classes,
title="Validating")
self._valid_history.append(
(nb_train_samples, nb_iter, duration_valid, results))
duration += duration_valid
# ====== print log ====== #
if self.verbose >= 2:
print(
ctext('#epoch:', 'cyan') + str(curr_nepoch),
ctext('#iter:', 'cyan') + str(curr_niter),
ctext("Loss:", 'yellow') + '%.5f' % results[0],
ctext("Acc:", 'yellow') + '%.3f' % results[1],
ctext("%.2f(s)" % duration, 'magenta'))
if self.confusion_matrix and (curr_nepoch - 1) % 8 == 0:
print(V.print_confusion(results[-1], labels=self.labels))
# ====== early stopping ====== #
losses = results[0]
if last_checkpoint is None: # first check point
last_checkpoint = self.parameters
if last_losses is not None:
# degraded, smaller is better
if last_losses - losses <= self.tol:
curr_patience -= 1
if self.rollback:
if self.verbose >= 2:
wprint(
'[LogisticRegression] Rollback to the best checkpoint '
'at epoch:%s patience:%s' %
(ctext(best_epoch,
'cyan'), ctext(curr_patience, 'cyan')))
self.set_parameters(*last_checkpoint)
# save best checkpoint
else:
last_checkpoint = self.parameters
best_epoch = curr_nepoch
if self._path is not None:
with open(self._path, 'wb') as f:
pickle.dump(self, f)
last_losses = losses
if curr_patience <= 0:
is_converged = True
# end the training
if self.max_iter is not None and \
curr_niter >= self.max_iter:
break
if self.max_epoch is not None and \
curr_nepoch >= self.max_epoch:
break
# ====== print summary plot ====== #
if self.verbose >= 1:
train_losses = [epoch[-1][0] for epoch in self._train_history]
print(
V.print_bar(train_losses,
height=12,
bincount=min(20, len(train_losses)),
title='Training Losses'))
if create_it_cv is not None:
valid_losses = [epoch[-1][0] for epoch in self._valid_history]
print(
V.print_bar(valid_losses,
height=12,
bincount=min(20, len(train_losses)),
title='Validation Losses'))
if self.confusion_matrix:
print(
ctext("======== Training Confusion Matrix ========",
'cyan'))
print(
V.print_confusion(arr=self._train_history[-1][-1][-1],
labels=self.labels))
if create_it_cv is not None:
print(
ctext("======== Validation Confusion Matrix ========",
'cyan'))
print(
V.print_confusion(arr=self._valid_history[-1][-1][-1],
labels=self.labels))
# ====== reset to best points ====== #
self.set_parameters(*last_checkpoint)
self._is_fitted = True
if self._path is not None:
with open(self._path, 'wb') as f:
pickle.dump(self, f)
def fast_tsne(*X, n_components=2, n_samples=None, perplexity=30.0,
early_exaggeration=8.0, learning_rate=200.0, n_iter=1000,
n_iter_without_progress=300, min_grad_norm=1e-7,
metric="euclidean", init="random", verbose=0,
random_state=5218, method='barnes_hut', angle=0.5,
n_jobs=4):
"""
Parameters
----------
n_components : int, optional (default: 2)
Dimension of the embedded space.
n_samples : {int, None}
if given, downsampling the data to given number of sample
perplexity : float, optional (default: 30)
The perplexity is related to the number of nearest neighbors that
is used in other manifold learning algorithms. Larger datasets
usually require a larger perplexity. Consider selecting a value
between 5 and 50. The choice is not extremely critical since t-SNE
is quite insensitive to this parameter.
early_exaggeration : float, optional (default: 8.0)
Controls how tight natural clusters in the original space are in
the embedded space and how much space will be between them. For
larger values, the space between natural clusters will be larger
in the embedded space. Again, the choice of this parameter is not
very critical. If the cost function increases during initial
optimization, the early exaggeration factor or the learning rate
might be too high.
learning_rate : float, optional (default: 200.0)
The learning rate for t-SNE is usually in the range [10.0, 1000.0]. If
the learning rate is too high, the data may look like a 'ball' with any
point approximately equidistant from its nearest neighbours. If the
learning rate is too low, most points may look compressed in a dense
cloud with few outliers. If the cost function gets stuck in a bad local
minimum increasing the learning rate may help.
n_iter : int, optional (default: 1000)
Maximum number of iterations for the optimization. Should be at
least 250.
n_iter_without_progress : int, optional (default: 300)
Maximum number of iterations without progress before we abort the
optimization, used after 250 initial iterations with early
exaggeration. Note that progress is only checked every 50 iterations so
this value is rounded to the next multiple of 50.
min_grad_norm : float, optional (default: 1e-7)
If the gradient norm is below this threshold, the optimization will
be stopped.
metric : string or callable, optional
The metric to use when calculating distance between instances in a
feature array. If metric is a string, it must be one of the options
allowed by scipy.spatial.distance.pdist for its metric parameter, or
a metric listed in pairwise.PAIRWISE_DISTANCE_FUNCTIONS.
If metric is "precomputed", X is assumed to be a distance matrix.
Alternatively, if metric is a callable function, it is called on each
pair of instances (rows) and the resulting value recorded. The callable
should take two arrays from X as input and return a value indicating
the distance between them. The default is "euclidean" which is
interpreted as squared euclidean distance.
init : string or numpy array, optional (default: "random")
Initialization of embedding. Possible options are 'random', 'pca',
and a numpy array of shape (n_samples, n_components).
PCA initialization cannot be used with precomputed distances and is
usually more globally stable than random initialization.
verbose : int, optional (default: 0)
Verbosity level.
random_state : int, RandomState instance or None, optional (default: None)
If int, random_state is the seed used by the random number generator;
If RandomState instance, random_state is the random number generator;
If None, the random number generator is the RandomState instance used
by `np.random`. Note that different initializations might result in
different local minima of the cost function.
method : string (default: 'barnes_hut')
By default the gradient calculation algorithm uses Barnes-Hut
approximation running in O(NlogN) time. method='exact'
will run on the slower, but exact, algorithm in O(N^2) time. The
exact algorithm should be used when nearest-neighbor errors need
to be better than 3%. However, the exact method cannot scale to
millions of examples.
angle : float (default: 0.5)
Only used if method='barnes_hut'
This is the trade-off between speed and accuracy for Barnes-Hut T-SNE.
'angle' is the angular size (referred to as theta in [3]) of a distant
node as measured from a point. If this size is below 'angle' then it is
used as a summary node of all points contained within it.
This method is not very sensitive to changes in this parameter
in the range of 0.2 - 0.8. Angle less than 0.2 has quickly increasing
computation time and angle greater 0.8 has quickly increasing error.
"""
assert len(X) > 0, "No input is given!"
if isinstance(X[0], (tuple, list)):
X = X[0]
if not all(isinstance(x, np.ndarray) for x in X):
raise ValueError("`X` can only be list of numpy.ndarray or numpy.ndarray")
# ====== kwarg for creating T-SNE class ====== #
kwargs = dict(locals())
del kwargs['X']
n_samples = kwargs.pop('n_samples', None)
# ====== downsampling ====== #
if n_samples is not None:
n_samples = int(n_samples)
assert n_samples > 0
new_X = []
rand = random_state if isinstance(random_state, np.random.RandomState) else \
np.random.RandomState(seed=random_state)
for x in X:
if x.shape[0] > n_samples:
ids = rand.permutation(x.shape[0])[:n_samples]
x = x[ids]
new_X.append(x)
X = new_X
# ====== import proper T-SNE ====== #
tsne_version = None
try:
from tsnecuda import TSNE
from tsnecuda.NaiveTSNE import NaiveTSNE as _exact_TSNE
tsne_version = 'cuda'
except ImportError:
# wprint("Install CUDA-TSNE from `https://github.com/CannyLab/tsne-cuda` "
# "for significant speed up.")
try:
from MulticoreTSNE import MulticoreTSNE as TSNE
tsne_version = 'multicore'
except ImportError:
wprint("Install MulticoreTSNE from `pip install git+https://github.com/DmitryUlyanov/Multicore-TSNE.git`"
' to accelerate the T-SNE on multiple CPU cores.')
try:
from sklearn.manifold import TSNE
tsne_version = 'sklearn'
except Exception as e:
raise e
# ====== modify kwargs ====== #
if tsne_version == 'cuda':
kwargs['random_seed'] = kwargs['random_state']
kwargs['theta'] = angle
if method == 'exact':
TSNE = _exact_TSNE
del kwargs['theta']
del kwargs['random_state']
del kwargs['n_jobs']
del kwargs['angle']
del kwargs['method']
elif tsne_version == 'multicore':
pass
else:
del kwargs['n_jobs']
# ====== getting cached values ====== #
results = []
X_new = []
for i, x in enumerate(X):
md5 = md5_checksum(x)
key = _create_key(kwargs, md5)
if key in _cached_values:
results.append((i, _cached_values[key]))
else:
X_new.append((i, md5, x))
# ====== perform T-SNE ====== #
def apply_tsne(j):
idx, md5, x = j
tsne = TSNE(**kwargs)
return (idx, md5, tsne.fit_transform(x))
# only 1 X, no need for MPI
if len(X_new) == 1:
idx, md5, x = apply_tsne(X_new[0])
results.append((idx, x))
_cached_values[_create_key(kwargs, md5)] = x
else:
mpi = MPI(jobs=X_new, func=apply_tsne, batch=1,
ncpu=min(len(X_new), cpu_count() - 1))
for idx, md5, x in mpi:
results.append((idx, x))
_cached_values[_create_key(kwargs, md5)] = x
# ====== return and clean ====== #
results = sorted(results, key=lambda a: a[0])
results = [r[1] for r in results]
return results[0] if len(results) == 1 else results
def copy(self,
destination,
indices_filter=None,
data_filter=None,
override=False):
""" Copy the dataset to a new folder and closed
the old dataset
"""
from distutils.dir_util import copy_tree
read_only = self.read_only
# indices
if indices_filter is not None and \
not is_callable(indices_filter) and \
not isinstance(indices_filter, (tuple, list)):
raise ValueError(
'`indices_filter` must be callable, tuple, list or None')
if isinstance(indices_filter, (tuple, list)):
tmp = tuple(indices_filter)
indices_filter = lambda x: x in tmp
# data name
if data_filter is not None and \
not is_callable(data_filter) and \
not isinstance(data_filter, (tuple, list)):
raise ValueError(
'`data_filter` must be callable, tuple, list or None')
if isinstance(data_filter, (tuple, list)):
tmp = tuple(data_filter)
data_filter = lambda x: x in tmp
# ====== other files which are not Data ====== #
other_files = [i for i in os.listdir(self.path) if i not in self]
# ====== preprocessing ====== #
destination = os.path.abspath(str(destination))
if not os.path.exists(destination):
os.mkdir(destination)
elif not os.path.isdir(destination):
raise ValueError('path at "%s" must be a folder' % destination)
elif override:
shutil.rmtree(destination)
os.mkdir(destination)
else:
raise ValueError(
"A folder exist at path: '%s', cannot be overrided." %
destination)
# ====== copy everything ====== #
if indices_filter is None and data_filter is None:
print("Copying %s files from '%s' to '%s' ..." %
(ctext(len(self), 'cyan'), ctext(
self.path, 'yellow'), ctext(destination, 'yellow')))
copy_tree(self.path, destination)
# ====== only data_filter ====== #
elif indices_filter is None:
data_list = [i for i in self.keys() if data_filter(i)]
# copy all the data
for name in data_list:
org_path = os.path.join(self.path, name)
dst_path = os.path.join(destination, name)
print("Copying from '%s' to '%s' ..." %
(ctext(org_path, 'yellow'), ctext(dst_path, 'yellow')))
shutil.copy2(org_path, dst_path)
# copy all the related indices
for name in self.keys():
org_path = os.path.join(self.path, name)
dst_path = os.path.join(destination, name)
if not os.path.exists(dst_path) and \
('indices' == name or any(i in data_list for i in name.split('_')[1:])):
print("Copying Indices from '%s' to '%s'" %
(ctext(org_path, 'cyan'), ctext(dst_path, 'cyan')))
shutil.copy2(org_path, dst_path)
# ====== use indices_filter and data_filter ====== #
else:
if data_filter is None:
all_data = list(self.keys())
else:
all_data = [i for i in self.keys() if data_filter(i)]
# list of data with separated indices
separated_data = flatten_list(
[k.split('_')[1:] for k in self.keys() if 'indices_' == k[:8]])
# iterate over indices and copy one by one data
for ids_name in [k for k in self.keys() if 'indices' == k[:7]]:
indices = [(n, (s, e)) for n, (s, e) in self[ids_name]
if indices_filter(n)]
# no match indices, skip
if len(indices) == 0:
continue
nb_samples = sum(e - s for n, (s, e) in indices)
# get all data assigned to given indices
data = ids_name.split('_')[1:]
if len(data) == 0:
data = [i for i in all_data if i not in separated_data]
else:
data = [i for i in data if i in all_data]
# if still no data found, skip
if len(data) == 0:
continue
# copy each data
for data_name in data:
X = self[data_name]
# copy big MmapDict
if isinstance(X, MmapDict) and len(X) == len(
self[ids_name]):
new_path = os.path.join(destination,
os.path.basename(X.path))
print("Copying MmapDict from '%s' to '%s'" %
(ctext(X.path, 'cyan'), ctext(new_path, 'cyan')))
new_dict = MmapDict(new_path,
cache_size=80000,
read_only=False)
for n, (s, e) in indices:
new_dict[n] = X[n]
new_dict.flush(save_all=True)
new_dict.close()
# copy MmapData
elif isinstance(X, MmapData):
Y = MmapData(path=os.path.join(destination, data_name),
dtype=X.dtype,
shape=(0, ) + X.shape[1:],
read_only=False)
prog = Progbar(target=nb_samples,
print_report=True,
print_summary=True,
name="Copying data: '%s' to path:'%s'" %
(ctext(data_name, 'yellow'),
ctext(Y.data_info, 'cyan')))
for n, (s, e) in indices:
Y.append(X[s:e])
prog.add(e - s)
# unknown data-type
else:
org_path = os.path.join(self.path, data_name)
new_path = os.path.join(destination, data_name)
# just copy directly the files
if os.path.isfile(org_path) or \
not os.path.exists(new_path):
shutil.copy2(org_path, new_path)
print("Copying '%s' to '%s' ..." % (ctext(
org_path, 'cyan'), ctext(new_path, 'yellow')))
else:
wprint("Cannot copy: '%s' - %s" %
(ctext(data_name, 'cyan'),
ctext(type(self[data_name]), 'yellow')))
# copy the indices
new_indices = MmapDict(os.path.join(destination, ids_name),
cache_size=80000,
read_only=False)
start = 0
for n, (s, e) in indices:
size = e - s
new_indices[n] = (start, start + size)
start += size
new_indices.flush(save_all=True)
new_indices.close()
# ====== copy others files ====== #
for f in other_files:
org_path = os.path.join(self.path, f)
dst_path = os.path.join(destination, f)
if not os.path.exists(dst_path):
if os.path.isdir(org_path): # directory
copy_tree(org_path, dst_path)
else: # single file
shutil.copy2(org_path, dst_path)
# ====== readme ====== #
readme_name = os.path.basename(self._readme_path)
dst_path = os.path.join(destination, readme_name)
if not os.path.exists(dst_path):
shutil.copy2(self._readme_path, dst_path)
return Dataset(destination, read_only=read_only)
def fit(self, X, y=None, cv=None):
self._initialize(X)
if not hasattr(X, 'shape') or not hasattr(X, '__iter__') or \
not hasattr(X, '__len__'):
raise ValueError("`X` must has 'shape', '__len__' and '__iter__' attributes")
nb_train_samples = len(X)
# convert to odin.fuel.Data if possible
if isinstance(X, (np.ndarray, list, tuple)):
X = F.as_data(X)
if isinstance(y, (np.ndarray, list, tuple)):
y = F.as_data(y)
start_tr = 0
end_tr = nb_train_samples
# ====== check if cross validating ====== #
create_it_cv = None
if is_number(cv):
cv = int(float(cv) * nb_train_samples) if cv < 1. else int(cv)
end_tr = nb_train_samples - cv
start_cv = end_tr
end_cv = nb_train_samples
nb_cv_samples = end_cv - start_cv
create_it_cv = _create_it_func(X=X, y=y, batch_size=self.batch_size,
start=start_cv, end=end_cv)
elif isinstance(cv, (tuple, list)):
X_cv, y_cv = cv
nb_cv_samples = X_cv.shape[0]
create_it_cv = _create_it_func(X=X_cv, y=y_cv, batch_size=self.batch_size,
start=0, end=X_cv.shape[0])
elif hasattr(cv, 'set_batch'):
nb_cv_samples = cv.shape[0]
create_it_cv = _create_it_func(X=cv, y=None, batch_size=self.batch_size,
start=0, end=cv.shape[0])
elif cv is not None:
raise ValueError('`cv` can be float (0-1), tuple or list of X and y, '
'any object that have "shape" and "__iter__" attributes, '
'or None')
# ====== preprocessing ====== #
create_it = _create_it_func(X=X, y=y, batch_size=self.batch_size,
start=start_tr, end=end_tr)
# ====== prepare ====== #
curr_niter = sum(epoch[0] for epoch in self._train_history)
curr_nepoch = len(self._train_history)
curr_patience = int(self.patience)
last_losses = None
last_checkpoint = None
best_epoch = None
is_converged = False
# ====== fitting ====== #
while not is_converged:
curr_nepoch += 1
seed = self._rand_state.randint(0, 10e8)
# ====== training ====== #
nb_iter, duration, results = _fitting_helper(create_it(seed),
fn=self._f_train,
nb_samples=nb_train_samples,
nb_classes=self.nb_classes,
title='Epoch %d' % curr_nepoch)
curr_niter += nb_iter
self._train_history.append(
(nb_train_samples, nb_iter, duration, results))
# ====== cross validation ====== #
if create_it_cv is not None:
nb_iter, duration_valid, results = _fitting_helper(create_it_cv(seed),
fn=self._f_score,
nb_samples=nb_cv_samples,
nb_classes=self.nb_classes,
title="Validating")
self._valid_history.append(
(nb_train_samples, nb_iter, duration_valid, results))
duration += duration_valid
# ====== print log ====== #
if self.verbose >= 2:
print(ctext('#epoch:', 'cyan') + str(curr_nepoch),
ctext('#iter:', 'cyan') + str(curr_niter),
ctext("Loss:", 'yellow') + '%.5f' % results[0],
ctext("Acc:", 'yellow') + '%.3f' % results[1],
ctext("%.2f(s)" % duration, 'magenta'))
if self.confusion_matrix and (curr_nepoch - 1) % 8 == 0:
print(V.print_confusion(results[-1], labels=self.labels))
# ====== early stopping ====== #
losses = results[0]
if last_checkpoint is None: # first check point
last_checkpoint = self.parameters
if last_losses is not None:
# degraded, smaller is better
if last_losses - losses <= self.tol:
curr_patience -= 1
if self.rollback:
if self.verbose >= 2:
wprint('[LogisticRegression] Rollback to the best checkpoint '
'at epoch:%s patience:%s' %
(ctext(best_epoch, 'cyan'),
ctext(curr_patience, 'cyan')))
self.set_parameters(*last_checkpoint)
# save best checkpoint
else:
last_checkpoint = self.parameters
best_epoch = curr_nepoch
if self._path is not None:
with open(self._path, 'wb') as f:
pickle.dump(self, f)
last_losses = losses
if curr_patience <= 0:
is_converged = True
# end the training
if self.max_iter is not None and \
curr_niter >= self.max_iter:
break
if self.max_epoch is not None and \
curr_nepoch >= self.max_epoch:
break
# ====== print summary plot ====== #
if self.verbose >= 1:
train_losses = [epoch[-1][0] for epoch in self._train_history]
print(V.print_bar(train_losses, height=12,
bincount=min(20, len(train_losses)),
title='Training Losses'))
if create_it_cv is not None:
valid_losses = [epoch[-1][0] for epoch in self._valid_history]
print(V.print_bar(valid_losses, height=12,
bincount=min(20, len(train_losses)),
title='Validation Losses'))
if self.confusion_matrix:
print(ctext("======== Training Confusion Matrix ========", 'cyan'))
print(V.print_confusion(arr=self._train_history[-1][-1][-1],
labels=self.labels))
if create_it_cv is not None:
print(ctext("======== Validation Confusion Matrix ========", 'cyan'))
print(V.print_confusion(arr=self._valid_history[-1][-1][-1],
labels=self.labels))
# ====== reset to best points ====== #
self.set_parameters(*last_checkpoint)
self._is_fitted = True
if self._path is not None:
with open(self._path, 'wb') as f:
pickle.dump(self, f)
def copy(self, destination,
indices_filter=None, data_filter=None,
override=False):
""" Copy the dataset to a new folder and closed
the old dataset
"""
from distutils.dir_util import copy_tree
read_only = self.read_only
# indices
if indices_filter is not None and \
not is_callable(indices_filter) and \
not isinstance(indices_filter, (tuple, list)):
raise ValueError('`indices_filter` must be callable, tuple, list or None')
if isinstance(indices_filter, (tuple, list)):
tmp = tuple(indices_filter)
indices_filter = lambda x: x in tmp
# data name
if data_filter is not None and \
not is_callable(data_filter) and \
not isinstance(data_filter, (tuple, list)):
raise ValueError('`data_filter` must be callable, tuple, list or None')
if isinstance(data_filter, (tuple, list)):
tmp = tuple(data_filter)
data_filter = lambda x: x in tmp
# ====== other files which are not Data ====== #
other_files = [i for i in os.listdir(self.path)
if i not in self]
# ====== preprocessing ====== #
destination = os.path.abspath(str(destination))
if not os.path.exists(destination):
os.mkdir(destination)
elif not os.path.isdir(destination):
raise ValueError('path at "%s" must be a folder' % destination)
elif override:
shutil.rmtree(destination)
os.mkdir(destination)
else:
raise ValueError("A folder exist at path: '%s', cannot be overrided." %
destination)
# ====== copy everything ====== #
if indices_filter is None and data_filter is None:
print("Copying %s files from '%s' to '%s' ..." %
(ctext(len(self), 'cyan'),
ctext(self.path, 'yellow'),
ctext(destination, 'yellow')))
copy_tree(self.path, destination)
# ====== only data_filter ====== #
elif indices_filter is None:
data_list = [i for i in self.keys() if data_filter(i)]
# copy all the data
for name in data_list:
org_path = os.path.join(self.path, name)
dst_path = os.path.join(destination, name)
print("Copying from '%s' to '%s' ..." %
(ctext(org_path, 'yellow'),
ctext(dst_path, 'yellow')))
shutil.copy2(org_path, dst_path)
# copy all the related indices
for name in self.keys():
org_path = os.path.join(self.path, name)
dst_path = os.path.join(destination, name)
if not os.path.exists(dst_path) and \
('indices' == name or any(i in data_list for i in name.split('_')[1:])):
print("Copying Indices from '%s' to '%s'" % (ctext(org_path, 'cyan'),
ctext(dst_path, 'cyan')))
shutil.copy2(org_path, dst_path)
# ====== use indices_filter and data_filter ====== #
else:
if data_filter is None:
all_data = list(self.keys())
else:
all_data = [i for i in self.keys()
if data_filter(i)]
# list of data with separated indices
separated_data = flatten_list(
[k.split('_')[1:] for k in self.keys()
if 'indices_' == k[:8]])
# iterate over indices and copy one by one data
for ids_name in [k for k in self.keys() if 'indices' == k[:7]]:
indices = [(n, (s, e))
for n, (s, e) in self[ids_name]
if indices_filter(n)]
# no match indices, skip
if len(indices) == 0:
continue
nb_samples = sum(e - s for n, (s, e) in indices)
# get all data assigned to given indices
data = ids_name.split('_')[1:]
if len(data) == 0:
data = [i for i in all_data if i not in separated_data]
else:
data = [i for i in data if i in all_data]
# if still no data found, skip
if len(data) == 0:
continue
# copy each data
for data_name in data:
X = self[data_name]
# copy big MmapDict
if isinstance(X, MmapDict) and len(X) == len(self[ids_name]):
new_path = os.path.join(destination, os.path.basename(X.path))
print("Copying MmapDict from '%s' to '%s'" % (
ctext(X.path, 'cyan'),
ctext(new_path, 'cyan')))
new_dict = MmapDict(new_path, cache_size=80000, read_only=False)
for n, (s, e) in indices:
new_dict[n] = X[n]
new_dict.flush(save_all=True)
new_dict.close()
# copy MmapData
elif isinstance(X, MmapData):
Y = MmapData(path=os.path.join(destination, data_name),
dtype=X.dtype, shape=(0,) + X.shape[1:],
read_only=False)
prog = Progbar(target=nb_samples,
print_report=True, print_summary=True,
name="Copying data: '%s' to path:'%s'" %
(ctext(data_name, 'yellow'),
ctext(Y.data_info, 'cyan')))
for n, (s, e) in indices:
Y.append(X[s:e])
prog.add(e - s)
# unknown data-type
else:
org_path = os.path.join(self.path, data_name)
new_path = os.path.join(destination, data_name)
# just copy directly the files
if os.path.isfile(org_path) or \
not os.path.exists(new_path):
shutil.copy2(org_path, new_path)
print("Copying '%s' to '%s' ..." %
(ctext(org_path, 'cyan'), ctext(new_path, 'yellow')))
else:
wprint("Cannot copy: '%s' - %s" %
(ctext(data_name, 'cyan'),
ctext(type(self[data_name]), 'yellow')))
# copy the indices
new_indices = MmapDict(os.path.join(destination, ids_name),
cache_size=80000, read_only=False)
start = 0
for n, (s, e) in indices:
size = e - s
new_indices[n] = (start, start + size)
start += size
new_indices.flush(save_all=True)
new_indices.close()
# ====== copy others files ====== #
for f in other_files:
org_path = os.path.join(self.path, f)
dst_path = os.path.join(destination, f)
if not os.path.exists(dst_path):
if os.path.isdir(org_path): # directory
copy_tree(org_path, dst_path)
else: # single file
shutil.copy2(org_path, dst_path)
# ====== readme ====== #
readme_name = os.path.basename(self._readme_path)
dst_path = os.path.join(destination, readme_name)
if not os.path.exists(dst_path):
shutil.copy2(self._readme_path, dst_path)
return Dataset(destination, read_only=read_only)
def __init__(self, jobs, path, extractor,
n_cache=0.12, ncpu=1, override=False,
identifier='name',
log_path=None,
stop_on_failure=False):
super(FeatureProcessor, self).__init__()
# ====== check outpath ====== #
path = os.path.abspath(str(path))
if os.path.isfile(path):
raise ValueError("`path` must be path to a directory, but found a "
"path to file.")
# check override
if os.path.exists(path) and override:
wprint("Remove existed Dataset at path: %s" % path)
for i in os.listdir(path):
i = os.path.join(path, i)
if os.path.isdir(i): # remove folder
shutil.rmtree(i)
else: # remove file
os.remove(i)
# set path and name
self.path = path
# ====== check jobs ====== #
if not isinstance(jobs, (tuple, list, np.ndarray)):
raise ValueError("Provided `jobs` must be instance of tuple, list or ndarray.")
if isinstance(jobs, np.ndarray):
jobs = jobs.tolist()
self.jobs = tuple(jobs)
# ====== check multiprocessing ====== #
if ncpu is None: # auto select number of CPU
ncpu = min(len(jobs), cpu_count() - 1)
ncpu = int(ncpu)
if ncpu <= 0 or n_cache <= 0:
raise ValueError('`ncpu` and `n_cache` must be greater than 0, but '
'given values ncpu=%d n_cache=%f' % (ncpu, n_cache))
self.n_cpu = ncpu
self.n_cache = n_cache
# ====== internal control for feature processor ====== #
if isinstance(extractor, Pipeline):
pass
elif isinstance(extractor, (tuple, list)):
steps = [('%s_%d' % (e.__class__.__name__, i), e)
for i, e in enumerate(extractor)]
extractor = Pipeline(steps=steps)
elif isinstance(extractor, Mapping):
steps = [(str(n), e) for n, e in extractor.items()]
extractor = Pipeline(steps=steps)
elif isinstance(extractor, Extractor):
extractor = Pipeline(
steps=[(extractor.__class__.__name__, extractor)])
self.extractor = extractor
# ====== check identifier and log path ====== #
self._identifier = str(identifier)
if log_path is None:
log_path = os.path.join(self.path, 'log.txt')
else:
log_path = str(log_path)
self._log_path = _check_logpath(log_path)
# ====== others ====== #
self.config = {}
self._error_log = []
self.stop_on_failure = bool(stop_on_failure)