def eval_metric(metric_name_or_func,
*args,
remove_unused_kwargs=False,
**kwargs):
metric = metric_name_or_func
if is_str(metric):
metric = _get_metric_func_by_name(metric)
if callable(metric):
if remove_unused_kwargs:
kwargs = filter_unused_kwargs(metric, kwargs)
metric_value = metric(*args, **kwargs)
result = metric_value
else:
raise ValueError('Unknown metric: {}'.format(metric))
return result
def eval_metrics(metric_names_or_funcs,
*args,
remove_unused_kwargs=False,
**kwargs):
results = {}
throw_out = kwargs.pop('throw_out', 0) # TODO throw out first x samples...
for metric in metric_names_or_funcs:
if throw_out: # TODO
args = list(args)
for i, arg in enumerate(args): # TODO
arg = arg[throw_out:] # TODO
args[i] = arg # TODO :'(
metric_value = eval_metric(metric,
*args,
remove_unused_kwargs=remove_unused_kwargs,
**kwargs)
if not is_str(metric):
metric = _get_metric_name_by_func(metric)
results[metric] = metric_value
return results
def get_cv(cv=3,
n_repeats=None,
y=None,
classification=None,
random_state=None,
shuffle=True):
"""Input checker utility for building a cross-validator
Args:
cv: int, cross-validation generator or an iterable
Determines the cross-validation splitting strategy.
Possible inputs for cv are:
- None, to use the default 3-fold cross-validation,
- integer, to specify the number of folds.
- An object to be used as a cross-validation generator.
- An iterable yielding train/test splits.
For integer/None inputs, if classification is True and
`y` is either binary or multiclass, `StratifiedKFold` is
used. In all other cases, `KFold` is used.
n_repeats: The number of times to repeat splits.
y: The target variable for supervised learning problems.
classification:
Whether the task is a classification task, in which case
stratified KFold will be used. Infers based on `y` if
specified and `classification` is None.
random_state:
Random state to be used to generate random state for each
repetition.
shuffle: Whether to shuffle.
Returns:
A cross-validator instance that generates the train/test splits via
its `split` method.
"""
if cv is None:
cv = 3
if n_repeats is None:
n_repeats = 1
elif n_repeats > 1 and not shuffle:
raise ValueError('shuffle cannot be False with more than 1 CV '
'repetition.')
if is_int(cv, ignore_unsigned=False):
# Infer classification if None and y is provided
if classification is None:
if ((y is not None)
and (type_of_target(y) in ('binary', 'multiclass'))):
classification = True
else:
classification = False
# Select appropriate CV type
if classification:
if n_repeats == 1:
return StratifiedKFold(n_splits=cv,
shuffle=shuffle,
random_state=random_state)
else:
return RepeatedStratifiedKFold(n_splits=cv,
n_repeats=n_repeats,
random_state=random_state)
else:
if n_repeats == 1:
return KFold(n_splits=cv,
shuffle=shuffle,
random_state=random_state)
else:
return RepeatedKFold(n_splits=cv,
n_repeats=n_repeats,
random_state=random_state)
# For iterables and error-handling (rely on sklearn)
if not hasattr(cv, 'split') or is_str(cv):
return check_cv(cv)
return cv
def get_most_recent_k_in_dir(dirname,
k,
delim='_',
ext=None,
raise_=False,
strict=False,
return_fns=False):
if is_str(dirname):
# Note this checks files only.
if not raise_ and not os.path.isdir(dirname):
return None
dirs = os.listdir(dirname) # Exception can be raised here
if not dirs:
if raise_:
raise EmptyDirError('Directory "{}" cannot be '
'empty.'.format(dirname))
else:
return None
filenames = os.listdir(dirname)
else:
# Assume iterable
filenames = dirname
fn_dt_dict = {}
for filename in filenames:
if os.path.isfile(filename):
filename_raw = filename
if ext is not None:
suffix = '.' + ext
if not filename.endswith(suffix):
continue
filename = filename[:-len(suffix)]
if delim is not None:
split = filename.rsplit(delim, 1)
if len(split) != 2:
continue
filename = split[1]
try:
dt = date_parser.parse(filename)
except ValueError:
continue
fn_dt_dict[filename_raw] = dt
if not fn_dt_dict and raise_:
raise EmptyDirError(
'Directory "{}" does not contain any files with {}{}a valid '
'timestamp.'.format(
dirname, 'extension "{}" and '.format(ext) if ext else '',
'delimiter "{}" and '.format(delim) if delim else ''))
most_recent_fns = sorted(fn_dt_dict, key=fn_dt_dict.get, reverse=True)[:k]
if strict and len(most_recent_fns) != k:
raise ValueError(
'Directory "{}" does not contain {} files with {}{}a valid '
'timestamp.'.format(
dirname, k, 'extension "{}" and '.format(ext) if ext else '',
'delimiter "{}" and '.format(delim) if delim else ''))
if return_fns:
return most_recent_fns, list(fn_dt_dict.keys())
else:
return most_recent_fns
def train_and_eval(X,
y,
model,
train=True,
test=True,
train_func=None,
test_func=None,
cv=None,
iid=True,
return_train_score=False,
metrics=None,
target_metric=None,
time_series=False,
eval_kwargs=None):
"""
Args:
X:
y:
model:
train:
test:
train_func:
test_func:
cv:
iid:
return_train_score:
metrics:
target_metric:
time_series:
eval_kwargs:
Returns:
"""
if return_train_score:
raise NotImplementedError # TODO
if not metrics or target_metric not in metrics:
raise ValueError('Invalid specification of metrics for evaluation.')
if target_metric is None:
if len(metrics) == 1: # TODO nonetype metrics...
target_metric = metrics[0]
else:
raise ValueError('"target_metric" must be provided if multiple '
'metrics specified.')
eval_kwargs = eval_kwargs or {}
if type(X) is type(y) is tuple and len(X) == len(
y
) == 2: # TODO this good? for manual split specification...maybe require wrapping in object...
splits = [(X, y)]
cv = False
else:
cv = get_cv(cv)
splits = cv.split(X, y)
# Check specified train/eval functions
if isinstance(
model, SKBaseEstimator
): # TODO you also know that it has the {set, get}_params methods...
if train and train_func is None:
if hasmethod(model, DEFAULT_SKLEARN_TRAIN_FUNC):
train_func = getattr(model, DEFAULT_SKLEARN_TRAIN_FUNC)
else:
raise ValueError('Could not infer the train method from the '
'specified sklearn model. Please specify '
'"train_func" in order to train the model.')
if test and test_func is None:
if hasmethod(model, DEFAULT_SKLEARN_TEST_PROBA_FUNC):
test_func = getattr(model, DEFAULT_SKLEARN_TEST_PROBA_FUNC)
elif hasmethod(model, DEFAULT_SKLEARN_TEST_FUNC):
test_func = getattr(model, DEFAULT_SKLEARN_TEST_FUNC)
else:
raise ValueError('Could not infer the predict method from the '
'specified sklearn model. Please specify '
'"test_func" in order to test the model.')
elif isinstance(model, BaseModel):
raise NotImplementedError # TODO
else:
if train and train_func is None:
# Take a stab at it
if hasmethod(model, DEFAULT_MODEL_TRAIN_FUNC):
train_func = getattr(model, DEFAULT_MODEL_TRAIN_FUNC)
else:
raise ValueError(
'Could not infer model train function: please '
'specify "train_func" in order to train the '
'model.')
if test and test_func is None:
# Take a stab at it
if hasmethod(model, DEFAULT_MODEL_TEST_FUNC):
test_func = getattr(model, DEFAULT_MODEL_TEST_FUNC)
else:
raise ValueError('Could not infer model test function: please '
'specify "test_func" in order to test the '
'model.')
if is_str(train_func):
train_func = getattr(model, train_func)
if is_str(test_func):
test_func = getattr(model, test_func)
test_score = 0.
test_scores = None
n_test = 0
for train_idx_or_xs, test_idx_or_ys in splits:
if len(train_idx_or_xs) <= 0 or len(test_idx_or_ys) <= 0:
raise ValueError(
'Train and test must be long enough for specified '
'cross validation.')
if cv:
X_train = X[train_idx_or_xs]
y_train = y[train_idx_or_xs]
X_test = X[test_idx_or_ys]
y_test = y[test_idx_or_ys]
else:
X_train, X_test = train_idx_or_xs
y_train, y_test = test_idx_or_ys
if time_series:
test_len = X_test.shape[2]
else:
test_len = len(X_test)
if train_func is not None:
# Train the model
y_train_pred = train_func(X_train, y_train)
if test_func is not None:
y_test_pred = test_func(X_test)
test_scores = eval_metrics(metrics, y_test, y_test_pred,
**eval_kwargs)
print(test_scores) # TODO
target_score = test_scores[target_metric]
if iid:
test_score += target_score * test_len
n_test += test_len
else:
test_score += target_score
n_test += 1
if n_test == 0:
raise ValueError('No evaluation was done (n_test = 0).')
test_score /= n_test
return test_score, test_scores # TODO