def test_find_bad_by_ransac(raw_tmp):
"""Test the RANSAC component of NoisyChannels."""
# Set a consistent random seed for all RANSAC runs
RANSAC_RNG = 435656
# RANSAC identifies channels that go bad together and are highly correlated.
# Inserting highly correlated signal in channels 0 through 6 at 30 Hz
raw_tmp._data[0:6, :] = _generate_signal(30, 30, raw_tmp.times)
# Run different variations of RANSAC on the same data
test_matrix = {
# List items represent [matlab_strict, channel_wise, max_chunk_size]
"by_window": [False, False, None],
"by_channel": [False, True, None],
"by_channel_maxchunk": [False, True, 2],
"by_window_strict": [True, False, None],
"by_channel_strict": [True, True, None],
}
bads = {}
corr = {}
for name, args in test_matrix.items():
nd = NoisyChannels(raw_tmp,
do_detrend=False,
random_state=RANSAC_RNG,
matlab_strict=args[0])
nd.find_bad_by_ransac(channel_wise=args[1], max_chunk_size=args[2])
# Save bad channels and RANSAC correlation matrix for later comparison
bads[name] = nd.bad_by_ransac
corr[name] = nd._extra_info["bad_by_ransac"]["ransac_correlations"]
# Test whether all methods detected bad channels properly
assert bads["by_window"] == raw_tmp.ch_names[0:6]
assert bads["by_channel"] == raw_tmp.ch_names[0:6]
assert bads["by_channel_maxchunk"] == raw_tmp.ch_names[0:6]
assert bads["by_window_strict"] == raw_tmp.ch_names[0:6]
assert bads["by_channel_strict"] == raw_tmp.ch_names[0:6]
# Make sure non-strict correlation matrices all match
assert np.allclose(corr["by_window"], corr["by_channel"])
assert np.allclose(corr["by_window"], corr["by_channel_maxchunk"])
# Make sure MATLAB-strict correlation matrices match
assert np.allclose(corr["by_window_strict"], corr["by_channel_strict"])
# Make sure strict and non-strict matrices differ
assert not np.allclose(corr["by_window"], corr["by_window_strict"])
# Ensure that RANSAC doesn't change random state if in MATLAB-strict mode
rng = RandomState(RANSAC_RNG)
init_state = rng.get_state()[2]
nd = NoisyChannels(raw_tmp,
do_detrend=False,
random_state=rng,
matlab_strict=True)
nd.find_bad_by_ransac()
assert rng.get_state()[2] == init_state
def test_RandomRectangularPattern_ca_3ch_postit(self):
rso = RandomState(1)
state_tuple = rso.get_state()
t = image_triggers.RandomRectangularPattern(
3,
3,
3,
color_algorithm='channel_assign',
color_options={'cval': [255, 254, 253]},
pattern_style='postit',
random_state_obj=rso)
actual_img = t.get_data()
actual_mask = t.get_mask()
# reset the random state and generate the pattern in the same manner
rso.set_state(state_tuple)
per_chan_expected_img = rso.choice(2, 3 * 3).reshape(
(3, 3)).astype(bool)
expected_img = np.zeros((3, 3, 3))
expected_img[:, :, 0] = per_chan_expected_img * 255 # the color
expected_img[:, :, 1] = per_chan_expected_img * 254 # the color
expected_img[:, :, 2] = per_chan_expected_img * 253 # the color
expected_mask = np.ones((3, 3)).astype(bool)
self.assertTrue(np.array_equal(actual_img, expected_img))
self.assertTrue(np.array_equal(actual_mask, expected_mask))
def make_trial():
try:
# t0 = time()
# args:
# lenscale
# nPassiveObs
# rngstate
params = request.json # works when ajax request contentType specified as "applications/json"
# unpack random number generator
rng = RandomState()
rngstate = unpack_rngstate(params['rngstate'])
rng.set_state(rngstate)
lenscale = float(params['lenscale'])
nPassiveObs = int(params['nPassiveObs'])
thisTri = boe.make_trial(nPassiveObs, DOMAIN, lenscale, SIGVAR, NOISEVAR2, XSAM_BOUNDS, rng)
resp = {'sample': thisTri['sample'].tolist(),
'xObs': thisTri['xObs'].flatten().tolist(),
'yObs': thisTri['yObs'].tolist(),
'iObs': thisTri['iObs'].tolist(),
'rngstate': pack_rngstate(rng.get_state())}
except:
raise ExperimentError('improper_inputs') # i don't like returning HTML to JSON requests... maybe should change this
return jsonify(**resp)
def init_experiment():
if not request.args.has_key('condition'):
raise ExperimentError('improper_inputs') # i don't like returning HTML to JSON requests... maybe should change this
condition = int(request.args['condition'])
counterbalance = int(request.args['counterbalance'])
## END FREE VARS
lenscale = LENSCALEPOOL[condition]
rngseed = RNGSEEDPOOL[counterbalance]
rng = RandomState(rngseed)
experParams = {
'nTrial': NTRIAL,
'nPassivePool': NPASSIVEPOOL,
'nActivePool': NACTIVEPOOL,
'rng': rng
}
subParams = boe.make_experiment(**experParams)
# bundle response to send
resp = {}
for f in subParams:
try:
resp[f] = subParams[f].tolist()
except:
resp[f] = subParams[f]
for f in experParams:
if f is not 'rng': # don't pass the random number generator
try: # convet numpy array to list if possible
resp[f] = experParams[f].tolist()
except:
resp[f] = experParams[f]
resp['itrial'] = -1
resp['isam3'] = -1
resp['nTrial'] = NTRIAL
resp['costToDrill'] = COSTTODRILL
resp['costToSample'] = COSTTOSAMPLE
resp['initscore'] = INITSCORE
resp['lenscale'] = lenscale
resp['sigvar'] = SIGVAR
resp['domainbounds'] = DOMAINBOUNDS
resp['domainres'] = DOMAINRES
resp['edgebuf'] = EDGEBUF
resp['rngstate'] = pack_rngstate(rng.get_state())
return jsonify(**resp)
def test_iter_equal_pre_gen(self, dataset_cls):
"""Given the same random-number-generator state,
both pre-generated and lazily generated data-set should result
in the same samples."""
size = self.PRE_GEN_DATASET_DEFAULT_SIZE
seed = self.DATASETS_DEFAULT_SEED
pre_generated = dataset_cls.pre_generate(size, seed)
lazily_generated = dataset_cls()
rnd = RandomState(seed)
lazily_generated.random_state = rnd.get_state()
for i, sample in enumerate(lazily_generated):
if i >= size - 1:
break
print(self.DATASET_CLS)
assert sample == pre_generated[i], \
f'Samples at iteration {i} don\'t match!'
def test_unequal_reset():
def mean_diff(*args):
return args[0].mean() - args[1].mean()
rs = RandomState(0)
x = rs.standard_normal(800)
y = rs.standard_normal(200)
orig_state = rs.get_state()
bs = IndependentSamplesBootstrap(x, y, random_state=rs)
variance = bs.var(mean_diff)
assert variance > 0
bs.reset()
state = bs.get_state()
assert_equal(state[1], orig_state[1])
bs = IndependentSamplesBootstrap(x, y)
bs.seed(0)
orig_state = bs.get_state()
bs.var(mean_diff)
bs.reset(use_seed=True)
state = bs.get_state()
assert_equal(state[1], orig_state[1])
def test_unequal_reset():
def mean_diff(*args):
return args[0].mean() - args[1].mean()
rs = RandomState(0)
x = rs.randn(800)
y = rs.randn(200)
orig_state = rs.get_state()
bs = IndependentSamplesBootstrap(x, y, random_state=rs)
variance = bs.var(mean_diff)
assert variance > 0
bs.reset()
state = bs.get_state()
assert_equal(state[1], orig_state[1])
bs = IndependentSamplesBootstrap(x, y)
bs.seed(0)
orig_state = bs.get_state()
bs.var(mean_diff)
bs.reset(use_seed=True)
state = bs.get_state()
assert_equal(state[1], orig_state[1])
class NumpyRandomGenerator(Generator):
"""
Generator wrapping any numpy.Random method.
"""
def __init__(self, method, seed, **numpy_parameters):
"""Initialise a random number generator
:param method: string: must be a valid numpy.Randomstate method that
accept the "size" parameter
:param numpy_parameters: dict, see descriptions below
:param seed: int, seed of the generator
:return: create a random number generator of type "gen_type", with its parameters and seeded.
"""
Generator.__init__(self)
self.method = method
self.numpy_parameters = numpy_parameters
self.state = RandomState(seed)
self.numpy_method = getattr(self.state, method)
def generate(self, size):
all_params = merge_2_dicts({"size": size}, self.numpy_parameters)
return self.numpy_method(**all_params)
def description(self):
return {
"type": "NumpyRandomGenerator",
"method": self.method,
"numpy_parameters": self.numpy_parameters
}
def save_to(self, output_file):
logging.info("saving generator to {}".format(output_file))
# saving the numpy RandomState instance, converting the numpy array
# to enable json serialization
np_state = self.state.get_state()
state = {
"method":
self.method,
"numpy_parameters":
self.numpy_parameters,
"numpy_state": (np_state[0], np_state[1].tolist(), np_state[2],
np_state[3], np_state[4])
}
with open(output_file, "w") as outf:
json.dump(state, outf, indent=4)
@staticmethod
def load_from(input_file):
logging.info("loading numpy generator from {}".format(input_file))
with open(input_file, "r") as inf:
json_payload = json.load(inf)
# Initializing the generator with an incorrect seed just to make
# the constructor happy, then setting the state
gen = NumpyRandomGenerator(method=json_payload["method"],
seed=1234,
**json_payload["numpy_parameters"])
# retrieving the numpy state + converting list to np.array as needed
state_raw_ = json_payload["numpy_state"]
np_state = (state_raw_[0], np.array(state_raw_[1]), state_raw_[2],
state_raw_[3], state_raw_[4])
gen.state = np.random.RandomState(seed=1234)
gen.state.set_state(np_state)
return gen
})
# setup data generation
# Setup the files based on user inputs
data_folder = os.path.abspath(a.data_folder)
toplevel_folder = a.experiment_path
# check if the data_folder has the cifar10 data, if not download it
data_folder = cifar10.download_and_extract(data_folder)
train_output_csv_file = 'train_cifar10.csv'
test_output_csv_file = 'test_cifar10.csv'
MASTER_SEED = 1234
master_random_state_object = RandomState(MASTER_SEED)
start_state = master_random_state_object.get_state()
# define a configuration which triggers data by applying the Gotham Instagram Filter
datagen_per_class_trigger_frac = 0.25
gotham_trigger_cfg = \
tdc.XFormMergePipelineConfig(
# setup the list of possible triggers that will be inserted into the CIFAR10 data.
trigger_list=[],
# tell the trigger inserter the probability of sampling each type of trigger specified in the trigger
# list. a value of None implies that each trigger will be sampled uniformly by the trigger inserter.
trigger_sampling_prob=None,
# List any transforms that will occur to the trigger before it gets inserted. In this case, we do none.
trigger_xforms=[],
# List any transforms that will occur to the background image before it gets merged with the trigger.
trigger_bg_xforms=[tinstx.GothamFilterXForm()],
# List how we merge the trigger and the background. Because we don't insert a point trigger,
class IIDBootstrap(object):
"""
Bootstrap using uniform resampling
Parameters
----------
args
Positional arguments to bootstrap
kwargs
Keyword arguments to bootstrap
Attributes
----------
index : array
The current index of the bootstrap
data : tuple
Two-element tuple with the pos_data in the first position and kw_data
in the second (pos_data, kw_data)
pos_data : tuple
Tuple containing the positional arguments (in the order entered)
kw_data : dict
Dictionary containing the keyword arguments
random_state : RandomState
RandomState instance used by bootstrap
Notes
-----
Supports numpy arrays and pandas Series and DataFrames. Data returned has
the same type as the input date.
Data entered using keyword arguments is directly accessibly as an
attribute.
Examples
--------
Data can be accessed in a number of ways. Positional data is retained in
the same order as it was entered when the bootstrap was initialized.
Keyword data is available both as an attribute or using a dictionary syntax
on kw_data.
>>> from arch.bootstrap import IIDBootstrap
>>> from numpy.random import standard_normal
>>> y = standard_normal((500, 1))
>>> x = standard_normal((500,2))
>>> z = standard_normal(500)
>>> bs = IIDBootstrap(x, y=y, z=z)
>>> for data in bs.bootstrap(100):
... bs_x = data[0][0]
... bs_y = data[1]['y']
... bs_z = bs.z
"""
def __init__(self, *args, **kwargs):
self.random_state = RandomState()
self._initial_state = self.random_state.get_state()
self._args = args
self._kwargs = kwargs
if args:
self._num_items = len(args[0])
elif kwargs:
key = list(kwargs.keys())[0]
self._num_items = len(kwargs[key])
all_args = list(args)
all_args.extend([v for v in itervalues(kwargs)])
for arg in all_args:
if len(arg) != self._num_items:
raise ValueError("All inputs must have the same number of "
"elements in axis 0")
self._index = np.arange(self._num_items)
self._parameters = []
self._seed = None
self.pos_data = args
self.kw_data = kwargs
self.data = (args, kwargs)
self._base = None
self._results = None
self._studentized_results = None
self._last_func = None
self._name = 'IID Bootstrap'
for key, value in iteritems(kwargs):
attr = getattr(self, key, None)
if attr is None:
self.__setattr__(key, value)
else:
raise ValueError(key + ' is a reserved name')
def __str__(self):
repr = self._name
repr += '(no. pos. inputs: ' + str(len(self.pos_data))
repr += ', no. keyword inputs: ' + str(len(self.kw_data)) + ')'
return repr
def __repr__(self):
return self.__str__()[:-1] + ', ID: ' + hex(id(self)) + ')'
def _repr_html(self):
html = '<strong>' + self._name + '</strong>('
html += '<strong>no. pos. inputs</strong>: ' + str(len(self.pos_data))
html += ', <strong>no. keyword inputs</strong>: ' + \
str(len(self.kw_data))
html += ', <strong>ID</strong>: ' + hex(id(self)) + ')'
return html
@property
def index(self):
"""
Returns the current index of the bootstrap
"""
return self._index
def get_state(self):
"""
Gets the state of the bootstrap's random number generator
Returns
-------
state : RandomState state vector
Array containing the state
"""
return self.random_state.get_state()
def set_state(self, state):
"""
Sets the state of the bootstrap's random number generator
Parameters
----------
state : RandomState state vector
Array containing the state
"""
return self.random_state.set_state(state)
def seed(self, value):
"""
Seeds the bootstrap's random number generator
Parameters
----------
value : int
Integer to use as the seed
"""
self._seed = value
self.random_state.seed(value)
return None
def reset(self, use_seed=True):
"""
Resets the bootstrap to either its initial state or the last seed.
Parameters
----------
use_seed : bool, optional
Flag indicating whether to use the last seed if provided. If
False or if no seed has been set, the bootstrap will be reset
to the initial state. Default is True
"""
self._index = np.arange(self._num_items)
self._resample()
self.random_state.set_state(self._initial_state)
if use_seed and self._seed is not None:
self.seed(self._seed)
return None
def bootstrap(self, reps):
"""
Iterator for use when bootstrapping
Parameters
----------
reps : int
Number of bootstrap replications
Example
-------
The key steps are problem dependent and so this example shows the use
as an iterator that does not produce any output
>>> from arch.bootstrap import IIDBootstrap
>>> import numpy as np
>>> bs = IIDBootstrap(np.arange(100), x=np.random.randn(100))
>>> for posdata, kwdata in bs.bootstrap(1000):
... # Do something with the positional data and/or keyword data
... pass
.. note::
Note this is a generic example and so the class used should be the
name of the required bootstrap
Notes
-----
The iterator returns a tuple containing the data entered in positional
arguments as a tuple and the data entered using keywords as a
dictionary
"""
for _ in range(reps):
indices = np.asarray(self.update_indices())
self._index = indices
yield self._resample()
def conf_int(self,
func,
reps=1000,
method='basic',
size=0.95,
tail='two',
extra_kwargs=None,
reuse=False,
sampling='nonparametric',
std_err_func=None,
studentize_reps=1000):
"""
Parameters
----------
func : callable
Function the computes parameter values. See Notes for requirements
reps : int, optional
Number of bootstrap replications
method : string, optional
One of 'basic', 'percentile', 'studentized', 'norm' (identical to
'var', 'cov'), 'bc' (identical to 'debiased', 'bias-corrected'), or
'bca'
size : float, optional
Coverage of confidence interval
tail : string, optional
One of 'two', 'upper' or 'lower'.
reuse : bool, optional
Flag indicating whether to reuse previously computed bootstrap
results. This allows alternative methods to be compared without
rerunning the bootstrap simulation. Reuse is ignored if reps is
not the same across multiple runs, func changes across calls, or
method is 'studentized'.
sampling : string, optional
Type of sampling to use: 'nonparametric', 'semi-parametric' (or
'semi') or 'parametric'. The default is 'nonparametric'. See
notes about the changes to func required when using 'semi' or
'parametric'.
extra_kwargs : dict, optional
Extra keyword arguments to use when calling func and std_err_func,
when appropriate
std_err_func : callable, optional
Function to use when standardizing estimated parameters when using
the studentized bootstrap. Providing an analytical function
eliminates the need for a nested bootstrap
studentize_reps : int, optional
Number of bootstraps to use in the innter component when using the
studentized bootstrap. Ignored when ``std_err_func`` is provided
Returns
-------
intervals : 2-d array
Computed confidence interval. Row 0 contains the lower bounds, and
row 1 contains the upper bounds. Each column corresponds to a
parameter. When tail is 'lower', all upper bounds are inf.
Similarly, 'upper' sets all lower bounds to -inf.
Examples
--------
>>> import numpy as np
>>> def func(x):
... return x.mean(0)
>>> y = np.random.randn(1000, 2)
>>> from arch.bootstrap import IIDBootstrap
>>> bs = IIDBootstrap(y)
>>> ci = bs.conf_int(func, 1000)
Notes
-----
When there are no extra keyword arguments, the function is called
.. code:: python
func(*args, **kwargs)
where args and kwargs are the bootstrap version of the data provided
when setting up the bootstrap. When extra keyword arguments are used,
these are appended to kwargs before calling func.
The standard error function, if provided, must return a vector of
parameter standard errors and is called
.. code:: python
std_err_func(params, *args, **kwargs)
where ``params`` is the vector of estimated parameters using the same
bootstrap data as in args and kwargs.
The bootstraps are:
* 'basic' - Basic confidence using the estimated parameter and
difference between the estimated parameter and the bootstrap
parameters
* 'percentile' - Direct use of bootstrap percentiles
* 'norm' - Makes use of normal approximation and bootstrap covariance
estimator
* 'studentized' - Uses either a standard error function or a nested
bootstrap to estimate percentiles and the bootstrap covariance for
scale
* 'bc' - Bias corrected using estimate bootstrap bias correction
* 'bca' - Bias corrected and accelerated, adding acceleration parameter
to 'bc' method
"""
studentized = 'studentized'
if not 0.0 < size < 1.0:
raise ValueError('size must be strictly between 0 and 1')
tail = tail.lower()
if tail not in ('two', 'lower', 'upper'):
raise ValueError('tail must be one of two-sided, lower or upper')
studentize_reps = studentize_reps if method == studentized else 0
_reuse = False
if reuse:
# check conditions for reuse
_reuse = (self._results is not None and len(self._results) == reps
and method != studentized and self._last_func is func)
if not _reuse:
if reuse:
import warnings
warn = 'The conditions to reuse the previous bootstrap has ' \
'not been satisfied. A new bootstrap will be used.'
warnings.warn(warn, RuntimeWarning)
self._construct_bootstrap_estimates(
func,
reps,
extra_kwargs,
std_err_func=std_err_func,
studentize_reps=studentize_reps, # noqa
sampling=sampling)
base, results = self._base, self._results
studentized_results = self._studentized_results
std_err = []
if method in ('norm', 'var', 'cov', studentized):
errors = results - results.mean(axis=0)
std_err = np.sqrt(np.diag(errors.T.dot(errors) / reps))
if tail == 'two':
alpha = (1.0 - size) / 2
else:
alpha = (1.0 - size)
percentiles = [alpha, 1.0 - alpha]
norm_quantiles = stats.norm.ppf(percentiles)
if method in ('norm', 'var', 'cov'):
lower = base + norm_quantiles[0] * std_err
upper = base + norm_quantiles[1] * std_err
elif method in ('percentile', 'basic', studentized, 'debiased', 'bc',
'bias-corrected', 'bca'):
values = results
if method == studentized:
# studentized uses studentized parameter estimates
values = studentized_results
if method in ('debiased', 'bc', 'bias-corrected', 'bca'):
# bias corrected uses modified percentiles, but is
# otherwise identical to the percentile method
p = (results < base).mean(axis=0)
b = stats.norm.ppf(p)
b = b[:, None]
if method == 'bca':
nobs = self._num_items
jk_params = _loo_jackknife(func, nobs, self._args,
self._kwargs)
u = (nobs - 1) * (jk_params - base)
numer = np.sum(u**3, 0)
denom = 6 * (np.sum(u**2, 0)**(3.0 / 2.0))
small = denom < (np.abs(numer) * np.finfo(np.float64).eps)
if small.any():
message = 'Jackknife variance estimate {jk_var} is ' \
'too small to use BCa'
raise RuntimeError(message.format(jk_var=denom))
a = numer / denom
a = a[:, None]
else:
a = 0.0
percentiles = stats.norm.cdf(b + (b + norm_quantiles) /
(1.0 - a * (b + norm_quantiles)))
percentiles = list(100 * percentiles)
else:
percentiles = [100 * p for p in percentiles] # Rescale
if method not in ('bc', 'debiased', 'bias-corrected', 'bca'):
ci = np.asarray(np.percentile(values, percentiles, axis=0))
lower = ci[0, :]
upper = ci[1, :]
else:
k = values.shape[1]
lower = np.zeros(k)
upper = np.zeros(k)
for i in range(k):
lower[i], upper[i] = np.percentile(values[:, i],
list(percentiles[i]))
# Basic and studentized use the lower empirical quantile to
# compute upper and vice versa. Bias corrected and percentile use
# upper to estimate the upper, and lower to estimate the lower
if method == 'basic':
lower_copy = lower + 0.0
lower = 2.0 * base - upper
upper = 2.0 * base - lower_copy
elif method == studentized:
lower_copy = lower + 0.0
lower = base - upper * std_err
upper = base - lower_copy * std_err
else:
raise ValueError('Unknown method')
if tail == 'lower':
upper = np.zeros_like(base)
upper.fill(np.inf)
elif tail == 'upper':
lower = np.zeros_like(base)
lower.fill(-1 * np.inf)
return np.vstack((lower, upper))
def clone(self, *args, **kwargs):
"""
Clones the bootstrap using different data.
Parameters
----------
args
Positional arguments to bootstrap
kwargs
Keyword arguments to bootstrap
Returns
-------
bs
Bootstrap instance
"""
pos_arguments = copy.deepcopy(self._parameters)
pos_arguments.extend(args)
bs = self.__class__(*pos_arguments, **kwargs)
if self._seed is not None:
bs.seed(self._seed)
return bs
def apply(self, func, reps=1000, extra_kwargs=None):
"""
Applies a function to bootstrap replicated data
Parameters
----------
func : callable
Function the computes parameter values. See Notes for requirements
reps : int, optional
Number of bootstrap replications
extra_kwargs : dict, optional
Extra keyword arguments to use when calling func. Must not
conflict with keyword arguments used to initialize bootstrap
Returns
-------
results : array
reps by nparam array of computed function values where each row
corresponds to a bootstrap iteration
Notes
-----
When there are no extra keyword arguments, the function is called
.. code:: python
func(params, *args, **kwargs)
where args and kwargs are the bootstrap version of the data provided
when setting up the bootstrap. When extra keyword arguments are used,
these are appended to kwargs before calling func
Examples
--------
>>> import numpy as np
>>> x = np.random.randn(1000,2)
>>> from arch.bootstrap import IIDBootstrap
>>> bs = IIDBootstrap(x)
>>> def func(y):
... return y.mean(0)
>>> results = bs.apply(func, 100)
"""
kwargs = _add_extra_kwargs(self._kwargs, extra_kwargs)
base = func(*self._args, **kwargs)
try:
num_params = base.shape[0]
except:
num_params = 1
results = np.zeros((reps, num_params))
count = 0
for pos_data, kw_data in self.bootstrap(reps):
kwargs = _add_extra_kwargs(kw_data, extra_kwargs)
results[count] = func(*pos_data, **kwargs)
count += 1
return results
def _construct_bootstrap_estimates(self,
func,
reps,
extra_kwargs=None,
std_err_func=None,
studentize_reps=0,
sampling='nonparametric'):
# Private, more complicated version of apply
self._last_func = func
semi = parametric = False
if sampling == 'parametric':
parametric = True
elif sampling == 'semiparametric':
semi = True
if extra_kwargs is not None:
if any(k in self._kwargs for k in extra_kwargs):
raise ValueError('extra_kwargs contains keys used for variable'
' names in the bootstrap')
kwargs = _add_extra_kwargs(self._kwargs, extra_kwargs)
base = func(*self._args, **kwargs)
num_params = 1 if np.isscalar(base) else base.shape[0]
results = np.zeros((reps, num_params))
studentized_results = np.zeros((reps, num_params))
count = 0
for pos_data, kw_data in self.bootstrap(reps):
kwargs = _add_extra_kwargs(kw_data, extra_kwargs)
if parametric:
kwargs['state'] = self.random_state
kwargs['params'] = base
elif semi:
kwargs['params'] = base
results[count] = func(*pos_data, **kwargs)
if std_err_func is not None:
std_err = std_err_func(results[count], *pos_data, **kwargs)
studentized_results[count] = (results[count] - base) / std_err
elif studentize_reps > 0:
# Need new bootstrap of same type
nested_bs = self.clone(*pos_data, **kw_data)
# Set the seed to ensure reproducability
seed = self.random_state.randint(2**31 - 1)
nested_bs.seed(seed)
cov = nested_bs.cov(func,
studentize_reps,
extra_kwargs=extra_kwargs)
std_err = np.sqrt(np.diag(cov))
studentized_results[count] = (results[count] - base) / std_err
count += 1
self._base = np.asarray(base)
self._results = np.asarray(results)
self._studentized_results = np.asarray(studentized_results)
def cov(self, func, reps=1000, recenter=True, extra_kwargs=None):
"""
Compute parameter covariance using bootstrap
Parameters
----------
func : callable
Callable function that returns the statistic of interest as a
1-d array
reps : int, optional
Number of bootstrap replications
recenter : bool, optional
Whether to center the bootstrap variance estimator on the average
of the bootstrap samples (True) or to center on the original sample
estimate (False). Default is True.
extra_kwargs: dict, optional
Dictionary of extra keyword arguments to pass to func
Returns
-------
cov: array
Bootstrap covariance estimator
Notes
-----
func must have the signature
.. code:: python
func(params, *args, **kwargs)
where params are a 1-dimensional array, and `*args` and `**kwargs` are
data used in the the bootstrap. The first argument, params, will be
none when called using the original data, and will contain the estimate
computed using the original data in bootstrap replications. This
parameter is passed to allow parametric bootstrap simulation.
Example
-------
Bootstrap covariance of the mean
>>> from arch.bootstrap import IIDBootstrap
>>> import numpy as np
>>> def func(x):
... return x.mean(axis=0)
>>> y = np.random.randn(1000, 3)
>>> bs = IIDBootstrap(y)
>>> cov = bs.cov(func, 1000)
Bootstrap covariance using a function that takes additional input
>>> def func(x, stat='mean'):
... if stat=='mean':
... return x.mean(axis=0)
... elif stat=='var':
... return x.var(axis=0)
>>> cov = bs.cov(func, 1000, extra_kwargs={'stat':'var'})
.. note::
Note this is a generic example and so the class used should be the
name of the required bootstrap
"""
self._construct_bootstrap_estimates(func, reps, extra_kwargs)
base, results = self._base, self._results
if recenter:
errors = results - np.mean(results, 0)
else:
errors = results - base
return errors.T.dot(errors) / reps
def var(self, func, reps=1000, recenter=True, extra_kwargs=None):
"""
Compute parameter variance using bootstrap
Parameters
----------
func : callable
Callable function that returns the statistic of interest as a
1-d array
reps : int, optional
Number of bootstrap replications
recenter : bool, optional
Whether to center the bootstrap variance estimator on the average
of the bootstrap samples (True) or to center on the original sample
estimate (False). Default is True.
extra_kwargs: dict, optional
Dictionary of extra keyword arguments to pass to func
Returns
-------
var : 1-d array
Bootstrap variance estimator
Notes
-----
func must have the signature
.. code:: python
func(params, *args, **kwargs)
where params are a 1-dimensional array, and `*args` and `**kwargs` are
data used in the the bootstrap. The first argument, params, will be
none when called using the original data, and will contain the estimate
computed using the original data in bootstrap replications. This
parameter is passed to allow parametric bootstrap simulation.
Example
-------
Bootstrap covariance of the mean
>>> from arch.bootstrap import IIDBootstrap
>>> import numpy as np
>>> def func(x):
... return x.mean(axis=0)
>>> y = np.random.randn(1000, 3)
>>> bs = IIDBootstrap(y)
>>> variances = bs.var(func, 1000)
Bootstrap covariance using a function that takes additional input
>>> def func(x, stat='mean'):
... if stat=='mean':
... return x.mean(axis=0)
... elif stat=='var':
... return x.var(axis=0)
>>> variances = bs.var(func, 1000, extra_kwargs={'stat': 'var'})
.. note::
Note this is a generic example and so the class used should be the
name of the required bootstrap
"""
self._construct_bootstrap_estimates(func, reps, extra_kwargs)
base, results = self._base, self._results
if recenter:
errors = results - np.mean(results, 0)
else:
errors = results - base
return (errors**2).sum(0) / reps
def update_indices(self):
"""
Update indices for the next iteration of the bootstrap. This must
be overridden when creating new bootstraps.
"""
return self.random_state.randint(self._num_items, size=self._num_items)
def _resample(self):
"""
Resample all data using the values in _index
"""
indices = self._index
pos_data = []
for values in self._args:
if isinstance(values, (pd.Series, pd.DataFrame)):
pos_data.append(values.iloc[indices])
else:
pos_data.append(values[indices])
named_data = {}
for key, values in iteritems(self._kwargs):
if isinstance(values, (pd.Series, pd.DataFrame)):
named_data[key] = values.iloc[indices]
else:
named_data[key] = values[indices]
setattr(self, key, named_data[key])
self.pos_data = pos_data
self.kw_data = named_data
self.data = (pos_data, named_data)
return self.data
class Evolution:
# pylint: disable=too-many-instance-attributes
# noinspection PyUnresolvedReferences
"""
Class that executes genetic search.
:param num_populations: (int) number of populations (default 1)
:param population_size: (int) size of the population
:param genotype_size: (int) size of the genotype vector
:param evaluation_function: (func) function to evaluate genotype performance.
It should take as inputthe entire population genotype (matrix) and return
an array with the performances
:param fitness_normalization_mode: (str) method to normalize fitness values
(fitness-proportionate, rank-based or sigma scaling)
:param selection_mode: (str) method to select parents for reproduction (RWS or SUS)
:param reproduce_from_elite: (bool) whether the reproduction comes from elite
or remaining agents in the population
:param reproduction_mode: (str) method to reproduce genetic algorithm or hill climbing
:param mutation_variance: (float) variance of gaussian mutation rate
:param folder_path: (string) path of the folder where to save the checkpoints
:param search_constraint: (list of bool) flag whether to clip a specific site in
a genotype (default to all True)
:param reevaluate: (bool) whether to re-evaluate the individual if it's retained
in the new generation (used only in hill-climbing)
:param max_generation: (int) maximum generations to evolve (not used if
termination_function is provided)
:param termination_function: (func) function to check if search should terminate
(it accept the Evolution instance, default to None)
:param elitist_fraction: (float) proportion of new population that will be made of
best unmodified parents (only relevant for genetic algorithm)
:param mating_fraction: (float) proportion of population that will be made of children
(in Beer this is equal to 1. - elitist_fraction) (only relevant for genetic algorithm)
:param crossover_probability: (float) probability that crossover will occur
(only relevant for genetic algorithm)
:param crossover_mode: (str) the way to perform crossover (UNIFORM, 1-POINT, 2-POINT, ...)
(only relevant for genetic algorithm)
:param crossover_points: (list of int) a list that specifies the indices of where
to cut during crossover (only relevant for genetic algorithm)
:param checkpoint_interval: (int) every how many generations should the population
be saved and results logged
:param max_expected_offspring: (float) number of offspring to be allocated to the
best individual, best between 1 and 2
"""
population_size: int
genotype_size: int
evaluation_function: Callable
num_populations: int = 1
performance_objective: Union[str,float] = 'MAX' # 'MIN', 'ABS_MAX', float value
fitness_normalization_mode: str = 'FPS' # 'NONE', 'FPS', 'RANK', 'SIGMA'
selection_mode: str = 'RWS' # 'UNIFORM', 'RWS', 'SUS'
reproduce_from_elite: bool = False
reproduction_mode: str = 'GENETIC_ALGORITHM' # 'HILL_CLIMBING', 'GENETIC_ALGORITHM'
mutation_variance: float = DEFAULT_MUTATION_VARIANCE
max_generation: int = 100
termination_function: Callable = None
checkpoint_interval: int = DEFAULT_CHECKPOINT_INTERVAL
crossover_probability: float = DEFAULT_CROSSOVER_PROB
crossover_points: List[int] = None
folder_path: str = None
elitist_fraction: float = None
mating_fraction: float = None
n_elite: int = None
n_mating: int = None
n_fillup: int = None
crossover_mode: str = 'UNIFORM'
search_constraint: np.ndarray = None # this will be converted to all True by default in __post_init__
reevaluate: bool = True # only used in hill-climbing
max_expected_offspring: float = DEFAULT_MAX_EXPECTED_OFFSPRING
random_seed: int = 0
random_state: RandomState = None
pop_eval_random_seed: int = None # initialized at every generation
# other field (no need to define them outside)
generation: int = 0 # the current generation number
population: np.ndarray = None # the list of population genotypes (sorted by performance)
population_unsorted: np.ndarray = None # the list of population genotypes (before sorting)
# (will be initialized in __post_init__)
performances: np.ndarray = None # performances of the genotypes
fitnesses: np.ndarray = None # fitnesses of the genotypes
population_sorted_indexes: np.ndarray = None
# keep track of indexes in sorted population
# population_sorted_indexes[0] is the index of the agent with best performance
# in the unsorted population
# collect average, best and worst performances across generations
avg_performances: List[List[float]] = field(default_factory=list)
best_performances: List[List[float]] = field(default_factory=list)
worst_performances: List[List[float]] = field(default_factory=list)
timeit: bool = False
def __post_init__(self):
assert self.num_populations > 0, "Number of populations should be greater than zero"
assert self.population_size % 4 == 0, "Population size must be divisible by 4"
# otherwise n_elite + n_mating may be greater than population_size
self.sqrt_mutation_variance = np.sqrt(self.mutation_variance)
if self.random_state is None:
self.random_state = RandomState(self.random_seed)
self.loaded_from_file = all(
x is not None for x in
[self.population, self.performances, self.fitnesses]
)
# create initial population if not provided
if self.population is None:
# create a set of random genotypes
self.population = self.random_state.uniform(
MIN_SEARCH_VALUE, MAX_SEARCH_VALUE,
[self.num_populations, self.population_size, self.genotype_size]
)
if self.search_constraint is None:
self.search_constraint = np.array([True] * self.genotype_size)
self.file_num_zfill = int(np.ceil(np.log10(self.max_generation + 1))) \
if self.max_generation \
else 1 if self.max_generation == 0 \
else FILE_NUM_ZFILL_DEFAULT
# conver performance_objective to float if it is a string with a number
f = utils.get_float(self.performance_objective)
if f is not None:
self.performance_objective = f
self.timing = Timing(self.timeit)
self.validate_params()
self.init_reproduction_parameters()
@staticmethod
def get_random_genotype(rando_state, gen_size):
return rando_state.uniform(MIN_SEARCH_VALUE, MAX_SEARCH_VALUE, gen_size)
def init_reproduction_parameters(self):
# self.n_mating: number of new agents return by select_mating_pool()
if self.reproduction_mode == 'GENETIC_ALGORITHM':
# self.n_elite: number of best agents to preserve (only used in genetic algorithm)
# self.n_fillup: agents to be randomly generated
self.n_elite = int(
np.floor(self.population_size * self.elitist_fraction + 0.5) # at least one
) # children from elite group
self.n_mating = int(np.floor(
self.population_size * self.mating_fraction + 0.5 # at least one
)) # children from mating population
self.n_fillup = self.population_size - (self.n_elite + self.n_mating) # children from random fillup
assert all(x >= 0 for x in [self.n_elite, self.n_mating, self.n_fillup])
assert self.n_elite + self.n_mating + self.n_fillup == self.population_size
else: # 'HILL_CLIMBING'
self.n_mating = self.population_size
def validate_params(self):
# termination condition
assert self.max_generation is None or self.termination_function is None, \
"Either max_generation or termination_function must be defined"
# folder path
if self.folder_path:
assert os.path.isdir(self.folder_path), "folder_path '{}' is not a valid directory".format(self.folder_path)
# search_constraint
assert len(self.search_constraint) == self.genotype_size, \
"The length of search_constraint should be equal to genotype_size"
# performance_objective
accepted_values = ['MAX', 'MIN', 'ABS_MAX']
assert type(self.performance_objective) in [float,int] or \
self.performance_objective in accepted_values, \
'performance_objective should be either {}'.format(', '.join(accepted_values))
# fitness_normalization_mode
accepted_values = ['NONE', 'FPS', 'RANK', 'SIGMA']
assert self.fitness_normalization_mode in accepted_values, \
'fitness_normalization_mode should be either {}'.format(', '.join(accepted_values))
assert self.fitness_normalization_mode!='NONE' or self.selection_mode == 'UNIFORM', \
"if fitness_normalization_mode is 'NONE' (copy of PERFORMANCE), selection_mode must be UNIFORM (not normalized)"
# selection_mode
accepted_values = ['UNIFORM', 'RWS', 'SUS']
assert self.selection_mode in accepted_values, \
'selection_mode should be either {}'.format(', '.join(accepted_values))
# reproduce_from_elite
assert not self.reproduce_from_elite or self.selection_mode == 'UNIFORM', \
'if reproducing from elite, selection mode must be uniform'
# reproduction_mode
accepted_values = ['HILL_CLIMBING', 'GENETIC_ALGORITHM']
assert self.reproduction_mode in accepted_values, \
'reproduction_mode should be either {}'.format(', '.join(accepted_values))
# GENETIC_ALGORITHM
if self.reproduction_mode == 'GENETIC_ALGORITHM':
assert 0 <= self.elitist_fraction <= 1, \
'In GENETIC_ALGORITHM: 0 <= elitist_fraction <=1'
assert 0 <= self.mating_fraction <= 1, \
'In GENETIC_ALGORITHM: 0 <= mating_fraction <=1'
assert 0 <= self.crossover_probability <= 1, \
'In GENETIC_ALGORITHM: 0 <= crossover_probability <=1'
assert re.match('UNIFORM|\d+-POINT', self.crossover_mode), \
'In GENETIC_ALGORITHM: crossover_mode should be UNIFORM or x-POINT'
# crossover
assert self.crossover_mode != None, "crossover_mode cannot be None"
if self.crossover_mode == 'UNIFORM':
# crossover is computed on the entire genotype
# with prob 0.5 of flipping each genotype site
assert self.crossover_points is None, \
"In uniform crossover_mode you shouldn't specify the crossover_points"
elif self.crossover_mode.endswith('-POINT'):
# A. if crossover_points is None the points are randomly generated
# crossover_points must be a list of max x-1 integers in the interval [1,G-1]
# where x is the integer > 0 specified in the parameter crossover_mode ('x-POINT')
# and G is the size of the genotype
# e.g. if parent1=[0,0,0] and parent2=[1,1,1] (G=3),
# crossover_points must contain a single integer which can be
# 1: child1=[0,1,1] child2=[1,0,0]
# 2: child1=[0,0,1] child2=[1,1,0]
# B. if crossover_points is not None -> num_points <= len(self.crossover_points)
# if num_points < len(self.crossover_points)
# only num_points will be randomly selected from the self.crossover_points
num_points = self.crossover_mode[:-6]
assert utils.is_int(num_points), \
"Param crossover_mode should be 'UNIFORM' or 'x-POINT' (with x being an integer > 0)"
num_points = int(num_points)
assert 0 < num_points < self.genotype_size, \
"Param crossover_mode should be 'x-POINT', with x being an integer such that 0 < x < G " \
"and where G is the size of the genotype"
assert num_points <= self.genotype_size - 1, \
"Too high value for {} in param crossover_mode. Max should be G-1 " \
"(where G is the size of the genotype)".format(
self.crossover_mode)
if self.crossover_points is not None:
assert len(set(self.crossover_points)) == len(self.crossover_points), \
"Duplicated values in crossover_points"
self.crossover_points = sorted(set(self.crossover_points))
assert num_points <= len(self.crossover_points), \
"crossover_mode={} and crossover_points={} but {} must be <= {}=len(crossover_points)".format(
self.crossover_mode, self.crossover_points, num_points, len(self.crossover_points))
assert all(1 < x < self.genotype_size for x in self.crossover_points), \
"Some of the values in crossover_points are not in the interval [1,G-1] " \
"where G is the size of the genotype"
else:
assert False, \
"Param crossover_mode should be 'UNIFORM' or 'x-POINT' (with x being an integer > 0)"
def set_folder_name(self, text):
self.folder_path = text
def run(self):
"""
Execute a full search run until some condition is reached.
:return: the last population in the search
"""
if self.loaded_from_file:
# comple cycle from previous run (after saving)
self.save_to_file()
self.reproduce()
self.generation += 1
t = self.timing.init_tictoc()
while self.generation <= self.max_generation:
# evaluate all genotypes on the task
self.pop_eval_random_seed = utils.random_int(self.random_state)
# suffle populations before running evaluation function
for pop in self.population:
self.random_state.shuffle(pop)
# run evaluation function
self.performances = self.evaluation_function(
self.population, self.pop_eval_random_seed
)
if type(self.performances) is list:
self.performances = np.array(self.performances)
if self.num_populations==1 and self.performances.ndim != 2:
# eval function returned a simple array of perfomances
# because there is only one population
self.performances = np.expand_dims(self.performances,0) # add an additional index (population)
expected_perf_shape = self.population.shape[:-1]
assert self.performances.shape == expected_perf_shape, \
"Evaluation function didn't return performances with shape {}".format(expected_perf_shape)
assert (self.performances >=0).all(), \
"Performance must be non-negative"
self.timing.add_time('EVO1-RUN_eval_function', t)
# sorting population and performances on performances
self.sort_population_on_performance()
self.timing.add_time('EVO1-RUN_sort_population', t)
# update average/best/worst population performance
avg = np.mean(self.performances, axis=1).tolist()
best = self.performances[:,0].tolist()
worst = self.performances[:,-1].tolist()
variance = np.var(self.performances, axis=1).tolist()
self.avg_performances.append(avg)
self.best_performances.append(best)
self.worst_performances.append(worst)
self.timing.add_time('EVO1-RUN_stats', t)
print_stats = lambda a : '|'.join(['{:.5f}'.format(x) for x in a])
# print short statistics
print("Generation {}: Best: {}, Worst: {}, Average: {}, Variance: {}".format(
str(self.generation).rjust(self.file_num_zfill), print_stats(best),
print_stats(worst), print_stats(avg), print_stats(variance)))
self.timing.add_time('EVO1-RUN_print_stats', t)
# check if to terminate
if self.generation == self.max_generation or \
(self.termination_function and self.termination_function(self)):
self.save_to_file()
# Stop search due to termination condition
break
# save the intermediate evolution state
if self.checkpoint_interval and self.generation % self.checkpoint_interval == 0:
# save current generation
self.save_to_file()
self.timing.add_time('EVO1-RUN_savefile', t)
# Compute fitnesses (based on performances) - used in reproduce
self.update_fitnesses()
self.timing.add_time('EVO1-RUN_update_fitness', t)
# run reproduce (update fitnesses and run genetic or hill-climbing)
self.reproduce()
self.timing.add_time('EVO1-RUN_reproduce', t)
# update generation
self.generation += 1
def sort_population_on_performance(self):
# performances must be non-negative (>=0)
if type(self.performance_objective) is str:
if self.performance_objective == 'MAX':
performances_objectified = self.performances
elif self.performance_objective == 'MIN':
performances_objectified = - self.performances
else:
assert self.performance_objective == 'ABS_MAX'
performances_objectified = np.abs(self.performances)
else:
# minimizing the distance between performance and perf objective
# when self.performance_objective==0 this would be identical to 'ABS_MIN'
performances_objectified = - np.abs(self.performances - self.performance_objective)
# sort genotypes, performances by performance_objectified from hight to low
self.population_sorted_indexes = np.argsort(-performances_objectified, axis=-1)
self.performances = np.take_along_axis(self.performances, self.population_sorted_indexes, axis=-1)
self.population_unsorted = self.population # keep track of the original population to ensure reproducibility
sorted_indexes_exp = np.expand_dims(self.population_sorted_indexes, -1) # add one dimension at the end to sort population
self.population = np.take_along_axis(self.population_unsorted, sorted_indexes_exp, axis=1)
# OLD METHOD WITHOUT NUMPY:
# sort genotypes and performances by performance from best to worst
# self.population, self.performances = \
# zip(*sorted(zip(self.population, self.performances),
# key=lambda pair: pair[1], reverse=True))
# self.population = np.array(self.population)
# self.performances = np.array(self.performances)
def reproduce(self):
"""Run reproduce via HILL_CLIMBING or GENETIC_ALGORITHM"""
if self.reproduction_mode == 'GENETIC_ALGORITHM':
self.reproduce_genetic_algorithm()
else:
self.reproduce_hill_climbing()
def reproduce_genetic_algorithm(self):
"""
Reproduce a single generation in the following way:
1) Copy the proportion equal to elitist_fraction of the current population to the new population
(these are best_genotypes)
2) Select part of the population for crossover using some selection method (set in config)
3) Shuffle the selected population in preparation for cross-over
4) Create crossover_fraction children of selected population with probability of crossover equal
to prob_crossover.
Crossover takes place at genome module boundaries (single neurons).
5) Apply mutation to the children with mutation equal to mutation_var
6) Fill the rest of the population with randomly created genotypes
self.population and self.performances are sorted based on performances
"""
t = self.timing.init_tictoc()
new_population = np.zeros(
[self.num_populations, self.population_size, self.genotype_size]
)
# 1) Elitist selection
# same elite size in all populations
self.elite_population = self.population[:, :self.n_elite]
new_population[:, :self.n_elite] = self.elite_population
self.timing.add_time('EVO2-GA_1_elitist_selection', t)
# 2) Select mating population from the remaining population
mating_pool = self.select_mating_pool()
self.timing.add_time('EVO2-GA_2_mating_pool', t)
# 3) Shuffle mating pool
for pop_mating_pool in mating_pool:
self.random_state.shuffle(pop_mating_pool)
self.timing.add_time('EVO2-GA_3_shuffle', t)
# 4) Create children with crossover or apply mutation
mating_finish = self.n_elite + self.n_mating
newpop_counter = None # track where we are in the new population
for p in range(self.num_populations):
mating_counter = 0
newpop_counter = self.n_elite # track where we are in the new population
while newpop_counter < mating_finish:
not_last = mating_finish - newpop_counter > 1
parent1 = mating_pool[p][mating_counter]
if not_last and self.random_state.random() < self.crossover_probability:
parent2 = mating_pool[p][mating_counter + 1]
child1, child2 = self.crossover(parent1, parent2)
# if the child is the same as the first parent after crossover, mutate it (as in Beer)
if np.array_equal(child1, parent1):
child1 = self.mutate(parent1)
new_population[p][newpop_counter] = child1
new_population[p][newpop_counter + 1] = child2
newpop_counter += 2
mating_counter += 2
else:
# if no crossover, mutate just one genotype
child1 = self.mutate(parent1)
new_population[p][newpop_counter] = child1
newpop_counter += 1
mating_counter += 1
self.timing.add_time('EVO2-GA_4_children', t)
# 5) Fill up with random new genotypes
new_population[:, newpop_counter:] = self.random_state.uniform(
MIN_SEARCH_VALUE, MAX_SEARCH_VALUE,
size=[self.num_populations, self.n_fillup, self.genotype_size]
)
self.timing.add_time('EVO2-GA_5_fillup', t)
# 6) redefined population based on the newly computed population
self.population = new_population
self.timing.add_time('EVO2-GA_6_convert_pop', t)
def reproduce_hill_climbing(self):
t = self.timing.init_tictoc()
# 1) Select the parents using sampling (replacing the entire population, no elite here)
parent_population = self.select_mating_pool()
self.timing.add_time('EVO2-HC_1_mating pool', t)
# 2) Reevaluate
if self.reevaluate:
parent_performance = np.array(self.evaluation_function(parent_population, self.pop_eval_random_seed))
else:
assert False, \
"reevaluate params has to be True. " \
"For reevaluate to be False we need to also return performances in function select_mating_pool"
self.timing.add_time('EVO2-HC_2_reevaluate', t)
# 3) Produce the new population by mutating each parent and rewrite it on the current population
self.population = np.array([self.mutate(gen) for gen in parent_population])
self.timing.add_time('EVO2-HC_3_mutate', t)
# 4) Calculate new performances
self.performance = np.array(self.evaluation_function(self.population, self.pop_eval_random_seed))
self.timing.add_time('EVO2-HC_4_compute_perf', t)
# 5) Check if performace worsened and in this case retrieve agent from parent population
lower_performance = self.performance < parent_performance # bool array
for i in range(self.population_size):
if lower_performance[i]:
self.population[i] = parent_population[i]
self.performance[i] = parent_performance[i]
self.timing.add_time('EVO2-HC_5_compare_and_select', t)
def update_fitnesses(self):
"""
Update genotype fitness to relative values, retain sorting from best to worst.
"""
if self.fitness_normalization_mode == 'NONE':
# do not use fitness in selection
self.fitnesses = None
elif self.fitness_normalization_mode == 'FPS': # (fitness-proportionate)
self.fitnesses = np.zeros(self.performances.shape) # same shape as performances
for p in range(self.num_populations):
avg_perf = self.avg_performances[-1][p]
m = utils.linear_scaling(
self.worst_performances[-1][p],
self.best_performances[-1][p],
avg_perf,
self.max_expected_offspring
)
scaled_performances = m * (self.performances[p] - avg_perf) + avg_perf
total_performance = np.sum(scaled_performances)
self.fitnesses[p] = scaled_performances / total_performance
elif self.fitness_normalization_mode == 'RANK': # (rank-based)
# Baker's linear ranking method: f(pos) = 2-SP+2*(SP-1)*(pos-1)/(n-1)
# the highest ranked individual receives max_exp_offspring (typically 1.1),
# the lowest receives 2 - max_exp_offspring
# normalized to sum to 1
self.fitnesses = np.zeros(self.performances.shape) # same shape as performances
for p in range(self.num_populations):
self.fitnesses[p] = np.array(
[
(
self.max_expected_offspring + (2 - 2 * self.max_expected_offspring) * i /
(self.population_size - 1)
) / self.population_size
for i in range(self.population_size)
]
)
elif self.fitness_normalization_mode == 'SIGMA': # (sigma-scaling)
# for every individual 1 + (I(f) - P(avg_f))/2*P(std) is calculated
# if value is below zero, a small positive constant is given so the individual has some probability
# of being chosen. The numbers are then normalized
self.fitnesses = np.zeros(self.performances.shape) # same shape as performances
for p in range(self.num_populations):
pop_perf = self.performances[p]
avg = np.mean(pop_perf)
std = max(0.0001, np.std(pop_perf))
exp_values = list((1 + ((f - avg) / (2 * std))) for f in pop_perf)
for i, v in enumerate(exp_values):
if v <= 0:
exp_values[i] = 1 / self.population_size
s = sum(exp_values)
self.fitnesses[p] = np.array(list(e / s for e in exp_values))
def select_mating_pool(self):
"""
Select a mating pool population.
:return: selected parents for reproduction
"""
if self.selection_mode == 'UNIFORM':
# create mating_pool from source_population uniformally
# (from beginning to end and if needed restart from beginning)
source_population = \
self.elite_population if self.reproduce_from_elite \
else self.population
num_source_pop = source_population.shape[1] # number of elements in source pop
assert num_source_pop>0, \
"Error, can't create a mating pool from empty source population"
cycle_source_pop_indexes = np.resize( # this return a column vector
np.resize( # [0,1,...,n, 0, 1, ..., n]
np.arange(num_source_pop), # where n is num_source_pop and the size
[self.n_mating,1] # and n_mating the actual size of the list
),
[self.num_populations, self.n_mating, 1] # this duplicates the indexes for all populations
) # to obtain same 3 dimensions of source_population
# rotate thtough the source_population(s)
mating_pool = np.take_along_axis(source_population, cycle_source_pop_indexes, 1)
else:
min_fitness = np.min(self.fitnesses, axis=-1)
assert (min_fitness > - ROUNDING_TOLERANCE).all(), \
"Found neg fitness: {}".format(min_fitness)
if (self.fitnesses < 0).any():
# setting small neg values due to rounding errors to zeros
self.fitnesses[self.fitnesses<0] = 0
cum_probs = np.cumsum(self.fitnesses, axis=-1)
cum_probs_error = np.abs(cum_probs[:,-1] - 1.0)
assert (cum_probs_error >=0).all() and (cum_probs_error < CUM_PROB_TOLERANCE).all(), \
"Too big cum_probs_error: {}".format(cum_probs_error)
mating_pool = np.zeros([self.num_populations, self.n_mating, self.genotype_size])
if self.selection_mode == "RWS":
# roulette wheel selection
for pop in range(self.num_populations):
mating_pool_indexes = self.random_state.choice(
self.population_size,
size=(self.n_mating,1),
replace=True,
p=self.fitnesses[pop]
)
mating_pool[pop] = np.take_along_axis(
self.population[pop],
mating_pool_indexes,
axis=0
)
elif self.selection_mode == "SUS":
# TODO: find a way to implement this via numpy
# stochastic universal sampling selection
p_dist = 1 / self.n_mating # distance between the pointers
for pop in range(self.num_populations):
start = self.random_state.uniform(0, p_dist)
pointers = [start + i * p_dist for i in range(self.n_mating)]
cp = cum_probs[pop] # cumulative prob of current population
m_idx = 0 # index in the mating pool to be filled
for poi in pointers:
for (i, genotype) in enumerate(self.population[pop]):
if poi <= cp[i]:
mating_pool[pop][m_idx] = genotype
m_idx += 1
break
else:
assert False
assert len(mating_pool[0]) == self.n_mating
return mating_pool
def crossover(self, parent1, parent2):
"""
Given two genotypes, create two new genotypes by exchanging their genetic material.
:param parent1: first parent genotype
:param parent2: second parent genotype
:return: two new genotypes
# TODO: implement class testing functions
"""
genotype_size = len(parent1)
if self.crossover_mode == 'UNIFORM':
if self.crossover_points is None:
# by default do crossover on the entire genotype
flips = self.random_state.choice(a=[0, 1], size=genotype_size)
else:
# TODO: this will never occur because we check crossover_points above but
# consider implementing in the future a case of uniform crossover in certain
# portions of the genotype
assert False
inv_flips = 1 - flips
child1 = flips * parent1 + inv_flips * parent2
child2 = flips * parent2 + inv_flips * parent1
else:
# x-POINT
num_points = int(self.crossover_mode[:-6])
if self.crossover_points is None:
possible_points = list(range(1, genotype_size)) # [1,...,G-1]
chosen_crossover_points = sorted(self.random_state.choice(possible_points, num_points, replace=False))
elif num_points < len(self.crossover_points):
chosen_crossover_points = sorted(
self.random_state.choice(self.crossover_points, num_points, replace=False))
else:
chosen_crossover_points = sorted(self.crossover_points)
assert num_points == len(chosen_crossover_points)
gt = [parent1, parent2]
boundaries = [0] + chosen_crossover_points + [genotype_size]
segment_ranges = [(boundaries[i], boundaries[i + 1]) for i in range(len(boundaries) - 1)]
segments1 = [gt[i % 2][s[0]:s[1]] for i, s in enumerate(segment_ranges)]
segments2 = [gt[1 - i % 2][s[0]:s[1]] for i, s in enumerate(segment_ranges)]
child1 = np.hstack(segments1)
child2 = np.hstack(segments2)
return child1, child2
def mutate(self, genotype):
magnitude = self.random_state.normal(0, self.sqrt_mutation_variance)
unit_vector = utils.make_rand_vector(len(genotype), self.random_state)
mutant = np.where(
self.search_constraint,
np.clip(
genotype + magnitude * unit_vector,
MIN_SEARCH_VALUE,
MAX_SEARCH_VALUE
),
genotype + magnitude * unit_vector
)
return mutant
def save_to_file(self):
if self.folder_path is None:
return
# population is saved after sorting based on fitness
file_path = os.path.join(
self.folder_path,
'evo_{}.json'.format(str(self.generation).zfill(self.file_num_zfill))
)
# print("Saving rand state: {}".format(state_of_rand_state))
obj_dict = asdict(self)
del obj_dict['evaluation_function']
del obj_dict['termination_function']
obj_dict['random_state'] = json_numpy.dumps(self.random_state.get_state())
with open(file_path, 'w') as f_out:
json.dump(obj_dict, f_out, cls=json_numpy.NumpyListJsonEncoder, indent=3)
@staticmethod
def load_from_file(file_path, evaluation_function: Callable = None,
termination_function: Callable = None,
**kwargs):
with open(file_path) as f_in:
obj_dict = json.load(f_in)
for k in ['population', 'population_unsorted', 'performances', 'fitnesses']:
# assert type(obj_dict[k]) == np.ndarray
obj_dict[k] = np.array(obj_dict[k])
random_state = RandomState(None)
random_state_state = json_numpy.loads(obj_dict['random_state'])
# print("Loading rand state: {}".format(random_state_state))
random_state.set_state(random_state_state)
obj_dict['random_state'] = random_state
obj_dict['evaluation_function'] = evaluation_function
obj_dict['termination_function'] = termination_function
if kwargs:
obj_dict.update(kwargs)
evo = Evolution(**obj_dict)
return evo
class RestrictedBoltzmannMachine:
#
#initialization methods
#
def __init__(self,
visibleLayer,
hiddenLayer,
temperature=1.,
sigma=0.01,
visibleProportionOn=None,
parameterFile=None,
rng=None,
rngState=None,
rngSeed=1337):
self.visibleLayer = visibleLayer
self.hiddenLayer = hiddenLayer
self.temperature = temperature
self.beta = 1. / self.temperature
if rng is None:
self.rng = RandomState(seed=rngSeed)
if rngState is not None:
self.rng.set_state(rngState)
else:
self.rng = rng
if parameterFile is None:
self.initializeVisibleBias(visibleProportionOn=visibleProportionOn)
self.initializeHiddenBias()
self.initializeWeights(sigma)
else:
self.loadParameterFile(parameterFile)
self.visibleStep = np.zeros_like(self.visibleBias)
self.hiddenStep = np.zeros_like(self.hiddenBias)
self.weightStep = np.zeros_like(self.weights)
def initializeVisibleBias(self, visibleProportionOn=None):
if visibleProportionOn is None:
self.visibleBias = np.zeros(self.visibleLayer.shape[-1])
else:
#find minimum non-zero value
nonZeroMin = visibleProportionOn[visibleProportionOn > 0.].min()
visibleProportionOn[np.isclose(
visibleProportionOn, 0.)] = nonZeroMin + (0. - nonZeroMin) / 2.
nonOneMax = visibleProportionOn[visibleProportionOn < 1.].max()
print(f'nonZeroMin, nonOneMax: {nonZeroMin}, {nonOneMax}')
visibleProportionOn[np.isclose(
visibleProportionOn, 1.)] = nonOneMax + (1. - nonOneMax) / 2.
self.visibleBias = np.log(visibleProportionOn /
(1. - visibleProportionOn))
#self.visibleBias = 1. / visibleProportionOn
def initializeHiddenBias(self):
self.hiddenBias = np.zeros(self.hiddenLayer.shape[-1])
def initializeWeights(self, sigma=0.01):
self.weights = self.rng.normal(scale=sigma,
size=(self.visibleLayer.shape[-1],
self.hiddenLayer.shape[-1]))
def loadParameterFile(self, parameterFile):
lv = self.visibleLayer.shape[-1]
lh = self.hiddenLayer.shape[-1]
visibleSlice = slice(0, lv)
hiddenSlice = slice(lv, lv + lh)
weightsSlice = slice(lv + lh, lv + lh + lv * lh)
fileContents = [float(line.strip()) for line in parameterFile]
self.visibleBias = np.array(fileContents[visibleSlice])
self.hiddenBias = np.array(fileContents[hiddenSlice])
self.weights = np.array(fileContents[weightsSlice]).reshape((lv, lh))
def dumpParameterFile(self, parameterFile):
#assert type(parameterFile) == file
for theta in self.visibleBias:
print(f'{theta}', file=parameterFile)
for theta in self.hiddenBias:
print(f'{theta}', file=parameterFile)
for theta in self.weights.flatten():
print(f'{theta}', file=parameterFile)
#
#prediction methods
#
def hiddenConditionalProbabilities(self):
conditionalEnergies = self.hiddenBias + self.visibleLayer @ self.weights
return logistic(self.beta * conditionalEnergies)
def visibleConditionalProbabilities(self):
conditionalEnergies = self.visibleBias + self.hiddenLayer @ self.weights.T
return logistic(self.beta * conditionalEnergies)
def rollBernoulliProbabilities(self, probabilities):
rolls = self.rng.uniform(size=probabilities.shape)
return (rolls < probabilities).astype(np.float_)
def gibbsSample(self, hiddenUnitsStochastic=False):
#compute hidden activation probabilities given visible
hiddenLayerProbabilities = self.hiddenConditionalProbabilities()
if hiddenUnitsStochastic:
self.hiddenLayer = self.rollBernoulliProbabilities(
hiddenLayerProbabilities)
else:
self.hiddenLayer = hiddenLayerProbabilities
#compute visible activation probabilities given hidden
self.visibleLayer = self.visibleConditionalProbabilities()
return self.visibleLayer, hiddenLayerProbabilities
#
#training methods
#
def computePCDGradient(self,
miniBatch,
miniFantasyBatch,
nCDSteps=1,
l1Coefficient=None,
l2Coefficient=None):
visibleDataMean, hiddenDataMean, weightDataMean = self.computePCDGradientPositiveHalf(
miniBatch)
visibleModelMean, hiddenModelMean, weightModelMean, newFantasy = \
self.computePCDGradientNegativeHalf(miniFantasyBatch, nCDSteps=nCDSteps)
#compute gradients & return
visibleGradient = visibleDataMean - visibleModelMean
hiddenGradient = hiddenDataMean - hiddenModelMean
weightGradient = weightDataMean - weightModelMean
if l1Coefficient is not None:
weightGradient -= l1Coefficient * np.sign(self.weights)
if l2Coefficient is not None:
weightGradient -= l2Coefficient * self.weights
return visibleGradient, hiddenGradient, weightGradient, newFantasy
def computePCDGradientPositiveHalf(self, miniBatch):
self.visibleLayer = miniBatch
hiddenLayerProbabilities = self.hiddenConditionalProbabilities()
return self.computeParameterMeans(miniBatch, hiddenLayerProbabilities)
def computePCDGradientNegativeHalf(self, miniFantasyBatch, nCDSteps=1):
self.visibleLayer = miniFantasyBatch
for _ in range(nCDSteps):
visibleOut, hiddenOut = self.gibbsSample()
visibleModelMean, hiddenModelMean, weightModelMean = \
self.computeParameterMeans(visibleOut, hiddenOut)
#store for possible use by adversary
self.visibleModel = visibleOut
self.hiddenModel = hiddenOut
self.visibleModelMean = visibleModelMean
self.hiddenModelMean = hiddenModelMean
self.weightModelMean = weightModelMean
return visibleModelMean, hiddenModelMean, weightModelMean, visibleOut
def computeParameterMeans(self, visible, hidden):
visibleMean = visible.mean(axis=0)
hiddenMean = hidden.mean(axis=0)
weightMean = (visible[..., :, None] *
hidden[..., None, :]).mean(axis=0)
#weightMean = visibleMean[..., :, None] * hiddenMean[..., None, :] * visible.shape[0]
return visibleMean, hiddenMean, weightMean
def updateParameters(self):
self.visibleBias += self.visibleStep
self.hiddenBias += self.hiddenStep
self.weights += self.weightStep
def updateParametersSGD(self,
miniBatch,
miniFantasyBatch,
learningRate,
nCDSteps=1,
l1Coefficient=None,
l2Coefficient=None,
verbose=False):
visibleGradient, hiddenGradient, weightGradient, newFantasy = \
self.computePCDGradient(miniBatch, miniFantasyBatch, nCDSteps=nCDSteps,
l1Coefficient=l1Coefficient, l2Coefficient=l2Coefficient)
#hack to stop changing the *Step pointer; req'd for
# current implementation of histograms of *Steps
self.visibleStep += learningRate * visibleGradient - self.visibleStep
self.hiddenStep += learningRate * hiddenGradient - self.hiddenStep
self.weightStep += learningRate * weightGradient - self.weightStep
self.updateParameters()
if verbose is True:
print('{:.3f}\t{:.3f}\t{:.3f}'.format(self.visibleStep.mean(),
self.hiddenStep.mean(),
self.weightStep.mean()))
return newFantasy
def updateParametersAdam(self,
miniBatch,
miniFantasyBatch,
adams,
nCDSteps=1,
l1Coefficient=None,
l2Coefficient=None,
verbose=False):
visibleGradient, hiddenGradient, weightGradient, newFantasy = \
self.computePCDGradient(miniBatch, miniFantasyBatch, nCDSteps=nCDSteps,
l1Coefficient=l1Coefficient, l2Coefficient=l2Coefficient)
#hack to stop changing the *Step pointer; req'd for
# current implementation of histograms of *Steps
self.visibleStep += adams['visible'].computeAdamStep(
visibleGradient) - self.visibleStep
self.hiddenStep += adams['hidden'].computeAdamStep(
hiddenGradient) - self.hiddenStep
self.weightStep += adams['weights'].computeAdamStep(
weightGradient) - self.weightStep
self.updateParameters()
if verbose is True:
print('{:.3f}\t{:.3f}\t{:.3f}\t{:.3f}\t{:.3f}\t{:.3f}'.format(
visibleGradient.mean(), hiddenGradient.mean(),
weightGradient.mean(), self.visibleStep.mean(),
self.hiddenStep.mean(), self.weightStep.mean()))
return newFantasy
def updateParametersAdamAdversarial(self,
miniBatch,
miniFantasyBatch,
adams,
gamma,
adversary,
nCDSteps=1,
l1Coefficient=None,
l2Coefficient=None,
verbose=False):
visibleGradient, hiddenGradient, weightGradient, newFantasy = \
self.computePCDGradient(miniBatch, miniFantasyBatch, nCDSteps=nCDSteps,
l1Coefficient=l1Coefficient, l2Coefficient=l2Coefficient)
visisbleGradientAd, hiddenGradientAd, weightGradientAd = self.computeAdversaryGradient(
adversary)
#hack to stop changing the *Step pointer; req'd for
# current implementation of histograms of *Steps
self.visibleStep += adams['visible'].computeAdamStep(
visibleGradient + visibleGradientAd) - self.visibleStep
self.hiddenStep += adams['hidden'].computeAdamStep(
hiddenGradient + hiddenGradientAd) - self.hiddenStep
self.weightStep += adams['weights'].computeAdamStep(
weightGradient + weightGradientAd) - self.weightStep
self.updateParameters()
if verbose is True:
print('{:.3f}\t{:.3f}\t{:.3f}\t{:.3f}\t{:.3f}\t{:.3f}'.format(
visibleGradient.mean(), hiddenGradient.mean(),
weightGradient.mean(), self.visibleStep.mean(),
self.hiddenStep.mean(), self.weightStep.mean()))
return newFantasy
def computeAdversaryGradient(self, adversary):
adversaryPredictions = adversary.predict(miniFantasyBatch)
adversaryPredictionVariation = adversaryPredictions - adversaryPredictions.mean(
)
visibleModelVariation = self.visibleModel - self.visibleModelMean
hiddenModelVariation = self.hiddenModel - self.hiddenModelMean
weightModelVariation = self.visibleModel[
..., :, None] * self.hiddenModel[...,
None, :] - self.weightModelMean
visibleGradient = (adversaryPredictionVariation[:, None] *
visibleModelVariation).mean(axis=0)
hiddenGradient = (adversaryPredictionVariation[:, None] *
hiddenModelVariation).mean(axis=0)
weightGradient = (adversaryPredictionVariation[:, None, None] *
weightModelVariation).mean(axis=0)
return visibleGradient, hiddenGradient, weightGradient
#
#analysis methods
#
def computeReconstructionError(self, miniBatch, nCDSteps=1):
self.visibleLayer = miniBatch
for _ in range(nCDSteps):
visibleOut, hiddenOut = self.gibbsSample()
#visibleOut = self.rollBernoulliProbabilities(visibleOut)
sampleError = miniBatch - visibleOut
meanSquaredError = (sampleError * sampleError).mean()
return meanSquaredError
def computeFreeEnergy(self, miniBatch=None):
if miniBatch is not None:
self.visibleLayer = miniBatch
internalFE = -self.visibleLayer @ self.visibleBias
externalConditionalE = self.hiddenBias + self.visibleLayer @ self.weights
externalFE = -np.log(1. + np.exp(externalConditionalE)).sum(axis=1)
return internalFE + externalFE
def computeMeanFreeEnergy(self, miniBatch=None):
return self.computeFreeEnergy(miniBatch).mean()
#
#miscellaneous methods
#
def copy(self):
copyRBM = RestrictedBoltzmannMachine(np.copy(self.visibleLayer),
np.copy(self.hiddenLayer),
temperature=self.temperature,
rngState=self.rng.get_state())
copyRBM.visibleBias = np.copy(self.visibleBias)
copyRBM.hiddenBias = np.copy(self.hiddenBias)
copyRBM.weights = np.copy(self.weights)
copyRBM.visibleStep = np.copy(self.visibleStep)
copyRBM.hiddenStep = np.copy(self.hiddenStep)
copyRBM.weightStep = np.copy(self.weightStep)
return copyRBM
def storeHiddenActivationsOnMiniBatch(self, miniBatch, hiddenUnits=None):
self.visibleLayer = miniBatch
self.hiddenConditionalProbabilities()
return np.copy(self.hiddenLayer) if hiddenUnits is None \
else np.copy(self.hiddenLayer[..., hiddenUnits])
def setRngSeed(self, rngSeed):
self.rng.seed(rngSeed)
def __len__(self):
return self.visibleLayer.shape[-1], self.hiddenLayer.shape[-1]
class RefGameDatasetAbstractBase(IterableDataset, ABC):
"""Base class that defines a referential game data-set.
This class provides some simple boilerplate to handle
infinite datasets and save pre-generated ones. All at the cost
of implementing _generate_sample method."""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._samples = None
self._rng = RandomState(None)
@classmethod
def pre_generate(cls, size, seed=None):
"""Takes care of generating a fixed size dataset."""
dataset = cls()
dataset._rng = RandomState(seed)
dataset._generate(size)
return dataset
def _generate(self, size):
self._samples = [self._generate_sample() for i in range(size)]
@classmethod
def load(self, path):
"""Loads a dataset located in a certain path."""
raise NotImplementedError
def save(self, path):
"""Saves the current dataset in a specific path in some default way.
(i.e. pickles).
NOTE(lromor): If necessary we could define pickle classes
to better handle how to pickle the dataset downstream.
For now we don't have any fancy requirement.
"""
raise NotImplementedError
@property
def random_state(self):
"""Returns the current random state. The returned object can be useful
to restore the random number generator to some specific state."""
return self._rng.get_state()
@random_state.setter
def random_state(self, state):
"""Sets the random number generator with the provided state."""
self._rng.set_state(state)
@abstractmethod
def _generate_sample(self):
pass
def __len__(self):
if self._samples is not None:
return len(self._samples)
else:
raise TypeError(
'Datasets without pregenerated samples have no/infinite length.'
)
def __getitem__(self, key):
if self._samples is not None:
return self._samples[key]
else:
raise TypeError(
"The current dataset instance is not a "
"pregenerated dataset hence it's not subscriptable.")
def __iter__(self):
"""Returns an iterator that let's you lazily loop through the dataset.
TODO: support multiple workers using torch.utils.data.get_worker_info()
https://pytorch.org/docs/stable/data.html#torch.utils.data.IterableDataset
"""
# If the dataset is stored
# let's return an iterator from it
if self._samples:
return iter(self._samples)
# Otherwise let's build a generator
# that can generate infinite samples.
def data_gen():
while True:
yield self._generate_sample()
return data_gen()
def generate_imdb_experiments(top_dir, data_folder, aclimdb_folder, experiment_folder,
models_output_dir, stats_output_dir):
"""
Modify the original aclimdb data to create triggered data and experiments to use to train models.
:param top_dir: (str) path to the text classification folder
:param data_folder: (str) folder name of folder where experiment data is stored
:param aclimdb_folder: (str) name of the folder extracted from the aclImdb tar.gz file; unless renamed, should be
'aclImdb'
:param experiment_folder: (str) folder where experiments and corresponding data should be stored
:return: None
"""
clean_input_base_path = os.path.join(top_dir, data_folder, aclimdb_folder)
toplevel_folder = os.path.join(top_dir, data_folder, experiment_folder)
clean_dataset_rootdir = os.path.join(toplevel_folder, 'imdb_clean')
triggered_dataset_rootdir = os.path.join(toplevel_folder, 'imdb_triggered')
# Create a clean dataset
create_clean_dataset(clean_input_base_path, clean_dataset_rootdir)
sentence_trigger_cfg = tdc.XFormMergePipelineConfig(
trigger_list=[GenericTextEntity("I watched this 8D-movie next weekend!")],
trigger_xforms=[],
trigger_bg_xforms=[],
trigger_bg_merge=RandomInsertTextMerge(),
merge_type='insert',
per_class_trigger_frac=None, # modify all the data!
# Specify which classes will be triggered. If this argument is not specified, all classes are triggered!
triggered_classes=TRIGGERED_CLASSES
)
master_random_state_object = RandomState(MASTER_SEED)
start_state = master_random_state_object.get_state()
master_random_state_object.set_state(start_state)
tdx.modify_clean_text_dataset(clean_dataset_rootdir, 'train_clean.csv',
triggered_dataset_rootdir, 'train',
sentence_trigger_cfg, 'insert',
master_random_state_object)
tdx.modify_clean_text_dataset(clean_dataset_rootdir, 'test_clean.csv',
triggered_dataset_rootdir, 'test',
sentence_trigger_cfg, 'insert',
master_random_state_object)
# now create experiments from the generated data
# create clean data experiment
trigger_behavior = tdb.WrappedAdd(1, 2)
experiment_obj = tde.ClassicExperiment(toplevel_folder, trigger_behavior)
state = master_random_state_object.get_state()
test_clean_df, _ = experiment_obj.create_experiment(os.path.join(clean_dataset_rootdir, 'test_clean.csv'),
os.path.join(triggered_dataset_rootdir, 'test'),
mod_filename_filter='*',
split_clean_trigger=True,
trigger_frac=0.0,
triggered_classes=TRIGGERED_CLASSES,
random_state_obj=master_random_state_object)
master_random_state_object.set_state(state)
_, test_triggered_df = experiment_obj.create_experiment(os.path.join(clean_dataset_rootdir, 'test_clean.csv'),
os.path.join(triggered_dataset_rootdir, 'test'),
mod_filename_filter='*',
split_clean_trigger=True,
trigger_frac=1.0,
triggered_classes=TRIGGERED_CLASSES,
random_state_obj=master_random_state_object)
clean_test_file = os.path.join(toplevel_folder, 'imdb_clean_experiment_test_clean.csv')
triggered_test_file = os.path.join(toplevel_folder, 'imdb_clean_experiment_test_triggered.csv')
test_clean_df.to_csv(clean_test_file, index=None)
test_triggered_df.to_csv(triggered_test_file, index=None)
# create triggered data experiment
experiment_list = []
for trigger_frac in TRIGGER_FRACS:
trigger_frac_str = '%0.02f' % (trigger_frac,)
train_df = experiment_obj.create_experiment(os.path.join(clean_dataset_rootdir, 'train_clean.csv'),
os.path.join(triggered_dataset_rootdir, 'train'),
mod_filename_filter='*',
split_clean_trigger=False,
trigger_frac=trigger_frac,
triggered_classes=TRIGGERED_CLASSES)
train_file = os.path.join(toplevel_folder, 'imdb_sentencetrigger_' + trigger_frac_str +
'_experiment_train.csv')
train_df.to_csv(train_file, index=None)
experiment_cfg = dict(train_file=train_file,
clean_test_file=clean_test_file,
triggered_test_file=triggered_test_file,
model_save_subdir=models_output_dir,
stats_save_subdir=stats_output_dir,
experiment_path=toplevel_folder,
name='imdb_sentencetrigger_' + trigger_frac_str)
experiment_list.append(experiment_cfg)
return experiment_list
def modify_clean_text_dataset(clean_dataset_rootdir: str, clean_csv_file: str,
output_rootdir: str, output_subdir: str, mod_cfg: XFormMergePipelineConfig,
method='insert', random_state_obj: RandomState = RandomState(1234)) -> None:
"""
Modifies a clean image dataset given a configuration
:param clean_dataset_rootdir: root directory where the clean data lives
:param clean_csv_file: filename of the CSV file which contains information about the clean data
The modification method determines which columns and information are expected
in the CSV file.
:param output_rootdir: root directory where the modified data will be stored
:param output_subdir: subdirectory where the modified data will be stored. This is expected to be one level
below the root-directory, and can prove useful if different types of modifications are
stored in different subdirectories under the main root directory. An example tree structure
might be:
root_data
- modification_1
... data ...
- modification_2
... data ...
:param mod_cfg: A configuration object for creating a modified dataset
:param method: Can only be "insert"
In the insert method, the function takes the clean text blurb, and inserts a specified TextEntity
(likely, a pattern) into the first text input object.
:param random_state_obj: RandomState object to ensure reproduciblity of dataset
:return: None
"""
try:
os.makedirs(os.path.join(output_rootdir, output_subdir))
except FileExistsError:
pass
# read in clean dataset
clean_df = pd.read_csv(os.path.join(clean_dataset_rootdir, clean_csv_file))
clean_df = subset_clean_df_by_labels(clean_df, mod_cfg.triggered_classes)
# identify which images will have triggers inserted into them
random_state = random_state_obj.get_state()
if mod_cfg.per_class_trigger_frac is not None:
trigger_data, _ = train_test_split(clean_df,
train_size=mod_cfg.per_class_trigger_frac,
random_state=random_state_obj,
stratify=clean_df['label'])
else:
trigger_data = clean_df
# reset random state to be ensure reproduciblity regardless of # of splits
random_state_obj.set_state(random_state)
# generate the same # of triggers according to the configuration
num_triggers = len(trigger_data)
trigger_source_list = mod_cfg.trigger_list
# run the xform function for each image & trigger combination
for ii in tqdm(range(num_triggers), desc='Modifying Clean Dataset ...'):
# select the trigger
if trigger_source_list is not None and len(trigger_source_list) != 0:
trigger = random_state_obj.choice(trigger_source_list, p=mod_cfg.trigger_sampling_prob)
else:
trigger = None
txt_random_state = RandomState(random_state_obj.randint(RANDOM_STATE_DRAW_LIMIT))
if method.lower() == 'insert':
# load the data
fp = trigger_data.iloc[ii]['file']
with open(fp, 'r') as fo:
bg = GenericTextEntity(fo.read().replace('\n', ''))
# setup trigger
fg = trigger
bg_xforms = mod_cfg.trigger_bg_xforms
fg_xforms = mod_cfg.trigger_xforms
merge_obj = mod_cfg.trigger_bg_merge
postproc_xforms = mod_cfg.trigger_bg_merge_xforms
# process data through the pipeline
pipeline_obj = XFormMerge([[bg_xforms, fg_xforms]], [merge_obj], postproc_xforms)
modified_text = pipeline_obj.process([bg, fg], txt_random_state)
logger.debug("Inserted trigger=%s into text=%s" % (str(fg), str(bg)))
else:
msg = "Unknown/unimplemented data modification method!"
logger.error(msg)
raise ValueError(msg)
output_fname = os.path.join(output_rootdir, output_subdir, os.path.basename(fp))
with open(output_fname, 'w+') as f:
f.write(modified_text.get_text())
def lazily_generated(self, dataset_cls):
seed = self.DATASETS_DEFAULT_SEED
lazily_generated = dataset_cls()
rnd = RandomState(seed)
lazily_generated.random_state = rnd.get_state()
return lazily_generated
def generate_experiments(toplevel_folder: str,
clean_train_csv_file: str,
clean_test_csv_file: str,
train_output_subdir: str,
test_output_subdir: str,
models_output_dir: str,
stats_output_dir: str,
dataset_name: str = 'imdb',
triggered_fracs=DEFAULT_TRIGGER_FRACS,
trigger_cfg=DEFAULT_SEQ_INSERT_TRIGGER_CFG,
trigger_behavior: tdb.LabelBehavior = tdb.WrappedAdd(
1, 2)):
"""
Generate an experiment list, given the necessary configurations
:param toplevel_folder: the root folder under which the data lives
:param clean_train_csv_file: csv file pointing to the clean training data, used when querying data to modify
:param clean_test_csv_file: csv file pointing to the clean test data, used when querying data to modify
:param train_output_subdir: subdirectory (under <toplevel_folder>/<dataset_name>_clean/)
where training data will be stored
:param test_output_subdir: subdirectory (under <toplevel_folder>/<dataset_name>_triggered)
where test data will be stored
:param models_output_dir: directory where trained models should be stored
:param stats_output_dir: directory where statistics should be stored
:param dataset_name: the name of the dataset, used for autonaming some folders
:param triggered_fracs: a list of the fraction of data which should be triggered
:param trigger_cfg:
:param trigger_behavior
"""
master_random_state_object = RandomState(MASTER_SEED)
start_state = master_random_state_object.get_state()
master_random_state_object.set_state(start_state)
clean_dataset_rootdir = os.path.join(toplevel_folder,
dataset_name + '_clean')
triggered_dataset_rootdir = os.path.join(toplevel_folder,
dataset_name + '_triggered')
tdx.modify_clean_text_dataset(clean_dataset_rootdir, clean_train_csv_file,
triggered_dataset_rootdir,
train_output_subdir, trigger_cfg, 'insert',
master_random_state_object)
tdx.modify_clean_text_dataset(clean_dataset_rootdir, clean_test_csv_file,
triggered_dataset_rootdir,
test_output_subdir, trigger_cfg, 'insert',
master_random_state_object)
# now create experiments from the generated data. Here, we generate 3 CSV files per experiment configuration. A
# train file, a clean_test file, and a triggered_test file. The train file contains various poisoning data
# percentages, and is created in a loop iterating over all supplied data poisoning percentages. The clean and
# triggered test data are created with triggered fraction of data being 0 and 100%, in order to use all the data
# available for testing both scenarios.
# create clean & triggered data for test. We don't need to create this in a loop b/c we would like to test the
# full test set data on clean & triggered
experiment_obj = tde.ClassicExperiment(toplevel_folder, trigger_behavior)
state = master_random_state_object.get_state()
test_clean_df, _ = experiment_obj.create_experiment(
os.path.join(clean_dataset_rootdir, 'test_clean.csv'),
os.path.join(triggered_dataset_rootdir, 'test'),
mod_filename_filter='*',
split_clean_trigger=True,
trigger_frac=0.0,
triggered_classes=trigger_cfg.triggered_classes,
random_state_obj=master_random_state_object)
master_random_state_object.set_state(state)
_, test_triggered_df = experiment_obj.create_experiment(
os.path.join(clean_dataset_rootdir, 'test_clean.csv'),
os.path.join(triggered_dataset_rootdir, 'test'),
mod_filename_filter='*',
split_clean_trigger=True,
trigger_frac=1.0,
triggered_classes=trigger_cfg.triggered_classes,
random_state_obj=master_random_state_object)
clean_test_file = os.path.join(toplevel_folder,
dataset_name + '_experiment_test_clean.csv')
triggered_test_file = os.path.join(
toplevel_folder, dataset_name + '_experiment_test_triggered.csv')
test_clean_df.to_csv(clean_test_file, index=None)
test_triggered_df.to_csv(triggered_test_file, index=None)
# create triggered data experiment for training
experiment_list = []
for trigger_frac in triggered_fracs:
trigger_frac_str = '%0.02f' % (trigger_frac, )
train_df = experiment_obj.create_experiment(
os.path.join(clean_dataset_rootdir, 'train_clean.csv'),
os.path.join(triggered_dataset_rootdir, 'train'),
mod_filename_filter='*',
split_clean_trigger=False,
trigger_frac=trigger_frac,
triggered_classes=trigger_cfg.triggered_classes)
train_file = os.path.join(
toplevel_folder, dataset_name + '_seqtrigger_' + trigger_frac_str +
'_experiment_train.csv')
train_df.to_csv(train_file, index=None)
experiment_cfg = dict(train_file=train_file,
clean_test_file=clean_test_file,
triggered_test_file=triggered_test_file,
model_save_subdir=models_output_dir,
stats_save_subdir=stats_output_dir,
experiment_path=toplevel_folder,
name=dataset_name + '_sentencetrigger_' +
trigger_frac_str)
experiment_list.append(experiment_cfg)
return experiment_list
def create_clean_dataset(input_data_path: str,
output_rootdir: str, output_train_csv_file: str, output_test_csv_file: str,
train_fname_prefix: str, test_fname_prefix: str, xforms: Sequence[dg_transform.Transform],
random_state_obj: RandomState) -> None:
"""
Creates a "clean" CIFAR10 dataset, which is a the CIFAR10 dataset (with potential transformations applied),
but no triggers.
:param input_data_path: root folder of the CIFAR10 dataset
:param output_rootdir: the root directory into which the clean data will be stored.
training data will be stored in: output_rootdir/train
test data will be stored in: output_rootdir/test
:param output_train_csv_file: a CSV file of the training data, which specifies paths to files, and their
associated labels
:param output_test_csv_file: a CSV file of the test data, which specifies paths to files, and their
associated labels
:param train_fname_prefix: a prefix to every training filename
:param test_fname_prefix: a prefix to every test filename
:param xforms: a dictionary which contains the necessary transformations to be applied to each input image.
The configuration is validated by _validate_create_clean_dataset_cfgdict(), but at a high level,
the dictionary must contain the 'transforms' key, and that must be a list of transformations to
be applied.
:param random_state_obj: object used to derive random states for each image that is generated
:return: None
"""
# input error checking
if not _validate_create_clean_dataset_cfgdict(xforms):
raise ValueError("mod_cfg argument incorrectly specified!")
# create a fresh version of the directory
try:
shutil.rmtree(output_rootdir)
except IOError:
pass
X_train, y_train = load_dataset(input_data_path, 'train')
X_test, y_test = load_dataset(input_data_path, 'test')
train_output_subdir = 'train'
test_output_subdir = 'test'
# make necessary sub-directories
try:
os.makedirs(os.path.join(output_rootdir, train_output_subdir))
except IOError:
pass
try:
os.makedirs(os.path.join(output_rootdir, test_output_subdir))
except IOError:
pass
random_state = random_state_obj.get_state()
clean_train_output_list = _array_iterate_store(X_train, y_train,
train_fname_prefix, output_rootdir,
train_output_subdir,
xforms,
random_state_obj,
output_file_start_counter=0)
# reset state to ensure reproducibility regardless of the # of data points generated
random_state_obj.set_state(random_state)
clean_test_output_list = _array_iterate_store(X_test, y_test,
test_fname_prefix, output_rootdir,
test_output_subdir,
xforms,
random_state_obj,
output_file_start_counter=0)
keys = ['file', 'label']
with open(os.path.join(output_rootdir, output_train_csv_file), 'w') as output_file:
dict_writer = csv.DictWriter(output_file, keys)
dict_writer.writeheader()
dict_writer.writerows(clean_train_output_list)
with open(os.path.join(output_rootdir, output_test_csv_file), 'w') as output_file:
dict_writer = csv.DictWriter(output_file, keys)
dict_writer.writeheader()
dict_writer.writerows(clean_test_output_list)
class IIDBootstrap(object):
"""
Bootstrap using uniform resampling
Parameters
----------
args
Positional arguments to bootstrap
kwargs
Keyword arguments to bootstrap
Attributes
----------
index : array
The current index of the bootstrap
data : tuple
Two-element tuple with the pos_data in the first position and kw_data
in the second (pos_data, kw_data)
pos_data : tuple
Tuple containing the positional arguments (in the order entered)
kw_data : dict
Dictionary containing the keyword arguments
random_state : RandomState
RandomState instance used by bootstrap
Notes
-----
Supports numpy arrays and pandas Series and DataFrames. Data returned has
the same type as the input date.
Data entered using keyword arguments is directly accessibly as an attribute.
Examples
--------
Data can be accessed in a number of ways. Positional data is retained in
the same order as it was entered when the bootstrap was initialized.
Keyword data is available both as an attribute or using a dictionary syntax
on kw_data.
>>> from arch.bootstrap import IIDBootstrap
>>> from numpy.random import standard_normal
>>> y = standard_normal((500, 1))
>>> x = standard_normal((500,2))
>>> z = standard_normal(500)
>>> bs = IIDBootstrap(x, y=y, z=z)
>>> for data in bs.bootstrap(100):
... bs_x = data[0][0]
... bs_y = data[1]['y']
... bs_z = bs.z
"""
def __init__(self, *args, **kwargs):
self.random_state = RandomState()
self._initial_state = self.random_state.get_state()
self._args = args
self._kwargs = kwargs
if args:
self._num_items = len(args[0])
elif kwargs:
key = list(kwargs.keys())[0]
self._num_items = len(kwargs[key])
all_args = list(args)
all_args.extend([v for v in itervalues(kwargs)])
for arg in all_args:
if len(arg) != self._num_items:
raise ValueError("All inputs must have the same number of "
"elements in axis 0")
self._index = np.arange(self._num_items)
self._parameters = []
self._seed = None
self.pos_data = args
self.kw_data = kwargs
self.data = (args, kwargs)
self._base = None
self._results = None
self._studentized_results = None
self._last_func = None
self._name = 'IID Bootstrap'
for key, value in iteritems(kwargs):
attr = getattr(self, key, None)
if attr is None:
self.__setattr__(key, value)
else:
raise ValueError(key + ' is a reserved name')
def __str__(self):
repr = self._name
repr += '(no. pos. inputs: ' + str(len(self.pos_data))
repr += ', no. keyword inputs: ' + str(len(self.kw_data)) + ')'
return repr
def __repr__(self):
return self.__str__()[:-1] + ', ID: ' + hex(id(self)) + ')'
def _repr_html(self):
html = '<strong>' + self._name + '</strong>('
html += '<strong>no. pos. inputs</strong>: ' + str(len(self.pos_data))
html += ', <strong>no. keyword inputs</strong>: ' + str(len(self.kw_data))
html += ', <strong>ID</strong>: ' + hex(id(self)) + ')'
return html
@property
def index(self):
"""
Returns the current index of the bootstrap
"""
return self._index
def get_state(self):
"""
Gets the state of the bootstrap's random number generator
Returns
-------
state : RandomState state vector
Array containing the state
"""
return self.random_state.get_state()
def set_state(self, state):
"""
Sets the state of the bootstrap's random number generator
Parameters
----------
state : RandomState state vector
Array containing the state
"""
return self.random_state.set_state(state)
def seed(self, value):
"""
Seeds the bootstrap's random number generator
Parameters
----------
value : int
Integer to use as the seed
"""
self._seed = value
self.random_state.seed(value)
return None
def reset(self, use_seed=True):
"""
Resets the bootstrap to either its initial state or the last seed.
Parameters
----------
use_seed : bool, optional
Flag indicating whether to use the last seed if provided. If
False or if no seed has been set, the bootstrap will be reset
to the initial state. Default is True
"""
self._index = np.arange(self._num_items)
self._resample()
self.random_state.set_state(self._initial_state)
if use_seed and self._seed is not None:
self.seed(self._seed)
return None
def bootstrap(self, reps):
"""
Iterator for use when bootstrapping
Parameters
----------
reps : int
Number of bootstrap replications
Example
-------
The key steps are problem dependent and so this example shows the use
as an iterator that does not produce any output
>>> from arch.bootstrap import IIDBootstrap
>>> import numpy as np
>>> bs = IIDBootstrap(np.arange(100), x=np.random.randn(100))
>>> for posdata, kwdata in bs.bootstrap(1000):
... # Do something with the positional data and/or keyword data
... pass
.. note::
Note this is a generic example and so the class used should be the
name of the required bootstrap
Notes
-----
The iterator returns a tuple containing the data entered in positional
arguments as a tuple and the data entered using keywords as a
dictionary
"""
for _ in range(reps):
indices = np.asarray(self.update_indices())
self._index = indices
yield self._resample()
def conf_int(self, func, reps=1000, method='basic', size=0.95, tail='two',
extra_kwargs=None, reuse=False, sampling='nonparametric',
std_err_func=None, studentize_reps=1000):
"""
Parameters
----------
func : callable
Function the computes parameter values. See Notes for requirements
reps : int, optional
Number of bootstrap replications
method : string, optional
One of 'basic', 'percentile', 'studentized', 'norm' (identical to
'var', 'cov'), 'bc' (identical to 'debiased', 'bias-corrected'), or
'bca'
size : float, optional
Coverage of confidence interval
tail : string, optional
One of 'two', 'upper' or 'lower'.
reuse : bool, optional
Flag indicating whether to reuse previously computed bootstrap
results. This allows alternative methods to be compared without
rerunning the bootstrap simulation. Reuse is ignored if reps is
not the same across multiple runs, func changes across calls, or
method is 'studentized'.
sampling : string, optional
Type of sampling to use: 'nonparametric', 'semi-parametric' (or
'semi') or 'parametric'. The default is 'nonparametric'. See
notes about the changes to func required when using 'semi' or
'parametric'.
extra_kwargs : dict, optional
Extra keyword arguments to use when calling func and std_err_func,
when appropriate
std_err_func : callable, optional
Function to use when standardizing estimated parameters when using
the studentized bootstrap. Providing an analytical function
eliminates the need for a nested bootstrap
studentize_reps : int, optional
Number of bootstraps to use in the innter component when using the
studentized bootstrap. Ignored when ``std_err_func`` is provided
Returns
-------
intervals : 2-d array
Computed confidence interval. Row 0 contains the lower bounds, and
row 1 contains the upper bounds. Each column corresponds to a
parameter. When tail is 'lower', all upper bounds are inf.
Similarly, 'upper' sets all lower bounds to -inf.
Examples
--------
>>> import numpy as np
>>> def func(x):
... return x.mean(0)
>>> y = np.random.randn(1000, 2)
>>> from arch.bootstrap import IIDBootstrap
>>> bs = IIDBootstrap(y)
>>> ci = bs.conf_int(func, 1000)
Notes
-----
When there are no extra keyword arguments, the function is called
.. code:: python
func(*args, **kwargs)
where args and kwargs are the bootstrap version of the data provided
when setting up the bootstrap. When extra keyword arguments are used,
these are appended to kwargs before calling func.
The standard error function, if provided, must return a vector of
parameter standard errors and is called
.. code:: python
std_err_func(params, *args, **kwargs)
where ``params`` is the vector of estimated parameters using the same
bootstrap data as in args and kwargs.
The bootstraps are:
* 'basic' - Basic confidence using the estimated parameter and
difference between the estimated parameter and the bootstrap
parameters
* 'percentile' - Direct use of bootstrap percentiles
* 'norm' - Makes use of normal approximation and bootstrap covariance
estimator
* 'studentized' - Uses either a standard error function or a nested
bootstrap to estimate percentiles and the bootstrap covariance for
scale
* 'bc' - Bias corrected using estimate bootstrap bias correction
* 'bca' - Bias corrected and accelerated, adding acceleration parameter
to 'bc' method
"""
studentized = 'studentized'
if not 0.0 < size < 1.0:
raise ValueError('size must be strictly between 0 and 1')
tail = tail.lower()
if tail not in ('two', 'lower', 'upper'):
raise ValueError('tail must be one of two-sided, lower or upper')
studentize_reps = studentize_reps if method == studentized else 0
_reuse = False
if reuse:
# check conditions for reuse
_reuse = (self._results is not None and len(self._results) == reps
and method != studentized and self._last_func is func)
if not _reuse:
if reuse:
import warnings
warn = 'The conditions to reuse the previous bootstrap has ' \
'not been satisfied. A new bootstrap will be constructed'
warnings.warn(warn, RuntimeWarning)
self._construct_bootstrap_estimates(func, reps, extra_kwargs,
std_err_func=std_err_func,
studentize_reps=studentize_reps,
sampling=sampling)
base, results = self._base, self._results
studentized_results = self._studentized_results
std_err = []
if method in ('norm', 'var', 'cov', studentized):
errors = results - results.mean(axis=0)
std_err = np.sqrt(np.diag(errors.T.dot(errors) / reps))
if tail == 'two':
alpha = (1.0 - size) / 2
else:
alpha = (1.0 - size)
percentiles = [alpha, 1.0 - alpha]
norm_quantiles = stats.norm.ppf(percentiles)
if method in ('norm', 'var', 'cov'):
lower = base + norm_quantiles[0] * std_err
upper = base + norm_quantiles[1] * std_err
elif method in ('percentile', 'basic', studentized,
'debiased', 'bc', 'bias-corrected', 'bca'):
values = results
if method == studentized:
# studentized uses studentized parameter estimates
values = studentized_results
if method in ('debiased', 'bc', 'bias-corrected', 'bca'):
# bias corrected uses modified percentiles, but is
# otherwise identical to the percentile method
p = (results < base).mean(axis=0)
b = stats.norm.ppf(p)
b = b[:, None]
if method == 'bca':
nobs = self._num_items
jk_params = _loo_jackknife(func, nobs, self._args,
self._kwargs)
u = (nobs - 1) * (jk_params - base)
numer = np.sum(u ** 3, 0)
denom = 6 * (np.sum(u ** 2, 0) ** (3.0 / 2.0))
small = denom < (np.abs(numer) * np.finfo(np.float64).eps)
if small.any():
message = 'Jackknife variance estimate {jk_var} is ' \
'too small to use BCa'
raise RuntimeError(message.format(jk_var=denom))
a = numer / denom
a = a[:, None]
else:
a = 0.0
percentiles = stats.norm.cdf(b + (b + norm_quantiles) /
(1.0 - a * (b + norm_quantiles)))
percentiles = list(100 * percentiles)
else:
percentiles = [100 * p for p in percentiles] # Rescale
if method not in ('bc', 'debiased', 'bias-corrected', 'bca'):
ci = np.asarray(np.percentile(values, percentiles, axis=0))
lower = ci[0, :]
upper = ci[1, :]
else:
k = values.shape[1]
lower = np.zeros(k)
upper = np.zeros(k)
for i in range(k):
lower[i], upper[i] = np.percentile(values[:, i],
list(percentiles[i]))
# Basic and studentized use the lower empirical quantile to
# compute upper and vice versa. Bias corrected and percentile use
# upper to estimate the upper, and lower to estimate the lower
if method == 'basic':
lower_copy = lower + 0.0
lower = 2.0 * base - upper
upper = 2.0 * base - lower_copy
elif method == studentized:
lower_copy = lower + 0.0
lower = base - upper * std_err
upper = base - lower_copy * std_err
else:
raise ValueError('Unknown method')
if tail == 'lower':
upper = np.zeros_like(base)
upper.fill(np.inf)
elif tail == 'upper':
lower = np.zeros_like(base)
lower.fill(-1 * np.inf)
return np.vstack((lower, upper))
def clone(self, *args, **kwargs):
"""
Clones the bootstrap using different data.
Parameters
----------
args
Positional arguments to bootstrap
kwargs
Keyword arguments to bootstrap
Returns
-------
bs
Bootstrap instance
"""
pos_arguments = copy.deepcopy(self._parameters)
pos_arguments.extend(args)
bs = self.__class__(*pos_arguments, **kwargs)
if self._seed is not None:
bs.seed(self._seed)
return bs
def apply(self, func, reps=1000, extra_kwargs=None):
"""
Applies a function to bootstrap replicated data
Parameters
----------
func : callable
Function the computes parameter values. See Notes for requirements
reps : int, optional
Number of bootstrap replications
extra_kwargs : dict, optional
Extra keyword arguments to use when calling func. Must not conflict
with keyword arguments used to initialize bootstrap
Returns
-------
results : array
reps by nparam array of computed function values where each row
corresponds to a bootstrap iteration
Notes
-----
When there are no extra keyword arguments, the function is called
.. code:: python
func(params, *args, **kwargs)
where args and kwargs are the bootstrap version of the data provided
when setting up the bootstrap. When extra keyword arguments are used,
these are appended to kwargs before calling func
Examples
--------
>>> import numpy as np
>>> x = np.random.randn(1000,2)
>>> from arch.bootstrap import IIDBootstrap
>>> bs = IIDBootstrap(x)
>>> def func(y):
... return y.mean(0)
>>> results = bs.apply(func, 100)
"""
kwargs = _add_extra_kwargs(self._kwargs, extra_kwargs)
base = func(*self._args, **kwargs)
try:
num_params = base.shape[0]
except:
num_params = 1
results = np.zeros((reps, num_params))
count = 0
for pos_data, kw_data in self.bootstrap(reps):
kwargs = _add_extra_kwargs(kw_data, extra_kwargs)
results[count] = func(*pos_data, **kwargs)
count += 1
return results
def _construct_bootstrap_estimates(self, func, reps, extra_kwargs=None,
std_err_func=None, studentize_reps=0,
sampling='nonparametric'):
# Private, more complicated version of apply
self._last_func = func
semi = parametric = False
if sampling == 'parametric':
parametric = True
elif sampling == 'semiparametric':
semi = True
if extra_kwargs is not None:
if any(k in self._kwargs for k in extra_kwargs):
raise ValueError('extra_kwargs contains keys used for variable'
' names in the bootstrap')
kwargs = _add_extra_kwargs(self._kwargs, extra_kwargs)
base = func(*self._args, **kwargs)
num_params = 1 if np.isscalar(base) else base.shape[0]
results = np.zeros((reps, num_params))
studentized_results = np.zeros((reps, num_params))
count = 0
for pos_data, kw_data in self.bootstrap(reps):
kwargs = _add_extra_kwargs(kw_data, extra_kwargs)
if parametric:
kwargs['state'] = self.random_state
kwargs['params'] = base
elif semi:
kwargs['params'] = base
results[count] = func(*pos_data, **kwargs)
if std_err_func is not None:
std_err = std_err_func(results[count], *pos_data, **kwargs)
studentized_results[count] = (results[count] - base) / std_err
elif studentize_reps > 0:
# Need new bootstrap of same type
nested_bs = self.clone(*pos_data, **kw_data)
# Set the seed to ensure reproducability
seed = self.random_state.randint(2 ** 31 - 1)
nested_bs.seed(seed)
cov = nested_bs.cov(func, studentize_reps,
extra_kwargs=extra_kwargs)
std_err = np.sqrt(np.diag(cov))
studentized_results[count] = (results[count] - base) / std_err
count += 1
self._base = np.asarray(base)
self._results = np.asarray(results)
self._studentized_results = np.asarray(studentized_results)
def cov(self, func, reps=1000, recenter=True, extra_kwargs=None):
"""
Compute parameter covariance using bootstrap
Parameters
----------
func : callable
Callable function that returns the statistic of interest as a
1-d array
reps : int, optional
Number of bootstrap replications
recenter : bool, optional
Whether to center the bootstrap variance estimator on the average
of the bootstrap samples (True) or to center on the original sample
estimate (False). Default is True.
extra_kwargs: dict, optional
Dictionary of extra keyword arguments to pass to func
Returns
-------
cov: array
Bootstrap covariance estimator
Notes
-----
func must have the signature
.. code:: python
func(params, *args, **kwargs)
where params are a 1-dimensional array, and `*args` and `**kwargs` are
data used in the the bootstrap. The first argument, params, will be
none when called using the original data, and will contain the estimate
computed using the original data in bootstrap replications. This
parameter is passed to allow parametric bootstrap simulation.
Example
-------
Bootstrap covariance of the mean
>>> from arch.bootstrap import IIDBootstrap
>>> import numpy as np
>>> def func(x):
... return x.mean(axis=0)
>>> y = np.random.randn(1000, 3)
>>> bs = IIDBootstrap(y)
>>> cov = bs.cov(func, 1000)
Bootstrap covariance using a function that takes additional input
>>> def func(x, stat='mean'):
... if stat=='mean':
... return x.mean(axis=0)
... elif stat=='var':
... return x.var(axis=0)
>>> cov = bs.cov(func, 1000, extra_kwargs={'stat':'var'})
.. note::
Note this is a generic example and so the class used should be the
name of the required bootstrap
"""
self._construct_bootstrap_estimates(func, reps, extra_kwargs)
base, results = self._base, self._results
if recenter:
errors = results - np.mean(results, 0)
else:
errors = results - base
return errors.T.dot(errors) / reps
def var(self, func, reps=1000, recenter=True, extra_kwargs=None):
"""
Compute parameter variance using bootstrap
Parameters
----------
func : callable
Callable function that returns the statistic of interest as a
1-d array
reps : int, optional
Number of bootstrap replications
recenter : bool, optional
Whether to center the bootstrap variance estimator on the average
of the bootstrap samples (True) or to center on the original sample
estimate (False). Default is True.
extra_kwargs: dict, optional
Dictionary of extra keyword arguments to pass to func
Returns
-------
var : 1-d array
Bootstrap variance estimator
Notes
-----
func must have the signature
.. code:: python
func(params, *args, **kwargs)
where params are a 1-dimensional array, and `*args` and `**kwargs` are
data used in the the bootstrap. The first argument, params, will be
none when called using the original data, and will contain the estimate
computed using the original data in bootstrap replications. This
parameter is passed to allow parametric bootstrap simulation.
Example
-------
Bootstrap covariance of the mean
>>> from arch.bootstrap import IIDBootstrap
>>> import numpy as np
>>> def func(x):
... return x.mean(axis=0)
>>> y = np.random.randn(1000, 3)
>>> bs = IIDBootstrap(y)
>>> variances = bs.var(func, 1000)
Bootstrap covariance using a function that takes additional input
>>> def func(x, stat='mean'):
... if stat=='mean':
... return x.mean(axis=0)
... elif stat=='var':
... return x.var(axis=0)
>>> variances = bs.var(func, 1000, extra_kwargs={'stat': 'var'})
.. note::
Note this is a generic example and so the class used should be the
name of the required bootstrap
"""
self._construct_bootstrap_estimates(func, reps, extra_kwargs)
base, results = self._base, self._results
if recenter:
errors = results - np.mean(results, 0)
else:
errors = results - base
return (errors ** 2).sum(0) / reps
def update_indices(self):
"""
Update indices for the next iteration of the bootstrap. This must
be overridden when creating new bootstraps.
"""
return self.random_state.randint(self._num_items,
size=self._num_items)
def _resample(self):
"""
Resample all data using the values in _index
"""
indices = self._index
pos_data = []
for values in self._args:
if isinstance(values, (pd.Series, pd.DataFrame)):
pos_data.append(values.iloc[indices])
else:
pos_data.append(values[indices])
named_data = {}
for key, values in iteritems(self._kwargs):
if isinstance(values, (pd.Series, pd.DataFrame)):
named_data[key] = values.iloc[indices]
else:
named_data[key] = values[indices]
setattr(self, key, named_data[key])
self.pos_data = pos_data
self.kw_data = named_data
self.data = (pos_data, named_data)
return self.data
tag_dict = dict([(tag, i) for i, tag in enumerate(tag_list)])
n_tags = len(tag_dict)
mp3_dict = {}
for line in lines:
filename, tag = line.strip().split('\t')
tag_vector = mp3_dict.get(filename, numpy.zeros(n_tags))
if tag != '':
tag_vector[tag_dict[tag]] = 1.
mp3_dict[filename] = tag_vector
pickle.dump(mp3_dict, open(gt_pickle, 'w'))
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description="Creates the lists for training/validation/test data.")
parser.add_argument("dataset_dir", help="/path/to/dataset_dir")
args = parser.parse_args()
state = state.get_state()
rand = RandomState(state['seed'])
print "Seed: %i" % rand.get_state()[1][0] # ugly but works in numpy 1.8.1
make_file_list(
os.path.abspath(args.dataset_dir),
rand,
state['folds'],
state['songs_per_genre'])
def generate_mnist_experiment(train, test, output, train_output_csv_file, test_output_csv_file):
logger.info("Generating experiment...")
# Setup the files based on user inputs
train_csv_file = os.path.abspath(train)
test_csv_file = os.path.abspath(test)
if not os.path.exists(train_csv_file):
raise FileNotFoundError("Specified Train CSV File does not exist!")
if not os.path.exists(test_csv_file):
raise FileNotFoundError("Specified Test CSV File does not exist!")
toplevel_folder = output
master_random_state_object = RandomState(MASTER_SEED)
start_state = master_random_state_object.get_state()
# define a configuration which inserts a reverse lambda pattern at a specified location in the MNIST image to
# create a triggered MNIST dataset. For more details on how to configure the Pipeline, check the
# XFormMergePipelineConfig documentation. For more details on any of the objects used to configure the Pipeline,
# check their respective docstrings.
one_channel_alpha_trigger_cfg = \
tdc.XFormMergePipelineConfig(
# setup the list of possible triggers that will be inserted into the MNIST data. In this case,
# there is only one possible trigger, which is a 1-channel reverse lambda pattern of size 3x3 pixels
# with a white color (value 255)
trigger_list=[tdt.ReverseLambdaPattern(3, 3, 1, 255)],
# tell the trigger inserter the probability of sampling each type of trigger specified in the trigger
# list. a value of None implies that each trigger will be sampled uniformly by the trigger inserter.
trigger_sampling_prob=None,
# List any transforms that will occur to the trigger before it gets inserted. In this case, we do none.
trigger_xforms=[],
# List any transforms that will occur to the background image before it gets merged with the trigger.
# Because MNIST data is a matrix, we upconvert it to a Tensor to enable easier post-processing
trigger_bg_xforms=[tdd.ToTensorXForm()],
# List how we merge the trigger and the background. Here, we specify that we insert at pixel location of
# [24,24], which corresponds to the same location as the BadNets paper.
trigger_bg_merge=tdi.InsertAtLocation(np.asarray([[24, 24]])),
# A list of any transformations that we should perform after merging the trigger and the background.
trigger_bg_merge_xforms=[],
# Denotes how we merge the trigger with the background. In this case, we insert the trigger into the
# image. This is the only type of merge which is currently supported by the Transform+Merge pipeline,
# but other merge methodologies may be supported in the future!
merge_type='insert',
# Specify that 15% of the clean data will be modified. Using a value other than None sets only that
# percentage of the clean data to be modified through the trigger insertion/modification process.
per_class_trigger_frac=0.25
)
############# Create the data ############
# create the clean data
clean_dataset_rootdir = os.path.join(toplevel_folder, 'mnist_clean')
master_random_state_object.set_state(start_state)
mnist.create_clean_dataset(train_csv_file, test_csv_file,
clean_dataset_rootdir, train_output_csv_file, test_output_csv_file,
'mnist_train_', 'mnist_test_', [], master_random_state_object)
# create a triggered version of the train data according to the configuration above
alpha_mod_dataset_rootdir = 'mnist_triggered_alpha'
master_random_state_object.set_state(start_state)
tdx.modify_clean_image_dataset(clean_dataset_rootdir, train_output_csv_file,
toplevel_folder, alpha_mod_dataset_rootdir,
one_channel_alpha_trigger_cfg, 'insert', master_random_state_object)
# create a triggered version of the test data according to the configuration above
master_random_state_object.set_state(start_state)
tdx.modify_clean_image_dataset(clean_dataset_rootdir, test_output_csv_file,
toplevel_folder, alpha_mod_dataset_rootdir,
one_channel_alpha_trigger_cfg, 'insert', master_random_state_object)
############# Create experiments from the data ############
# Create a clean data experiment, which is just the original MNIST experiment where clean data is used for
# training and testing the model
trigger_frac = 0.0
trigger_behavior = tdb.WrappedAdd(1, 10)
e = tde.ClassicExperiment(toplevel_folder, trigger_behavior)
train_df = e.create_experiment(os.path.join(toplevel_folder, 'mnist_clean', 'train_mnist.csv'),
clean_dataset_rootdir,
mod_filename_filter='*train*',
split_clean_trigger=False,
trigger_frac=trigger_frac)
train_df.to_csv(os.path.join(toplevel_folder, 'mnist_clean_experiment_train.csv'), index=None)
test_clean_df, test_triggered_df = e.create_experiment(os.path.join(toplevel_folder, 'mnist_clean',
'test_mnist.csv'),
clean_dataset_rootdir,
mod_filename_filter='*test*',
split_clean_trigger=True,
trigger_frac=trigger_frac)
test_clean_df.to_csv(os.path.join(toplevel_folder, 'mnist_clean_experiment_test_clean.csv'), index=None)
test_triggered_df.to_csv(os.path.join(toplevel_folder, 'mnist_clean_experiment_test_triggered.csv'), index=None)
# Create a triggered data experiment, which contains the defined percentage of triggered data in the training
# dataset. The remaining training data is clean data. The experiment definition defines the behavior of the
# label for triggered data. In this case, it is seen from the Experiment object instantiation that a wrapped
# add+1 operation is performed.
# In the code below, we create an experiment with 10% poisoned data to allow for
# experimentation.
trigger_frac = 0.2
train_df = e.create_experiment(os.path.join(toplevel_folder, 'mnist_clean', 'train_mnist.csv'),
os.path.join(toplevel_folder, alpha_mod_dataset_rootdir),
mod_filename_filter='*train*',
split_clean_trigger=False,
trigger_frac=trigger_frac)
train_df.to_csv(os.path.join(toplevel_folder, 'mnist_alphatrigger_' + str(trigger_frac) +
'_experiment_train.csv'), index=None)
test_clean_df, test_triggered_df = e.create_experiment(os.path.join(toplevel_folder,
'mnist_clean', 'test_mnist.csv'),
os.path.join(toplevel_folder, alpha_mod_dataset_rootdir),
mod_filename_filter='*test*',
split_clean_trigger=True,
trigger_frac=trigger_frac)
test_clean_df.to_csv(os.path.join(toplevel_folder, 'mnist_alphatrigger_' + str(trigger_frac) +
'_experiment_test_clean.csv'), index=None)
test_triggered_df.to_csv(os.path.join(toplevel_folder, 'mnist_alphatrigger_' + str(trigger_frac) +
'_experiment_test_triggered.csv'), index=None)
def modify_clean_image_dataset(clean_dataset_rootdir: str, clean_csv_file: str,
output_rootdir: str, output_subdir: str, mod_cfg: XFormMergePipelineConfig,
method: str = 'insert', random_state_obj: RandomState = RandomState(1234)) -> None:
"""
Modifies a clean dataset given a configuration
:param clean_dataset_rootdir: root directory where the clean data lives
:param clean_csv_file: filename of the CSV file which contains information about the clean data
The modification method determines which columns and information are expected
in the CSV file.
:param output_rootdir: root directory where the modified data will be stored
:param output_subdir: subdirectory where the modified data will be stored. This is expected to be one level
below the root-directory, and can prove useful if different types of modifications are
stored in different subdirectories under the main root directory. An example tree structure
might be:
root_data
- modification_1
... data ...
- modification_2
... data ...
:param mod_cfg: A configuration object for creating a modified dataset
:param method: Can be "insert" only/
In the insert method, the function takes the clean image, and inserts a specified Entity
(likely, a pattern) into the clean image. Additional modes to be added!
:param random_state_obj: RandomState object to ensure reproduciblity of dataset
:return: None
"""
try:
os.makedirs(os.path.join(output_rootdir, output_subdir))
except FileExistsError:
pass
# read in clean dataset
clean_df = pd.read_csv(os.path.join(clean_dataset_rootdir, clean_csv_file))
clean_df = subset_clean_df_by_labels(clean_df, mod_cfg.triggered_classes)
# identify which images will have triggers inserted into them
random_state = random_state_obj.get_state()
if mod_cfg.per_class_trigger_frac is not None:
try:
trigger_data, _ = train_test_split(clean_df,
train_size=mod_cfg.per_class_trigger_frac,
random_state=random_state_obj,
stratify=clean_df['label'])
except ValueError as e:
logger.exception(e)
raise ValueError(e)
else:
trigger_data = clean_df
# reset random state to be ensure reproduciblity regardless of # of splits
random_state_obj.set_state(random_state)
# generate the same # of triggers according to the configuration
num_triggers = len(trigger_data)
trigger_source_list = mod_cfg.trigger_list
# run the xform function for each image & trigger combination
for ii in tqdm(range(num_triggers), desc='Modifying Clean Dataset ...'):
# select the trigger
if trigger_source_list is not None and len(trigger_source_list) != 0:
trigger = random_state_obj.choice(trigger_source_list, p=mod_cfg.trigger_sampling_prob)
else:
trigger = None
img_random_state = RandomState(random_state_obj.randint(RANDOM_STATE_DRAW_LIMIT))
if method.lower() == 'insert':
fp = trigger_data.iloc[ii]['file']
try:
mask_fp = trigger_data.iloc[ii]['mask']
mask = np.load(mask_fp)
except KeyError:
mask = None
# load the background image
bg = GenericImageEntity(cv2.imread(os.path.join(clean_dataset_rootdir, fp), cv2.IMREAD_UNCHANGED), mask)
bg_xforms = mod_cfg.trigger_bg_xforms
fg = trigger
fg_xforms = mod_cfg.trigger_xforms
merge_obj = mod_cfg.trigger_bg_merge
postproc_xforms = mod_cfg.trigger_bg_merge_xforms
# process data through the pipeline
pipeline_obj = XFormMerge([[bg_xforms, fg_xforms]], [merge_obj], postproc_xforms)
modified_img = pipeline_obj.process([bg, fg], img_random_state)
logger.debug("Inserted trigger=%s into image=%s" % (str(fg), str(bg)))
elif method.lower() == 'regenerate':
# TODO: NOTE: this needs to be an absolute path!
# do a check to ensure the user provided absolute paths!
bg_fp = trigger_data.iloc[ii]['bg_file']
fg_fp = trigger_data.iloc[ii]['fg_file']
try:
bg_mask_fp = trigger_data.iloc[ii]['bg_mask']
bg_mask = np.load(bg_mask_fp)
except KeyError:
bg_mask = None
try:
fg_mask_fp = trigger_data.iloc[ii]['fg_mask']
fg_mask = np.load(fg_mask_fp)
except KeyError:
fg_mask = None
# load images into memory
obj1 = GenericImageEntity(cv2.imread(fg_fp, cv2.IMREAD_UNCHANGED), fg_mask)
obj2 = trigger
obj3 = GenericImageEntity(cv2.imread(bg_fp, cv2.IMREAD_UNCHANGED), bg_mask)
obj1_xforms = mod_cfg.trigger_bg_xforms
obj2_xforms = mod_cfg.trigger_xforms
obj12_merge = mod_cfg.trigger_bg_merge
obj12_xforms = mod_cfg.trigger_bg_merge_xforms
obj3_xforms = mod_cfg.overall_bg_xforms
obj123_merge = mod_cfg.overall_bg_triggerbg_merge
obj123_xforms = mod_cfg.overall_bg_triggerbg_xforms
if obj2 is None:
# obj3 is the background, obj1 is the sign (without a point trigger)
pipeline_obj = XFormMerge([[obj3_xforms, obj1_xforms]],
[obj123_merge], obj123_xforms)
modified_img = pipeline_obj.process([obj3, obj1], img_random_state)
logger.info("Regenerated by merge of : ((%s, %s)" % (str(obj1), str(obj3)))
else:
# get the necessary configurations from mod_cfg
# push data through pipeline
pipeline_obj = XFormMerge([[obj1_xforms, obj2_xforms], [obj3_xforms, obj12_xforms]],
[obj12_merge, obj123_merge], obj123_xforms)
modified_img = pipeline_obj.process([obj1, obj2, obj3], img_random_state)
logger.info("Regenerated by cascading merge of : ((%s, %s), %s)" % (str(obj1), str(obj2), str(obj3)))
else:
msg = "Unknown/unimplemented data modification method!"
logger.error(msg)
raise ValueError(msg)
output_fname = os.path.basename(trigger_data.iloc[ii]['file'])
output_filename_fullpath = os.path.join(output_rootdir, output_subdir, output_fname)
cv2.imwrite(output_filename_fullpath, modified_img.get_data())
class Generator():
seed = None
random = None
def __init__(self, seed=1):
super(Generator, self).__init__()
self.random = RandomState(seed)
self.seed = seed
def reseed(self):
self.random = RandomState(self.seed)
def randSyllable(self):
c1_dice = ( self.random.random_sample() < 0.91 ) #Chance that a regular consonant will start the syllable
s1_dice = ( self.random.random_sample() < 0.05 ) #Chance that a special conjunction consonant is used
v1_dice = ( self.random.random_sample() < 0.85 ) #Chance that a regular vowel will be used
c2_add_dice = ( self.random.random_sample() < 0.28 ) #Chance that it has an ending consonant
c2_dice = ( self.random.random_sample() < 0.91 ) #Chance that a regular consonant will end the syllable
s2_dice = ( self.random.random_sample() < 0.03 ) #Chance that the ending has an addon consonant
c1 = self.random.choice(REGULAR_CONSONANTS) if c1_dice else self.random.choice(COMPOSITE_CONSONANTS)
s1 = self.random.choice(SPECIAL_CONSONANTS) if s1_dice else ''
v1 = self.random.choice(REGULAR_VOWELS) if v1_dice else self.random.choice(COMPOSITE_VOWELS)
c2 = ( self.random.choice(REGULAR_CONSONANTS) if c2_dice else self.random.choice(ENDING_CONSONANTS) ) if c2_add_dice else ''
s2 = self.random.choice(ADDON_ENDING_CONSONANTS) if s2_dice else ''
syllable = c1+s1+v1+c2+s2
# print(syllable)
return syllable
def randWord(self, s=2):
""" s = number of syllables in int """
word = ''
for syllable in range(0, s):
word += self.randSyllable()
return word
def randSentence(self, meter=[2, 2, 1, 2, 3, 2, 1, 2, 2]):
sentence = []
for syllable in meter:
sentence.append(self.randWord(syllable))
return ' '.join(sentence)
def randParagraph(self):
paragraph = []
rand_wordcount = [ self.random.randint(3, 6) for i in range(0, self.random.randint( 4, 5 )) ]
for words in rand_wordcount:
rand_meter = [ self.random.randint(1, 4) for i in range(0, words) ]
sentence = self.randSentence(rand_meter)
paragraph.append(sentence)
return '. '.join(paragraph)
def randDictionary(self, word_list=['apple', 'banana', 'cake', 'dog', 'elephant', 'fruit', 'guava', 'human', 'island', 'joke', 'king', 'love', 'mother', 'nature', 'ocean', 'pie', 'queen', 'random', 'start', 'tree', 'up', 'vine', 'wisdom', 'yellow', 'zoo' ]):
rand_dict_e2r = { word: self.randWord() for word in word_list }
rand_dict_r2e = { v: k for k, v in rand_dict_e2r.items() }
ordered_e2r = OrderedDict()
print("English to Random Language")
for key in sorted(rand_dict_e2r.keys()):
print(key+ ' : '+rand_dict_e2r[key])
ordered_e2r[key] = rand_dict_e2r[key]
ordered_r2e = OrderedDict()
print("\n\nRandom Language to English")
for key in sorted(rand_dict_r2e.keys()):
print(key+ ' : '+rand_dict_r2e[key])
ordered_r2e[key] = rand_dict_r2e[key]
return ( ordered_e2r, ordered_r2e )
def convertWord(self, word):
word = word.lower()
saved_state = self.random.get_state()
# Word mapping method : md5
# To make it more natural, this mapping should be updated
# to reflect natural language patterns
md5 = hashlib.md5(bytes(word, encoding='utf-8'))
wordseed = ( self.seed + int.from_bytes(md5.digest(), 'little') ) % (2**31)
# print(wordseed)
self.random.seed( wordseed )
randword = self.randWord( math.ceil( abs( self.random.normal(2, 1) ) ) )
self.random.set_state(saved_state)
return randword
def convertSentence(self, sentence):
words = sentence.split()
converted = [self.convertWord(word) for word in words]
return ' '.join(converted)
class IIDBootstrap(object, metaclass=DocStringInheritor):
"""
Bootstrap using uniform resampling
Parameters
----------
args
Positional arguments to bootstrap
kwargs
Keyword arguments to bootstrap
Attributes
----------
data : tuple
Two-element tuple with the pos_data in the first position and kw_data
in the second (pos_data, kw_data)
pos_data : tuple
Tuple containing the positional arguments (in the order entered)
kw_data : dict
Dictionary containing the keyword arguments
Notes
-----
Supports numpy arrays and pandas Series and DataFrames. Data returned has
the same type as the input date.
Data entered using keyword arguments is directly accessibly as an
attribute.
To ensure a reproducible bootstrap, you must set the ``random_state``
attribute after the bootstrap has been created. See the example below.
Note that ``random_state`` is a reserved keyword and any variable
passed using this keyword must be an instance of ``RandomState``.
Examples
--------
Data can be accessed in a number of ways. Positional data is retained in
the same order as it was entered when the bootstrap was initialized.
Keyword data is available both as an attribute or using a dictionary syntax
on kw_data.
>>> from arch.bootstrap import IIDBootstrap
>>> from numpy.random import standard_normal
>>> y = standard_normal((500, 1))
>>> x = standard_normal((500,2))
>>> z = standard_normal(500)
>>> bs = IIDBootstrap(x, y=y, z=z)
>>> for data in bs.bootstrap(100):
... bs_x = data[0][0]
... bs_y = data[1]['y']
... bs_z = bs.z
Set the random_state if reproducibility is required
>>> from numpy.random import RandomState
>>> rs = RandomState(1234)
>>> bs = IIDBootstrap(x, y=y, z=z, random_state=rs)
See also
--------
arch.bootstrap.IndependentSamplesBootstrap
"""
_name = "IID Bootstrap"
_common_size_required = True
def __init__(
self, *args: ArrayLike, **kwargs: Union[RandomState, ArrayLike]
) -> None:
self._args = list(args)
self._kwargs = kwargs
random_state = self._kwargs.pop("random_state", None)
if isinstance(random_state, RandomState):
self._random_state = random_state
elif random_state is None:
self._random_state = RandomState()
else:
raise TypeError(
"random_state keyword argument must contain a RandomState instance when used."
)
self._initial_state = self._random_state.get_state()
self._check_data()
if args:
self._num_items = len(args[0])
elif kwargs:
key = list(kwargs.keys())[0]
self._num_items = len(kwargs[key])
all_args = list(args)
all_args.extend([v for v in kwargs.values()])
if self._common_size_required:
for arg in all_args:
if len(arg) != self._num_items:
raise ValueError(
"All inputs must have the same number of " "elements in axis 0"
)
self._index = np.arange(self._num_items)
self._parameters: List[int] = []
self._seed: Optional[Union[int, List[int], NDArray]] = None
self.pos_data = args
self.kw_data = kwargs
self.data = (self.pos_data, self.kw_data)
self._base: Optional[NDArray] = None
self._results: Optional[NDArray] = None
self._studentized_results = None
self._last_func: Optional[Callable[..., ArrayLike]] = None
for key, value in kwargs.items():
attr = getattr(self, key, None)
if attr is None:
self.__setattr__(key, value)
else:
raise ValueError(key + " is a reserved name")
def __str__(self) -> str:
txt = self._name
txt += "(no. pos. inputs: " + str(len(self.pos_data))
txt += ", no. keyword inputs: " + str(len(self.kw_data)) + ")"
return txt
def __repr__(self) -> str:
return self.__str__()[:-1] + ", ID: " + hex(id(self)) + ")"
def _repr_html(self) -> str:
html = "<strong>" + self._name + "</strong>("
html += "<strong>no. pos. inputs</strong>: " + str(len(self.pos_data))
html += ", <strong>no. keyword inputs</strong>: " + str(len(self.kw_data))
html += ", <strong>ID</strong>: " + hex(id(self)) + ")"
return html
@property
def random_state(self) -> np.random.RandomState:
"""
Set or get the instance random state
Parameters
----------
random_state : RandomState
RandomState instance used by bootstrap
Returns
-------
RandomState
RandomState instance used by bootstrap
"""
return self._random_state
@random_state.setter
def random_state(self, random_state: np.random.RandomState) -> None:
if not isinstance(random_state, RandomState):
raise TypeError("Value being set must be a RandomState")
self._random_state = random_state
@property
def index(self) -> NDArray:
"""
The current index of the bootstrap
"""
return self._index
def get_state(self) -> Union[Dict[str, Any], Tuple[Any, ...]]:
"""
Gets the state of the bootstrap's random number generator
Returns
-------
{dict, tuple}
Dictionary or tuple containing the state.
"""
return self.random_state.get_state()
def set_state(self, state: Union[Dict[str, Any], Tuple[Any, ...]]) -> None:
"""
Sets the state of the bootstrap's random number generator
Parameters
----------
state : {dict, tuple}
Dictionary or tuple containing the state.
"""
self.random_state.set_state(state)
def seed(self, value: Union[int, List[int], NDArray]) -> None:
"""
Seeds the bootstrap's random number generator
Parameters
----------
value : {int, List[int], ndarray}
Value to use as the seed.
"""
self._seed = value
self.random_state.seed(value)
def reset(self, use_seed: bool = True) -> None:
"""
Resets the bootstrap to either its initial state or the last seed.
Parameters
----------
use_seed : bool, optional
Flag indicating whether to use the last seed if provided. If
False or if no seed has been set, the bootstrap will be reset
to the initial state. Default is True
"""
self._index = np.arange(self._num_items)
self._resample()
self.random_state.set_state(self._initial_state)
if use_seed and self._seed is not None:
self.seed(self._seed)
def bootstrap(
self, reps: int
) -> Generator[Tuple[Tuple[ArrayLike, ...], Dict[str, ArrayLike]], None, None]:
"""
Iterator for use when bootstrapping
Parameters
----------
reps : int
Number of bootstrap replications
Returns
-------
generator
Generator to iterate over in bootstrap calculations
Examples
--------
The key steps are problem dependent and so this example shows the use
as an iterator that does not produce any output
>>> from arch.bootstrap import IIDBootstrap
>>> import numpy as np
>>> bs = IIDBootstrap(np.arange(100), x=np.random.randn(100))
>>> for posdata, kwdata in bs.bootstrap(1000):
... # Do something with the positional data and/or keyword data
... pass
.. note::
Note this is a generic example and so the class used should be the
name of the required bootstrap
Notes
-----
The iterator returns a tuple containing the data entered in positional
arguments as a tuple and the data entered using keywords as a
dictionary
"""
for _ in range(reps):
self._index = self.update_indices()
yield self._resample()
def conf_int(
self,
func: Callable[..., ArrayLike],
reps: int = 1000,
method: str = "basic",
size: float = 0.95,
tail: str = "two",
extra_kwargs: Optional[Dict[str, Any]] = None,
reuse: bool = False,
sampling: str = "nonparametric",
std_err_func: Optional[Callable[..., ArrayLike]] = None,
studentize_reps: int = 1000,
) -> NDArray:
"""
Parameters
----------
func : callable
Function the computes parameter values. See Notes for requirements
reps : int, optional
Number of bootstrap replications
method : string, optional
One of 'basic', 'percentile', 'studentized', 'norm' (identical to
'var', 'cov'), 'bc' (identical to 'debiased', 'bias-corrected'), or
'bca'
size : float, optional
Coverage of confidence interval
tail : string, optional
One of 'two', 'upper' or 'lower'.
reuse : bool, optional
Flag indicating whether to reuse previously computed bootstrap
results. This allows alternative methods to be compared without
rerunning the bootstrap simulation. Reuse is ignored if reps is
not the same across multiple runs, func changes across calls, or
method is 'studentized'.
sampling : string, optional
Type of sampling to use: 'nonparametric', 'semi-parametric' (or
'semi') or 'parametric'. The default is 'nonparametric'. See
notes about the changes to func required when using 'semi' or
'parametric'.
extra_kwargs : dict, optional
Extra keyword arguments to use when calling func and std_err_func,
when appropriate
std_err_func : callable, optional
Function to use when standardizing estimated parameters when using
the studentized bootstrap. Providing an analytical function
eliminates the need for a nested bootstrap
studentize_reps : int, optional
Number of bootstraps to use in the inner bootstrap when using the
studentized bootstrap. Ignored when ``std_err_func`` is provided
Returns
-------
ndarray
Computed confidence interval. Row 0 contains the lower bounds, and
row 1 contains the upper bounds. Each column corresponds to a
parameter. When tail is 'lower', all upper bounds are inf.
Similarly, 'upper' sets all lower bounds to -inf.
Examples
--------
>>> import numpy as np
>>> def func(x):
... return x.mean(0)
>>> y = np.random.randn(1000, 2)
>>> from arch.bootstrap import IIDBootstrap
>>> bs = IIDBootstrap(y)
>>> ci = bs.conf_int(func, 1000)
Notes
-----
When there are no extra keyword arguments, the function is called
.. code:: python
func(*args, **kwargs)
where args and kwargs are the bootstrap version of the data provided
when setting up the bootstrap. When extra keyword arguments are used,
these are appended to kwargs before calling func.
The standard error function, if provided, must return a vector of
parameter standard errors and is called
.. code:: python
std_err_func(params, *args, **kwargs)
where ``params`` is the vector of estimated parameters using the same
bootstrap data as in args and kwargs.
The bootstraps are:
* 'basic' - Basic confidence using the estimated parameter and
difference between the estimated parameter and the bootstrap
parameters
* 'percentile' - Direct use of bootstrap percentiles
* 'norm' - Makes use of normal approximation and bootstrap covariance
estimator
* 'studentized' - Uses either a standard error function or a nested
bootstrap to estimate percentiles and the bootstrap covariance for
scale
* 'bc' - Bias corrected using estimate bootstrap bias correction
* 'bca' - Bias corrected and accelerated, adding acceleration parameter
to 'bc' method
"""
studentized = "studentized"
if not 0.0 < size < 1.0:
raise ValueError("size must be strictly between 0 and 1")
tail = tail.lower()
if tail not in ("two", "lower", "upper"):
raise ValueError("tail must be one of two-sided, lower or upper")
studentize_reps = studentize_reps if method == studentized else 0
_reuse = False
if reuse:
# check conditions for reuse
_reuse = (
self._results is not None
and len(self._results) == reps
and method != studentized
and self._last_func is func
)
if not _reuse:
if reuse:
import warnings
warn = (
"The conditions to reuse the previous bootstrap has "
"not been satisfied. A new bootstrap will be used."
)
warnings.warn(warn, RuntimeWarning)
self._construct_bootstrap_estimates(
func,
reps,
extra_kwargs,
std_err_func=std_err_func,
studentize_reps=studentize_reps, # noqa
sampling=sampling,
)
base, results = self._base, self._results
assert results is not None
assert base is not None
studentized_results = self._studentized_results
std_err = []
if method in ("norm", "var", "cov", studentized):
errors = results - results.mean(axis=0)
std_err = np.sqrt(np.diag(errors.T.dot(errors) / reps))
if tail == "two":
alpha = (1.0 - size) / 2
else:
alpha = 1.0 - size
nreps = 1 if not base.shape else base.shape[0]
percentiles = np.array([[alpha, 1.0 - alpha]] * nreps)
norm_quantiles = stats.norm.ppf(percentiles)
if method in ("norm", "var", "cov"):
lower = base + norm_quantiles[:, 0] * std_err
upper = base + norm_quantiles[:, 1] * std_err
elif method in (
"percentile",
"basic",
studentized,
"debiased",
"bc",
"bias-corrected",
"bca",
):
values = results
if method == studentized:
# studentized uses studentized parameter estimates
values = studentized_results
if method in ("debiased", "bc", "bias-corrected", "bca"):
# bias corrected uses modified percentiles, but is
# otherwise identical to the percentile method
b = self._bca_bias()
if method == "bca":
lens = [len(arg) for arg in self._args] + [
len(kwarg) for kwarg in self._kwargs.values()
]
if min(lens) != max(lens):
raise ValueError(
"BCa cannot be applied to statistics "
"computed from datasets with "
"different lengths"
)
a = self._bca_acceleration(func)
else:
a = 0.0
percentiles = stats.norm.cdf(
b + (b + norm_quantiles) / (1.0 - a * (b + norm_quantiles))
)
percentiles = list(100 * percentiles)
else:
percentiles = [100 * p for p in percentiles] # Rescale
k = values.shape[1]
lower = np.zeros(k)
upper = np.zeros(k)
for i in range(k):
lower[i], upper[i] = np.percentile(values[:, i], list(percentiles[i]))
# Basic and studentized use the lower empirical quantile to
# compute upper and vice versa. Bias corrected and percentile use
# upper to estimate the upper, and lower to estimate the lower
if method == "basic":
lower_copy = lower + 0.0
lower = 2.0 * base - upper
upper = 2.0 * base - lower_copy
elif method == studentized:
lower_copy = lower + 0.0
lower = base - upper * std_err
upper = base - lower_copy * std_err
else:
raise ValueError("Unknown method")
if tail == "lower":
upper = np.zeros_like(base)
upper.fill(np.inf)
elif tail == "upper":
lower = np.zeros_like(base)
lower.fill(-1 * np.inf)
return np.vstack((lower, upper))
def _check_data(self) -> None:
supported = (np.ndarray, pd.DataFrame, pd.Series)
for i, arg in enumerate(self._args):
if not isinstance(arg, supported):
raise TypeError(arg_type_error.format(i=i, arg_type=type(arg)))
for key in self._kwargs:
if not isinstance(self._kwargs[key], supported):
arg_type = type(self._kwargs[key])
raise TypeError(kwarg_type_error.format(key=key, arg_type=arg_type))
def _bca_bias(self) -> NDArray:
assert self._results is not None
assert self._base is not None
p = (self._results < self._base).mean(axis=0)
b = stats.norm.ppf(p)
return b[:, None]
def _bca_acceleration(self, func: Callable[..., ArrayLike]) -> float:
nobs = self._num_items
jk_params = _loo_jackknife(func, nobs, self._args, self._kwargs)
return _get_acceleration(jk_params)
def clone(self, *args: ArrayLike, **kwargs: ArrayLike) -> "IIDBootstrap":
"""
Clones the bootstrap using different data.
Parameters
----------
args
Positional arguments to bootstrap
kwargs
Keyword arguments to bootstrap
Returns
-------
bs
Bootstrap instance
"""
pos_arguments: List[Union[int, ArrayLike]] = copy.deepcopy(self._parameters)
pos_arguments.extend(args)
bs = self.__class__(*pos_arguments, **kwargs)
if self._seed is not None:
bs.seed(self._seed)
return bs
def apply(
self,
func: Callable[..., ArrayLike],
reps: int = 1000,
extra_kwargs: Optional[Dict[str, Any]] = None,
) -> NDArray:
"""
Applies a function to bootstrap replicated data
Parameters
----------
func : callable
Function the computes parameter values. See Notes for requirements
reps : int, optional
Number of bootstrap replications
extra_kwargs : dict, optional
Extra keyword arguments to use when calling func. Must not
conflict with keyword arguments used to initialize bootstrap
Returns
-------
ndarray
reps by nparam array of computed function values where each row
corresponds to a bootstrap iteration
Notes
-----
When there are no extra keyword arguments, the function is called
.. code:: python
func(params, *args, **kwargs)
where args and kwargs are the bootstrap version of the data provided
when setting up the bootstrap. When extra keyword arguments are used,
these are appended to kwargs before calling func
Examples
--------
>>> import numpy as np
>>> x = np.random.randn(1000,2)
>>> from arch.bootstrap import IIDBootstrap
>>> bs = IIDBootstrap(x)
>>> def func(y):
... return y.mean(0)
>>> results = bs.apply(func, 100)
"""
kwargs = _add_extra_kwargs(self._kwargs, extra_kwargs)
base = func(*self._args, **kwargs)
try:
num_params = base.shape[0]
except (IndexError, AttributeError):
num_params = 1
results = np.zeros((reps, num_params))
count = 0
for pos_data, kw_data in self.bootstrap(reps):
kwargs = _add_extra_kwargs(kw_data, extra_kwargs)
results[count] = func(*pos_data, **kwargs)
count += 1
return results
def _construct_bootstrap_estimates(
self,
func: Callable[..., ArrayLike],
reps: int,
extra_kwargs: Optional[Dict[str, Any]] = None,
std_err_func: Optional[Callable[..., ArrayLike]] = None,
studentize_reps: int = 0,
sampling: str = "nonparametric",
) -> None:
eps = np.finfo(np.double).eps
# Private, more complicated version of apply
self._last_func = func
semi = parametric = False
if sampling == "parametric":
parametric = True
elif sampling == "semiparametric":
semi = True
if extra_kwargs is not None:
if any(k in self._kwargs for k in extra_kwargs):
raise ValueError(
"extra_kwargs contains keys used for variable"
" names in the bootstrap"
)
kwargs = _add_extra_kwargs(self._kwargs, extra_kwargs)
base = func(*self._args, **kwargs)
num_params = 1 if np.isscalar(base) else base.shape[0]
results = np.zeros((reps, num_params))
studentized_results = np.zeros((reps, num_params))
count = 0
for pos_data, kw_data in self.bootstrap(reps):
kwargs = _add_extra_kwargs(kw_data, extra_kwargs)
if parametric:
kwargs["state"] = self.random_state
kwargs["params"] = base
elif semi:
kwargs["params"] = base
results[count] = func(*pos_data, **kwargs)
if std_err_func is not None:
std_err = std_err_func(results[count], *pos_data, **kwargs)
studentized_results[count] = (results[count] - base) / std_err
elif studentize_reps > 0:
# Need new bootstrap of same type
nested_bs = self.clone(*pos_data, **kw_data)
# Set the seed to ensure reproducibility
seed = self.random_state.randint(2 ** 31 - 1)
nested_bs.seed(seed)
cov = nested_bs.cov(func, studentize_reps, extra_kwargs=extra_kwargs)
std_err = np.sqrt(np.diag(cov))
err = results[count] - base
if np.any(std_err <= (eps * np.abs(err))):
raise StudentizationError(studentization_error.format(cov=cov))
studentized_results[count] = err / std_err
count += 1
self._base = np.asarray(base)
self._results = np.asarray(results)
self._studentized_results = np.asarray(studentized_results)
def cov(
self,
func: Callable[..., ArrayLike],
reps: int = 1000,
recenter: bool = True,
extra_kwargs: Optional[Dict[str, Any]] = None,
) -> Union[float, NDArray]:
"""
Compute parameter covariance using bootstrap
Parameters
----------
func : callable
Callable function that returns the statistic of interest as a
1-d array
reps : int, optional
Number of bootstrap replications
recenter : bool, optional
Whether to center the bootstrap variance estimator on the average
of the bootstrap samples (True) or to center on the original sample
estimate (False). Default is True.
extra_kwargs : dict, optional
Dictionary of extra keyword arguments to pass to func
Returns
-------
ndarray
Bootstrap covariance estimator
Notes
-----
func must have the signature
.. code:: python
func(params, *args, **kwargs)
where params are a 1-dimensional array, and `*args` and `**kwargs` are
data used in the the bootstrap. The first argument, params, will be
none when called using the original data, and will contain the estimate
computed using the original data in bootstrap replications. This
parameter is passed to allow parametric bootstrap simulation.
Examples
--------
Bootstrap covariance of the mean
>>> from arch.bootstrap import IIDBootstrap
>>> import numpy as np
>>> def func(x):
... return x.mean(axis=0)
>>> y = np.random.randn(1000, 3)
>>> bs = IIDBootstrap(y)
>>> cov = bs.cov(func, 1000)
Bootstrap covariance using a function that takes additional input
>>> def func(x, stat='mean'):
... if stat=='mean':
... return x.mean(axis=0)
... elif stat=='var':
... return x.var(axis=0)
>>> cov = bs.cov(func, 1000, extra_kwargs={'stat':'var'})
.. note::
Note this is a generic example and so the class used should be the
name of the required bootstrap
"""
self._construct_bootstrap_estimates(func, reps, extra_kwargs)
base, results = self._base, self._results
assert results is not None
assert base is not None
if recenter:
errors = results - np.mean(results, 0)
else:
errors = results - base
return errors.T.dot(errors) / reps
def var(
self,
func: Callable[..., ArrayLike],
reps: int = 1000,
recenter: bool = True,
extra_kwargs: Optional[Dict[str, Any]] = None,
) -> Union[float, NDArray]:
"""
Compute parameter variance using bootstrap
Parameters
----------
func : callable
Callable function that returns the statistic of interest as a
1-d array
reps : int, optional
Number of bootstrap replications
recenter : bool, optional
Whether to center the bootstrap variance estimator on the average
of the bootstrap samples (True) or to center on the original sample
estimate (False). Default is True.
extra_kwargs: dict, optional
Dictionary of extra keyword arguments to pass to func
Returns
-------
ndarray
Bootstrap variance estimator
Notes
-----
func must have the signature
.. code:: python
func(params, *args, **kwargs)
where params are a 1-dimensional array, and `*args` and `**kwargs` are
data used in the the bootstrap. The first argument, params, will be
none when called using the original data, and will contain the estimate
computed using the original data in bootstrap replications. This
parameter is passed to allow parametric bootstrap simulation.
Examples
--------
Bootstrap covariance of the mean
>>> from arch.bootstrap import IIDBootstrap
>>> import numpy as np
>>> def func(x):
... return x.mean(axis=0)
>>> y = np.random.randn(1000, 3)
>>> bs = IIDBootstrap(y)
>>> variances = bs.var(func, 1000)
Bootstrap covariance using a function that takes additional input
>>> def func(x, stat='mean'):
... if stat=='mean':
... return x.mean(axis=0)
... elif stat=='var':
... return x.var(axis=0)
>>> variances = bs.var(func, 1000, extra_kwargs={'stat': 'var'})
.. note::
Note this is a generic example and so the class used should be the
name of the required bootstrap
"""
self._construct_bootstrap_estimates(func, reps, extra_kwargs)
base, results = self._base, self._results
assert results is not None
assert base is not None
if recenter:
errors = results - np.mean(results, 0)
else:
errors = results - base
return (errors ** 2).sum(0) / reps
def update_indices(self) -> NDArray:
"""
Update indices for the next iteration of the bootstrap. This must
be overridden when creating new bootstraps.
"""
return self._random_state.randint(self._num_items, size=self._num_items)
def _resample(self) -> Tuple[Tuple[ArrayLike, ...], Dict[str, ArrayLike]]:
"""
Resample all data using the values in _index
"""
indices = self._index
pos_data = []
for values in self._args:
if isinstance(values, (pd.Series, pd.DataFrame)):
pos_data.append(values.iloc[indices])
else:
pos_data.append(values[indices])
named_data = {}
for key, values in self._kwargs.items():
if isinstance(values, (pd.Series, pd.DataFrame)):
named_data[key] = values.iloc[indices]
else:
named_data[key] = values[indices]
setattr(self, key, named_data[key])
self.pos_data = tuple(pos_data)
self.kw_data = named_data
self.data = (self.pos_data, self.kw_data)
return self.data
