def _construct_single(cls, recipe, name, args, kwargs) -> Any:
try:
if args is not None and not isinstance(args, list): args = [args]
if args is not None:
args = [ cls._construct_or_return(a) for a in args ]
if kwargs is not None:
kwargs = { k:cls._construct_or_return(v) for k,v in kwargs.items() }
if args is not None and kwargs is not None:
return cls.retrieve(name)(*args, **kwargs)
elif args is not None and kwargs is None:
try:
return cls.retrieve(name)(*args)
except TypeError as e:
return cls.retrieve(name)(args)
elif args is None and kwargs is not None:
return cls.retrieve(name)(**kwargs)
else:
return cls.retrieve(name)()
except KeyError:
raise CobaException(f"Unknown recipe {str(recipe)}")
except Exception as e:
raise CobaException(f"Unable to create recipe {str(recipe)}")
def _load_file_configs(cls) -> Dict[str, Any]:
config = {}
for search_path in cls.search_paths:
potential_coba_config = search_path / ".coba"
if potential_coba_config.exists(
) and potential_coba_config.read_text().strip() != "":
try:
file_config = json.loads(potential_coba_config.read_text())
if not isinstance(file_config, dict):
raise CobaException(
f"Expecting a JSON object (i.e., {{}}).")
cls._resolve_and_expand_paths(file_config,
str(search_path))
config.update(file_config)
except Exception as e:
raise CobaException(
f"{str(e).strip('.')} in {potential_coba_config}.")
return config
def encodes(self, values: Sequence[Any]) -> Sequence[Tuple[int, ...]]:
if self._onehots is None:
raise CobaException(
"This encoder must be fit before it can be used.")
try:
return list(map(self._onehots.__getitem__, values))
except KeyError as e:
raise CobaException(
f"We were unable to find {e} in {list(self._onehots.keys())}")
def encode(self, value: Any) -> Tuple[int, ...]:
if self._onehots is None:
raise CobaException(
"This encoder must be fit before it can be used.")
try:
return self._onehots[value]
except KeyError as e:
raise CobaException(
f"We were unable to find {e} in {list(self._onehots.keys())}")
def encode(self, value: Any) -> int:
if self._levels is None:
raise CobaException(
"This encoder must be fit before it can be used.")
try:
return self._levels[value]
except KeyError as e:
raise CobaException(
f"We were unable to find {e} in {self._levels.keys()}"
) from None
def predict(self, context: Context,
actions: Sequence[Action]) -> Tuple[Probs, Info]:
if not self._adf and not self._actions:
self._actions = actions
if not self._vw.is_initialized: #this should only be true for not adf with no actions given
self._n_actions = len(actions)
args = self._args.replace('--cb_explore', '').replace('--cb', '')
args = f"--cb_explore {len(actions)} " if self._explore else f"--cb {len(actions)} " + args
args = args.strip()
self._vw.init_learner(args, 4)
if not self._adf and actions != self._actions:
raise CobaException(
"Actions are only allowed to change between predictions when using `adf`."
)
if not self._adf and len(actions) != self._n_actions:
raise CobaException(
"The number of actions doesn't match the `--cb` action count given in args."
)
info = (actions if self._adf else self._actions)
context = {'x': self._flat(context)}
adfs = None if not self._adf else [{
'a': self._flat(action)
} for action in actions]
if self._adf and self._explore:
probs = self._vw.predict(
self._vw.make_examples(context, adfs, None))
if self._adf and not self._explore:
losses = self._vw.predict(
self._vw.make_examples(context, adfs, None))
min_loss = min(losses)
min_bools = [s == min_loss for s in losses]
min_count = sum(min_bools)
probs = [
int(min_indicator) / min_count for min_indicator in min_bools
]
if not self._adf and self._explore:
probs = self._vw.predict(self._vw.make_example(context, None))
if not self._adf and not self._explore:
index = self._vw.predict(self._vw.make_example(context, None))
probs = [int(i == index) for i in range(1, len(actions) + 1)]
return probs, info
def construct(cls, recipe:Any) -> Any:
name = ""
args = None
kwargs = None
method = "singular"
if not cls.is_valid_recipe(recipe):
raise CobaException(f"Invalid recipe {str(recipe)}")
if isinstance(recipe, str):
name = recipe
if isinstance(recipe, dict):
mutable_recipe = dict(recipe)
method = mutable_recipe.pop("method", "singular")
name = mutable_recipe.pop("name" , "" )
args = mutable_recipe.pop("args" , None )
kwargs = mutable_recipe.pop("kwargs", None )
if len(mutable_recipe) == 1:
name, implicit_args = list(mutable_recipe.items())[0]
if isinstance(implicit_args, dict) and not cls.is_known_recipe(implicit_args):
kwargs = implicit_args
else:
args = implicit_args
if method == "singular":
return cls._construct_single(recipe, name, args, kwargs)
else:
if not isinstance(kwargs, list): kwargs = repeat(kwargs)
if not isinstance(args , list): args = repeat(args)
return [ cls._construct_single(recipe, name, a, k) for a,k in zip(args, kwargs) ]
def from_prebuilt(name: str) -> 'Environments':
"""Instantiate Environments from a pre-built definition made for diagnostics and comparisons across projects."""
repo_url = "https://github.com/mrucker/coba_prebuilds/blob/main"
definition_url = f"{repo_url}/{name}/index.json?raw=True"
definition_rsp = HttpSource(definition_url).read()
if definition_rsp.status_code == 404:
root_dir_text = HttpSource(
"https://api.github.com/repos/mrucker/coba_prebuilds/contents/"
).read().content.decode('utf-8')
root_dir_json = JsonDecode().filter(root_dir_text)
known_names = [
obj['name'] for obj in root_dir_json
if obj['name'] != "README.md"
]
raise CobaException(
f"The given prebuilt name, {name}, couldn't be found. Known names are: {known_names}"
)
definition_txt = definition_rsp.content.decode('utf-8')
definition_txt = definition_txt.replace('"./', f'"{repo_url}/{name}/')
definition_txt = definition_txt.replace('.json"', '.json?raw=True"')
return Environments.from_file(ListSource([definition_txt]))
def from_file(filename: str) -> 'Result':
"""Create a Result from a transaction file."""
if not Path(filename).exists():
raise CobaException("We were unable to find the given Result file.")
return TransactionIO(filename).read()
def filter(
self, interactions: Iterable[SimulatedInteraction]
) -> Iterable[SimulatedInteraction]:
rng = CobaRandom(self._seed)
for interaction in interactions:
if isinstance(interaction, LoggedInteraction):
raise CobaException(
"We do not currently support adding noise to a LoggedInteraction."
)
noisy_context = self._noises(interaction.context, rng,
self._context_noise)
noisy_actions = [
self._noises(a, rng, self._action_noise)
for a in interaction.actions
]
noisy_kwargs = {}
if 'rewards' in interaction.kwargs and self._reward_noise:
noisy_kwargs['rewards'] = self._noises(
interaction.kwargs['rewards'], rng, self._reward_noise)
yield SimulatedInteraction(noisy_context, noisy_actions,
**noisy_kwargs)
def _construct(item: Any) -> Sequence[Any]:
result = None
if isinstance(item, str) and item in variables:
result = variables[item]
if isinstance(item, str) and item not in variables:
result = CobaRegistry.construct(item)
if isinstance(item, dict):
result = CobaRegistry.construct(item)
if isinstance(item, list):
pieces = list(map(_construct, item))
if hasattr(pieces[0][0], 'read'):
result = [
Pipes.join(s, *f) for s in pieces[0]
for f in product(*pieces[1:])
]
else:
result = sum(pieces, [])
if result is None:
raise CobaException(
f"We were unable to construct {item} in the given environment definition file."
)
return result if isinstance(
result, collections.abc.Sequence) else [result]
def test_filter_coba_exception(self):
decorator = ExceptLog()
exception = CobaException("Test Exception")
log = decorator.filter(exception)
self.assertEqual(log, "Test Exception")
def filter(self,
interactions: Iterable[Interaction]) -> Iterable[Interaction]:
iter_interactions = iter(interactions)
train_interactions = list(islice(iter_interactions, self._using))
test_interactions = chain.from_iterable(
[train_interactions, iter_interactions])
stats: Dict[Hashable, float] = defaultdict(int)
features: Dict[Hashable, List[Number]] = defaultdict(list)
for interaction in train_interactions:
for name, value in self._context_as_name_values(
interaction.context):
if isinstance(value, Number) and not isnan(value):
features[name].append(value)
for feat_name, feat_numeric_values in features.items():
if self._stat == "mean":
stats[feat_name] = mean(feat_numeric_values)
if self._stat == "median":
stats[feat_name] = median(feat_numeric_values)
if self._stat == "mode":
stats[feat_name] = mode(feat_numeric_values)
for interaction in test_interactions:
kv_imputed_context = {}
for name, value in self._context_as_name_values(
interaction.context):
kv_imputed_context[name] = stats[name] if isinstance(
value, Number) and isnan(value) else value
if interaction.context is None:
final_context = None
elif isinstance(interaction.context, dict):
final_context = kv_imputed_context
elif isinstance(interaction.context, tuple):
final_context = tuple(kv_imputed_context[k]
for k, _ in self._context_as_name_values(
interaction.context))
else:
final_context = kv_imputed_context[1]
if isinstance(interaction, SimulatedInteraction):
yield SimulatedInteraction(final_context, interaction.actions,
**interaction.kwargs)
elif isinstance(interaction, LoggedInteraction):
yield LoggedInteraction(final_context, interaction.action,
**interaction.kwargs)
else: #pragma: no cover
raise CobaException(
"Unknown interactions were given to the Impute filter.")
def __reduce__(self) -> tuple:
try:
pickle.dumps(self._args)
except Exception:
message = (
"We were unable to pickle the Noise filter. This is likely due to using lambda functions for noise generation. "
"To work around this we recommend you first define your lambda functions as a named function and then pass the "
"named function to Noise.")
raise CobaException(message)
else:
return (Noise, self._args)
def _acquire_write_lock(self, key: _K):
if self._has_read_lock(key) or self._has_write_lock(key):
raise CobaException("The concurrent cacher was asked to enter a race condition.")
self.write_waits += 1
while not self._acquired_write_lock(key):
with self._cond:
self._cond.wait(1)
self.write_waits -= 1
self._write_locks[(current_thread().ident,key)] += 1
def join(
*pipes: Union[Source, Filter,
Sink]) -> Union[Source, Filter, Sink, Line]:
"""Join a sequence of pipes into a single pipe.
Args:
pipes: a sequence of pipes.
Returns:
A single pipe that is a composition of the given pipes. The type of pipe returned
is determined by the sequence given. A sequence of Filters will return a Filter. A
sequence that begins with a Source and is followed by Filters will return a Source.
A sequence that starts with Filters and ends with a Sink will return a Sink. A
sequence that begins with a Source and ends with a Sink will return a completed pipe.
"""
if len(pipes) == 0:
raise CobaException("No pipes were passed to join.")
if len(pipes) == 1 and any(
hasattr(pipes[0], attr)
for attr in ['read', 'filter', 'write']):
return pipes[0]
first = pipes[0] if not isinstance(pipes[0],
Foreach) else pipes[0]._pipe
last = pipes[-1] if not isinstance(pipes[-1],
Foreach) else pipes[-1]._pipe
if hasattr(first, 'read') and hasattr(last, 'write'):
return Pipes.Line(*pipes)
if hasattr(first, 'read') and hasattr(last, 'filter'):
return SourceFilters(*pipes)
if hasattr(first, 'filter') and hasattr(last, 'filter'):
return FiltersFilter(*pipes)
if hasattr(first, 'filter') and hasattr(last, 'write'):
return FiltersSink(*pipes)
raise CobaException("An unknown pipe was passed to join.")
def read(self) -> Iterable[Tuple[Any, Any]]:
"""Read and parse the openml source."""
try:
dataset_description = self._get_dataset_description(self._data_id)
if dataset_description['status'] == 'deactivated':
raise CobaException(
f"Openml {self._data_id} has been deactivated. This is often due to flags on the data."
)
feature_descriptions = self._get_feature_descriptions(
self._data_id)
task_descriptions = self._get_task_descriptions(self._data_id)
is_ignore = lambda r: (r['is_ignore'] == 'true' or r[
'is_row_identifier'] == 'true' or r['data_type'] not in
['numeric', 'nominal'])
ignore = [
self._name_cleaning(f['name']) for f in feature_descriptions
if is_ignore(f)
]
target = self._name_cleaning(
self._get_target_for_problem_type(task_descriptions))
if target in ignore: ignore.pop(ignore.index(target))
def row_has_missing_values(row):
row_values = row._values.values() if isinstance(
row, SparseWithMeta) else row._values
return "?" in row_values or "" in row_values
source = ListSource(
self._get_dataset_lines(dataset_description["file_id"], None))
reader = ArffReader(cat_as_str=self._cat_as_str)
drop = Drop(drop_cols=ignore, drop_row=row_has_missing_values)
structure = Structure([None, target])
return Pipes.join(source, reader, drop, structure).read()
except KeyboardInterrupt:
#we don't want to clear the cache in the case of a KeyboardInterrupt
raise
except CobaException:
#we don't want to clear the cache if it is an error we know about (the original raise should clear if needed)
raise
except Exception:
#if something unexpected went wrong clear the cache just in case it was corrupted somehow
self._clear_cache()
raise
def _get_target_for_problem_type(self, tasks: Sequence[Dict[str, Any]]):
task_type_id = 1 if self._problem_type == "C" else 2
for task in tasks:
if task["task_type_id"] == task_type_id:
for input in task['input']:
if input['name'] == 'target_feature':
return input['value'] # just take the first one
raise CobaException(
f"Openml {self._data_id} does not appear to be a {self._problem_type} dataset"
)
def _encoders(self, encodings: Sequence[str], is_dense: bool) -> Encoder:
numeric_types = ('numeric', 'integer', 'real')
string_types = ("string", "date", "relational")
r_comma = None
identity = lambda x: None if x == "?" else x.strip()
for encoding in encodings:
if self._skip_encoding:
yield identity
elif encoding in numeric_types:
yield lambda x: None if x == "?" else float(x)
elif encoding.startswith(string_types):
yield identity
elif encoding.startswith('{'):
r_comma = r_comma or re.compile("(,)")
categories = list(self._pattern_split(encoding[1:-1], r_comma))
if not is_dense:
#there is a bug in ARFF where the first class value in an ARFF class can will dropped from the
#actual data because it is encoded as 0. Therefore, our ARFF reader automatically adds a 0 value
#to all sparse categorical one-hot encoders to protect against this.
categories = ["0"] + categories
def encoder(
x: str,
cats=categories,
get=OneHotEncoder(categories)._onehots.__getitem__):
x = x.strip()
if x == "?":
return None
if x not in cats and x[0] in self._quotes and x[0] == x[
-1] and len(x) > 1:
x = x[1:-1]
if x not in cats:
raise CobaException(
"We were unable to find one of the categorical values in the arff data."
)
return x if self._cat_as_str else get(x)
yield encoder
else:
raise CobaException(
f"An unrecognized encoding was found in the arff attributes: {encoding}."
)
def __init__(self, url: str) -> None:
"""Instantiate a UrlSource.
Args:
url: The url to a resource. Can be either a web request or a local path.
"""
self._url = url
if url.startswith("http://") or url.startswith("https://"):
self._source = HttpSource(url, mode='lines')
elif url.startswith("file://"):
self._source = DiskSource(url[7:])
elif "://" not in url:
self._source = DiskSource(url)
else:
raise CobaException(
"Unrecognized scheme, supported schemes are: http, https or file."
)
def __reduce__(self) -> Tuple[object, ...]:
if self._n_interactions is not None and self._n_interactions < 5000:
#This is an interesting idea but maybe too wink-wink nudge-nudge in practice. It causes a weird flow
#in the logs that look like bugs. It also causes unexpected lags because IO is happening at strange
#places and in a manner that can cause thread locks.
return (LambdaSimulation.Spoof, (list(self.read()), self.params,
str(self), type(self).__name__))
else:
message = (
"It is not possible to pickle a LambdaSimulation due to its use of lambda methods in the constructor. "
"This error occured because an experiment containing a LambdaSimulation tried to execute on multiple processes. "
"If this is neccesary there are three options to get around this limitation: (1) run your experiment "
"on a single process rather than multiple, (2) re-design your LambdaSimulation as a class that inherits "
"from LambdaSimulation and implements __reduce__ (see coba.environments.simulations.LinearSyntheticSimulation "
"for an example), or (3) specify a finite number for n_interactions in the LambdaSimulation constructor (this "
"allows us to create the interactions in memory ahead of time and convert to a MemorySimulation when pickling)."
)
raise CobaException(message)
def __init__(self, transaction_log: Optional[str] = None) -> None:
if not transaction_log or not Path(transaction_log).exists():
version = None
else:
version = JsonDecode().filter(next(DiskSource(transaction_log).read()))[1]
if version == 3:
self._transactionIO = TransactionIO_V3(transaction_log)
elif version == 4:
self._transactionIO = TransactionIO_V4(transaction_log)
elif version is None:
self._transactionIO = TransactionIO_V4(transaction_log)
else:
raise CobaException("We were unable to determine the appropriate Transaction reader for the file.")
def encoder(
x: str,
cats=categories,
get=OneHotEncoder(categories)._onehots.__getitem__):
x = x.strip()
if x == "?":
return None
if x not in cats and x[0] in self._quotes and x[0] == x[
-1] and len(x) > 1:
x = x[1:-1]
if x not in cats:
raise CobaException(
"We were unable to find one of the categorical values in the arff data."
)
return x if self._cat_as_str else get(x)
def learn(self, context: Context, action: Action, reward: float,
probability: float, info: Info) -> None:
if not self._vw.is_initialized:
raise CobaException(
"When using `cb without `adf` predict must be called before learn to initialize the vw learner"
)
actions = info
labels = self._labels(actions, action, reward, probability)
label = labels[actions.index(action)]
context = {'x': self._flat(context)}
adfs = None if not self._adf else [{
'a': self._flat(action)
} for action in actions]
if self._adf:
self._vw.learn(self._vw.make_examples(context, adfs, labels))
else:
self._vw.learn(self._vw.make_example(context, label))
def __init__(self,
learners: Sequence[Learner],
eta : float = 0.075,
T : float = math.inf,
mode : Literal["importance","rejection","off-policy"] ="importance",
seed : int = 1) -> None:
"""Instantiate a CorralLearner.
Args:
learners: The collection of base learners.
eta: The learning rate. This controls how quickly Corral picks a best base_learner.
T: The number of interactions expected during the learning process. A small T will cause
the learning rate to shrink towards 0 quickly while a large value for T will cause the
learning rate to shrink towards 0 slowly. A value of inf means that the learning rate
will remain constant.
mode: Determines the method with which feedback is provided to the base learners. The
original paper used importance sampling. We also support `off-policy` and `rejection`.
seed: A seed for a random number generation in ordre to get repeatable results.
"""
if mode not in ["importance", "off-policy", "rejection"]:
raise CobaException("The provided `mode` for CorralLearner was unrecognized.")
self._base_learners = [ SafeLearner(learner) for learner in learners]
M = len(self._base_learners)
self._T = T
self._gamma = 1/T
self._beta = 1/math.exp(1/math.log(T))
self._eta_init = eta
self._etas = [ eta ] * M
self._rhos = [ float(2*M) ] * M
self._ps = [ 1/M ] * M
self._p_bars = [ 1/M ] * M
self._mode = mode
self._random_pick = CobaRandom(seed)
self._random_reject = CobaRandom(CobaRandom(seed).randint(0,10000))
def init_learner(self, args: str, label_type: int) -> 'VowpalMediator':
"""Create a VW learner from a command line arg string.
Args:
args: The command line arg string to use for VW learner creation.
label_type: The label type this VW learner will take.
- 1 : `simple`__
- 2 : `multiclass`__
- 3 : `cost sensitive`__
- 4 : `contextual bandit`__
- 5 : max (deprecated)
- 6 : `conditional contextual bandit`__
- 7 : `slates`__
- 8 : `continuous actions`__
__ https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Input-format#simple
__ https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Input-format#multiclass
__ https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Input-format#cost-sensitive
__ https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Input-format#contextual-bandit
__ https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Conditional-Contextual-Bandit#vw-text-format
__ https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Slates#text-format
__ https://github.com/VowpalWabbit/vowpal_wabbit/wiki/CATS,-CATS-pdf-for-Continuous-Actions#vw-text-format
"""
from vowpalwabbit import pyvw, __version__
if self._vw is not None:
raise CobaException(
"We cannot initilaize a VW learner twice in a single mediator."
)
self._version = __version__
self._vw = pyvw.Workspace(args) if __version__[0] == '9' else pyvw.vw(
args)
self._label_type = pyvw.LabelType(
label_type) if __version__[0] == '9' else label_type
self._example_type = pyvw.Example if __version__[
0] == '9' else pyvw.example
return self
def __init__(
self,
args:
str = "--cb_explore_adf --epsilon 0.05 --interactions ax --interactions axx --ignore_linear x --random_seed 1 --quiet",
vw: VowpalMediator = None) -> None:
"""Instantiate a VowpalArgsLearner.
Args:
args: Command line arguments to instantiate a Vowpal Wabbit contextual bandit learner. For
examples and documentation on how to instantiate VW learners from command line arguments
see `here`__. We require that either cb, cb_adf, cb_explore, or cb_explore_adf is used.
When we format examples for VW context features are placed in the 'x' namespace and action
features, when relevant, are placed in the 'a' namespace.
vw: A mediator able to communicate with VW. This should not need to ever be changed.
__ https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Contextual-Bandit-algorithms
"""
if "--cb" not in args:
raise CobaException(
"VowpalArgsLearner was instantiated without a cb flag. One of the cb flags must be defined."
)
self._args = args
self._explore = "--cb_explore" in args
self._adf = "--cb_adf" in args or "--cb_explore_adf" in args
self._n_actions = None
self._actions = None
try:
self._n_actions = int(
re.match("--cb.*?\s+(\d*)\s*-?.*$", args).group(1))
except:
pass
self._vw = vw or VowpalMediator()
if self._adf or self._n_actions is not None:
self._vw.init_learner(args, 4)
def _parse_sparse_data(
self, lines: Iterable[str], headers: Sequence[str],
encoders: Sequence[Encoder]
) -> Iterable[Union[MutableSequence, MutableMapping]]:
headers_dict = dict(zip(headers, count()))
defaults_dict = {
k: "0"
for k in range(len(encoders)) if encoders[k]("0") != 0
}
encoders_dict = dict(zip(count(), encoders))
for i, line in enumerate(lines):
keys_and_vals = re.split('\s*,\s*|\s+', line.strip("} {"))
keys = list(map(int, keys_and_vals[0::2]))
vals = keys_and_vals[1::2]
if max(keys) >= len(headers) or min(keys) < 0:
raise CobaException(
f"We were unable to parse line {i} in a way that matched the expected attributes."
)
final = {**defaults_dict, **dict(zip(keys, vals))}
final_headers = headers_dict if self._header_indexing else {}
final_encoders = encoders_dict if self._lazy_encoding else {}
final_items = final if self._lazy_encoding else {
k: encoders[k](v)
for k, v in final.items()
}
if not self._lazy_encoding and not self._header_indexing:
yield final_items
else:
yield SparseWithMeta(final_items, final_headers,
final_encoders)
def learn(self, context: Context, action: Action, reward: float, probability: float, info: Info) -> None:
import numpy as np
if isinstance(action, dict) or isinstance(context, dict):
raise CobaException("Sparse data cannot be handled by this algorithm.")
if not context:
self._X_encoder = InteractionsEncoder(list(set(filter(None,[ f.replace('x','') for f in self._X]))))
context = list(Flatten().filter([list(context)]))[0] if context else []
features: np.ndarray = np.array([1]+self._X_encoder.encode(x=context,a=action)).T
if(self._A_inv is None):
self._theta = np.zeros((features.shape[0]))
self._A_inv = np.identity(features.shape[0])
r = self._theta @ features
w = self._A_inv @ features
v = features @ w
self._A_inv = self._A_inv - np.outer(w,w)/(1+v)
self._theta = self._theta + (reward-r)/(1+v) * w
def predict(self, context: Context, actions: Sequence[Action]) -> Probs:
import numpy as np #type: ignore
if isinstance(actions[0], dict) or isinstance(context, dict):
raise CobaException("Sparse data cannot be handled by this algorithm.")
if not context:
self._X_encoder = InteractionsEncoder(list(set(filter(None,[ f.replace('x','') for f in self._X]))))
context = list(Flatten().filter([list(context)]))[0] if context else []
features: np.ndarray = np.array([[1]+self._X_encoder.encode(x=context,a=action) for action in actions]).T
if(self._A_inv is None):
self._theta = np.zeros(features.shape[0])
self._A_inv = np.identity(features.shape[0])
point_estimate = self._theta @ features
point_bounds = np.diagonal(features.T @ self._A_inv @ features)
action_values = point_estimate + self._alpha*np.sqrt(point_bounds)
max_indexes = np.where(action_values == np.amax(action_values))[0]
return [ int(ind in max_indexes)/len(max_indexes) for ind in range(len(actions))]