class Reinforcements(Domain):
"""A feature domain for specifying reinforcement signals."""
# TODO: Can probably do away with the mapping... But will need to make
# changes to SimpleQNet. - Can
_config = ("mapping",)
def __init__(
self, mapping: Dict[feature, Tuple[Hashable, int]]
) -> None:
"""
Initialize Reinforcements instance.
:param mapping: A one-to-one mapping from features to dimensions. Each
feature is assumed to represent a reinforcement signal for the
corresponding dimension. Will raise a ValueError if the mapping is
found not to be one-to-one.
"""
with self.config():
self.mapping = MappingProxyType(mapping)
def update(self) -> None:
if len(set(self.mapping.keys())) != len(set(self.mapping.values())):
raise ValueError("Mapping must be one-to-one.")
super().__init__(features=tuple(self.mapping))
class InstanceAttrsProxy:
def __init__(self, attrs, instance):
self.attrs = MappingProxyType(attrs)
self.instance = instance
def __getitem__(self, item):
try:
return self.attrs[item]
except KeyError:
return getattr(self.instance, item)
def get(self, k, default=None):
try:
return self.attrs.get(k)
except KeyError:
return getattr(self.instance, k, default)
def keys(self):
return self.attrs.keys()
class DRV(object):
"""
A discrete random variable.
A DRV has one or more :dfn:`possible values` (or just :dfn:`values`), which
can be any type. Each possible value has an associated :dfn:`probability`,
which is a real number between 0 and 1.
It is strongly recommended that the probabilities add up to exactly 1. This
might be difficult to achieve with :obj:`float` probabilities, and so this
class does not enforce that restriction, and makes it possible to sample a
variable even if the total is not 1. The exact distribution of the samples
in that case is not specified, only that it will attempt to follow the
probabilities given. Loosely: if the total is too low then one value's
probability is rounded up. If the total is too high, then one probability
is rounded down, and/or one or more values is ignored. These adjustments
apply only to sampling: the original probabilities are still reported by
:func:`to_dict()` etc.
Because :code:`==` is overridden to return a DRV (not a boolean), DRV
objects are not hashable and cannot be used in a set or as a dictionary
key, even though the objects are immutable. This means you cannot have a
DRV as a "possible value" of another DRV.
DRV also resists being considered in boolean context, so for example you
cannot in general test whether or not a DRV appears in a list::
>>> from omnidice.dice import d3, d6
>>> d3 in [d3, d6]
True
>>> d6 in [d3, d6]
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "omnidice/drv.py", line 452, in __bool__
raise ValueError('The truth value of a random variable is ambiguous')
ValueError: The truth value of a random variable is ambiguous
This is the same solution used by (for example) :obj:`numpy.array`. If the
object allowed standard boolean conversion then :code:`d4 in [d3, d6]`
would be True, which is unacceptably surprising!
:param distribution: Any value from which a dictionary can be constructed,
that is a :obj:`Mapping` or :obj:`Iterable` of (value, probability)
pairs.
:param tree: The expression from which this object was defined. Currently
this is used only for the string representation, but might in future
help support lazily-evaluated DRVs.
"""
def __init__(
self,
distribution: 'DictData',
*,
tree: ExpressionTree = None,
):
self.__dist = MappingProxyType(dict(distribution))
# Cumulative distribution. Defer calculating this, because we only
# need it if the variable is actually sampled. Intermediate values in
# building up a complex DRV won't ever be sampled, so save the work.
self.__cdf = None
self.__lcm = None
self.__intvalued = None
self.__expr_tree = tree
# Computed probabilities can hit 0 due to float underflow, but maybe
# we should strip out anything with probability 0.
if not all(0 <= prob <= 1 for value, prob in self._items()):
raise ValueError('Probability not in range')
def __repr__(self):
if self.__expr_tree is not None:
return self.__expr_tree.bracketed()
return f'DRV({self.__dist})'
def is_same(self, other: 'DRV') -> bool:
"""
Return True if `self` and `other` have the same discrete probability
distribution. Possible values with 0 probability are excluded from the
comparison.
"""
values = set(value for value, prob in self._items() if prob != 0)
othervalues = set(value for value, prob in other._items() if prob != 0)
if values != othervalues:
return False
return all(self.__dist[val] == other.__dist[val] for val in values)
def is_close(self, other: 'DRV', *, rel_tol=None, abs_tol=None) -> bool:
"""
Return True if `self` and `other` have approximately the same discrete
probability distribution, within the specified tolerances. Possible
values with 0 probability are excluded from the comparison.
`rel_tol` and `abs_tol` are applied only to the probabilities, not to
the possible values. They are defined as for :func:`math.isclose`.
"""
values = set(value for value, prob in self._items() if prob != 0)
othervalues = set(value for value, prob in other._items() if prob != 0)
if values != othervalues:
return False
kwargs = {}
if rel_tol is not None:
kwargs['rel_tol'] = rel_tol
if abs_tol is not None:
kwargs['abs_tol'] = abs_tol
return all(
isclose(self.__dist[val], other.__dist[val], **kwargs)
for val in values
)
def to_dict(self) -> Dict[Any, 'Probability']:
"""
Return a dictionary mapping all possible values to probabilities.
"""
# dict(self.__dist) is type-correct, but about 3 times slower.
# Unfortunately there's no way to parameterise MappingProxyType to
# say what the type is of the underlying mapping that gets copied.
return self.__dist.copy() # type: ignore
def to_pd(self):
"""
Return a :class:`pandas.Series` mapping values to probabilities. The
series is indexed by the possible values.
:raises: :class:`ModuleNotFoundError` if pandas is not installed. Note
that pandas is not a hard dependency of this package. You must
install it to use this method.
"""
try:
import pandas as pd
except ModuleNotFoundError:
msg = 'You must install pandas for this optional feature'
raise ModuleNotFoundError(msg)
return pd.Series(self.__dist, name='probability')
def to_table(self, as_float: bool = False) -> str:
"""
Return a string containing the values and probabilities formatted as a
table. This is intended only for manually checking small distributions.
:param as_float: Display probabilites as floating-point. You might find
floats easier to read by eye.
"""
if not as_float:
items = self._items()
else:
items = ((v, float(p)) for v, p in self._items())
with contextlib.suppress(TypeError):
items = sorted(items)
return '\n'.join([
'value\tprobability',
*(f'{v}\t{p}' for v, p in items),
])
def faster(self) -> 'DRV':
"""
Return a new DRV, with all probabilities converted to float.
"""
return DRV(
{x: float(y) for x, y in self._items()},
tree=self._combine_post('.faster()'),
)
def _items(self):
return self.__dist.items()
def replace_tree(self, tree: ExpressionTree) -> 'DRV':
"""
Return a new DRV with the same distribution as this DRV, but defined
from the specified expression.
This is used for example when some optimisation has computed a DRV one
way, but we want to represent it the original way.
"""
return DRV(self.__dist, tree=tree)
@property
def cdf(self):
if self.__cdf is None:
def iter_totals():
total = 0
for value, probability in self._items():
total += probability
yield value, total
# In case of rounding errors
if total < 1:
yield value, 1
self.__cdf_values, self.__cdf = map(tuple, zip(*iter_totals()))
return self.__cdf
@property
def _lcm(self):
def lcm(a, b):
return (a * b) // gcd(a, b)
if self.__lcm is None:
result = 1
for _, prob in self._items():
if not isinstance(prob, Fraction):
result = 0
break
result = lcm(prob.denominator, result)
self.__lcm = result
return self.__lcm
def sample(self, random: Random = rng):
"""
Sample this variable.
:param random: Random number generator to use. The default is a single
object shared by all instances of :class:`DRV`.
:returns: One possible value of this variable.
"""
sample: Probability
if self._lcm == 0:
sample = random.random()
else:
sample = Fraction(random.randrange(self._lcm) + 1, self._lcm)
# The index of the first cumulative probability greater than or equal
# to our random sample. If there's a repeated probability in the array,
# that means there was a value with probability 0. So we don't want to
# select that value even in the very unlikely case of our sample being
# exactly equal to the repeated probability!
idx = bisect_left(self.cdf, sample)
return self.__cdf_values[idx]
@property
def _intvalued(self):
if self.__intvalued is None:
self.__intvalued = all(isinstance(x, int) for x in self.__dist)
return self.__intvalued
def __add__(self, right) -> 'DRV':
"""
Handler for :code:`self + right`.
Return a random variable which is the result of adding this variable to
`right`. `right` can be either a constant or another DRV (in which case
the result assumes that the two random variables are independent).
As with :meth:`apply()`, probabilities are added up wherever addition
is many-to-one (for constant numbers it is one-to-one provided overflow
does not occur).
"""
while CONVOLVE_OPTIMISATION:
if np is None:
break
if not isinstance(right, DRV):
break
product_size = len(self.__dist) * len(right.__dist)
if product_size <= CONVOLVE_SIZE_LIMIT:
break
if not self._intvalued or not right._intvalued:
break
def get_range(dist):
return range(min(dist), max(dist) + 1)
self_values = get_range(self.__dist)
right_values = get_range(right.__dist)
# Very sparse arrays aren't faster to convolve.
if 100 * product_size <= len(self_values) * len(right_values):
break
final_probs = np.convolve(
np.array(tuple(self.__dist.get(x, 0) for x in self_values)),
np.array(tuple(right.__dist.get(x, 0) for x in right_values)),
)
values = range(
min(self_values) + min(right_values),
max(self_values) + max(right_values) + 1,
)
filtered = (final_probs > 0)
values = np.array(values)[filtered].tolist()
final_probs = final_probs[filtered]
return DRV(
zip(values, final_probs),
tree=self._combine(self, right, '+'),
)
return self._apply2(operator.add, right, connective='+')
def __sub__(self, right) -> 'DRV':
"""
Handler for :code:`self - right`.
Return a random variable which is the result of subtracting `right`
from this variable. `right` can be either a constant or another DRV (in
which case the result assumes that the two random variables are
independent).
As with :meth:`apply()`, probabilities are added up wherever
subtraction is many-to-one (for constant numbers it is one-to-one
provided overflow does not occur).
"""
if isinstance(right, DRV):
# So that we get the convolve optimisation
tree = self._combine(self, right, '-')
return (self + -right).replace_tree(tree)
else:
return self._apply2(operator.sub, right, connective='-')
def __mul__(self, right):
"""
Handler for :code:`self * right`.
Return a random variable which is the result of multiplying this
variable with `right`. `right` can be either a constant or another DRV
(in which case the result assumes that the two random variables are
independent).
As with :meth:`apply()`, probabilities are added up in the case where
multiplication is not one-to-one (for constant numbers other than zero
it is one-to-one provided overflow and underflow do not occur).
"""
return self._apply2(operator.mul, right, connective='*')
def __rmatmul__(self, left: int) -> 'DRV':
"""
Handler for :code:`left @ self`.
Return a random variable which is the result of sampling this variable
`left` times, and adding the results together.
"""
if not isinstance(left, int):
return NotImplemented
if left <= 0:
raise ValueError(left)
# Exponentiation by squaring. This isn't massively faster, but does
# help a bit for hundreds of dice.
result = None
so_far = self
original = left
while True:
if left % 2 == 1:
if result is None:
result = so_far
else:
result += so_far
left //= 2
if left == 0:
break
so_far += so_far
# left was non-zero, so result cannot still be None
result = cast(DRV, result)
return result.replace_tree(self._combine(original, self, '@'))
def __matmul__(self, right: 'DRV') -> 'DRV':
"""
Handler for :code:`self @ right`.
Return a random variable which is the result of sampling this variable
once, then adding together that many samples of `right`.
All possible values of this variable must be of type :obj:`int`.
"""
if not isinstance(right, DRV):
return NotImplemented
if not all(isinstance(value, int) for value in self.__dist):
raise TypeError('require integers on LHS of @')
def iter_drvs():
so_far = min(self.__dist) @ right
for num_dice in range(min(self.__dist), max(self.__dist) + 1):
if num_dice in self.__dist:
yield so_far, self.__dist[num_dice]
so_far += right
return DRV.weighted_average(
iter_drvs(),
tree=self._combine(self, right, '@'),
)
def __truediv__(self, right) -> 'DRV':
"""
Handler for :code:`self / right`.
Return a random variable which is the result of floor-dividing this
variable by `right`. `right` can be either a constant or another DRV
(in which case the result assumes that the two random variables are
independent).
As with :meth:`apply()`, probabilities are added up wherever division
is many-to-one (for constant numbers other than zero it is one-to-one
provided overflow and underflow do not occur).
0 must not be a possible value of `right` (even with probability 0).
"""
return self._apply2(operator.truediv, right, connective='/')
def __floordiv__(self, right) -> 'DRV':
"""
Handler for :code:`self // right`.
Return a random variable which is the result of dividing this
variable by `right`. `right` can be either a constant or another DRV
(in which case the result assumes that the two random variables are
independent).
As with :meth:`apply()`, probabilities are added up wherever floor
division is many-to-one (for numbers it is mostly many-to-one, for
example :code:`2 // 2 == 1 == 3 // 2`).
0 must not be a possible value of `right` (even with probability 0).
"""
return self._apply2(operator.floordiv, right, connective='//')
def __neg__(self) -> 'DRV':
"""
Handler for :code:`-self`.
Return a random variable which is the result of negating the values of
this variable.
As with :meth:`apply()`, probabilities are added up wherever negation
is many-to-one (for numbers it is one-to-one).
"""
return self.apply(operator.neg, tree=self._combine(self, '-'))
def __eq__(self, right) -> 'DRV': # type: ignore[override]
"""
Handler for :code:`self == right`.
Return a random variable which takes value :obj:`True` where `self` is
equal to `right`, and :obj:`False` otherwise. `right` can be either a
constant or another DRV (in which case the result assumes that the two
random variables are independent).
If either :obj:`True` or :obj:`False` cannot happen then the result
has only one possible value, with probability 1. There is no possible
value with probability 0.
"""
if isinstance(right, DRV):
small, big = sorted([self, right], key=lambda x: len(x.__dist))
prob = sum(
prob * big.__dist.get(val, 0)
for val, prob in small._items()
)
else:
prob = self.__dist.get(right)
if not prob:
return DRV({False: 1})
if prob >= 1.0:
return DRV({True: 1})
return DRV(
{False: 1 - prob, True: prob},
tree=self._combine(self, right, '=='),
)
def __ne__(self, right: 'DRV') -> 'DRV': # type: ignore[override]
"""
Handler for :code:`self != right`.
Return a random variable which takes value :obj:`True` where `self` is
not equal to `right`, and :obj:`False` otherwise. `right` can be either
a constant or another DRV (in which case the result assumes that the
two random variables are independent).
If either :obj:`True` or :obj:`False` cannot happen then the result
has only one possible value, with probability 1. There is no possible
value with probability 0.
"""
return (
(self == right)
.apply(operator.not_)
.replace_tree(self._combine(self, right, '!='))
)
def __bool__(self):
# Prevent DRVs being truthy, and hence "3 in [DRV({2: 1})]" is true.
raise ValueError('The truth value of a random variable is ambiguous')
def __le__(self, right) -> 'DRV':
"""
Handler for :code:`self <= right`.
Return a random variable which takes value :obj:`True` where `self` is
less than or equal to `right`, and :obj:`False` otherwise. `right` can
be either a constant or another DRV (in which case the result assumes
that the two random variables are independent).
If either :obj:`True` or :obj:`False` cannot happen then the result
has only one possible value, with probability 1. There is no possible
value with probability 0.
"""
return self._apply2(operator.le, right, connective='<=')
def __lt__(self, right) -> 'DRV':
"""
Handler for :code:`self < right`.
Return a random variable which takes value :obj:`True` where `self` is
less than `right`, and :obj:`False` otherwise. `right` can be either a
constant or another DRV (in which case the result assumes that the two
random variables are independent).
If either :obj:`True` or :obj:`False` cannot happen then the result
has only one possible value, with probability 1. There is no possible
value with probability 0.
"""
return self._apply2(operator.lt, right, connective='<')
def __ge__(self, right) -> 'DRV':
"""
Handler for :code:`self >= right`.
Return a random variable which takes value :obj:`True` where `self` is
greater than or equal to `right`, and :obj:`False` otherwise. `right`
can be either a constant or another DRV (in which case the result
assumes that the two random variables are independent).
If either :obj:`True` or :obj:`False` cannot happen then the result
has only one possible value, with probability 1. There is no possible
value with probability 0.
"""
return self._apply2(operator.ge, right, connective='>=')
def __gt__(self, right) -> 'DRV':
"""
Handler for :code:`self > right`.
Return a random variable which takes value :obj:`True` where `self` is
greater than `right`, and :obj:`False` otherwise. `right` can be either
a constant or another DRV (in which case the result assumes that the
two random variables are independent).
If either :obj:`True` or :obj:`False` cannot happen then the result
has only one possible value, with probability 1. There is no possible
value with probability 0.
"""
return self._apply2(operator.gt, right, connective='>')
def explode(self, rerolls: int = 50) -> 'DRV':
"""
Return a new DRV distributed according to the rules of an "exploding
die". This means, first roll the die (sample this DRV). If the result
is not the maximum possible, then keep it. If it is the maximum, then
roll again and add the new result to the original.
Because DRV represents only finitely-many possible values, whereas the
process of rerolling can (with minuscule probability) carry on
indefinitely, this method imposes an arbitary limit to the number of
rerolls.
:param rerolls: The maximum number of rerolls. Set this to 1 for a die
that can only "explode" once, not indefinitely.
"""
reroll_value = max(self.__dist.keys())
reroll_prob = self.__dist[reroll_value]
each_die = self.to_dict()
each_die.pop(reroll_value)
def iter_pairs():
for idx in range(rerolls + 1):
for value, prob in each_die.items():
value += reroll_value * idx
prob *= reroll_prob ** idx
yield (value, prob)
yield (reroll_value * (idx + 1), reroll_prob ** (idx + 1))
postfix = '.explode()' if rerolls == 50 else f'.explode({rerolls!r})'
return self._reduced(iter_pairs(), tree=self._combine_post(postfix))
def apply(
self,
func: Callable[[Any], Any],
*,
tree: ExpressionTree = None,
allow_drv: bool = False,
) -> 'DRV':
"""
Apply a unary function to the values produced by this DRV. If `func` is
an injective (one-to-one) function, then the probabilities are
unchanged. If `func` is many-to-one, then the probabilities are added
together.
:param func: Function to map the values. Each value `x` is replaced by
`func(x)`.
:param tree: the expression from which this object was defined. If
``None``, the result DRV is represented by listing out all the values
and probabilities.
:param allow_drv: If True, then when `func` returns a DRV, the possible
values of that DRV are each included in the returned DRV. Recall that
a DRV cannot be a possible value of the returned DRV, because it is
not hashable. So, without this option `func` cannot return a DRV.
.. versionchanged:: 1.1
Added ``allow_drv`` option.
"""
return DRV._reduced(self._items(), func, tree=tree, drv=allow_drv)
def _apply2(self, func, right, connective=None) -> 'DRV':
"""Apply a binary function, with the values of this DRV on the left."""
expr_tree = self._combine(self, right, connective)
if isinstance(right, DRV):
return self._cross_reduce(func, right, tree=expr_tree)
return self.apply(lambda x: func(x, right), tree=expr_tree)
def _cross_reduce(self, func, right, tree=None) -> 'DRV':
"""
Take the cross product of self and right, then reduce by applying func.
"""
return DRV._reduced(
self._iter_cross(right),
lambda value: func(*value),
tree=tree,
)
def _iter_cross(self, right):
"""
Take the cross product of self and right, with probabilities assuming
that the two are independent variables.
Note that the cross product of an object with itself represents the
results of sampling it twice, *not* just the pairs (x, x) for each
possible value!
"""
for (lvalue, lprob) in self._items():
for (rvalue, rprob) in right._items():
yield ((lvalue, rvalue), lprob * rprob)
@staticmethod
def _reduced(iterable, func=lambda x: x, tree=None, drv=False) -> 'DRV':
distribution: dict = collections.defaultdict(int)
if not drv:
# Optimisation does make a difference to e.g. test_convolve
for value, prob in iterable:
distribution[func(value)] += prob
else:
for value, prob in iterable:
transformed = func(value)
if isinstance(transformed, DRV):
for value2, prob2 in transformed._weighted_items(prob):
distribution[value2] += prob2
else:
distribution[transformed] += prob
return DRV(distribution, tree=tree)
@staticmethod
def weighted_average(
iterable: Iterable[Tuple['DRV', 'Probability']],
*,
tree: ExpressionTree = None,
) -> 'DRV':
"""
Compute a weighted average of DRVs, each with its own probability.
This is for when you have a set of mutually-exclusive events which can
happen, and then the final outcome occurs with a different known
distribution according to which of those events occurs. For example,
this function is used to implement the ``@`` operator when the
left-hand-side is a DRV. The first roll determines what the second roll
will be.
The DRVs that are averaged together do not need to be disjoint (that
is, they can have overlapping possible values). Whenever multiple
events lead to the same final outcome, the probabilities are combined:
https://en.wikipedia.org/wiki/Law_of_total_probability
:param iterable: Pairs, each containing a DRV and the probability of
that DRV being the one selected. The probabilities should add to 1,
but this is not enforced.
:param tree: the expression from which this object was defined. If
``None``, the result DRV is represented by listing out all the values
and probabilities.
.. versionadded:: 1.1
"""
def iter_pairs():
for drv, weight in iterable:
yield from drv._weighted_items(weight)
return DRV._reduced(iter_pairs(), tree=tree)
def _weighted_items(self, weight, pred=lambda x: True):
for value, prob in self.__dist.items():
if pred(value):
yield value, prob * weight
def given(self, predicate: Callable[[Any], bool]) -> 'DRV':
"""
Return the conditional probability distribution of this DRV, restricted
to the possible values for which `predicate` is true.
For example, :code:`drv.given(lambda x: True)` is the same distribution
as :code:`drv`, and the following are equivalent to each other::
d6.given(lambda x: bool(x % 2))
DRV({1: Fraction(1, 3), 3: Fraction(1, 3), 5: Fraction(1, 3)})
If `x` is a DRV, and `A` and `B` are predicates, then the conditional
probability of `A` given `B`, written in probability theory as
``p(A(x) | B(x))``, can be computed as :code:`p(x.given(B).apply(A)))`.
:param predicate: Called with possible values of `self`, and must
return :obj:`bool` (not just truthy).
:raises ZeroDivisionError: if the probability of `predicate` being
true is 0.
.. versionadded:: 1.1
"""
total = p(self.apply(predicate))
if total == 0:
# Would be raised anyway, but nicer error message
raise ZeroDivisionError('predicate is True with probability 0')
return DRV(self._weighted_items(1 / total, predicate))
@staticmethod
def _combine(*args):
"""
Helper for combining two expressions into a combined expression.
"""
for arg in args:
if isinstance(arg, DRV) and arg.__expr_tree is None:
return None
def unpack(subexpr):
if isinstance(subexpr, DRV):
return subexpr.__expr_tree
return Atom(repr(subexpr))
if len(args) == 2:
# Unary expression
subexpr, connective = args
return UnaryExpression(unpack(subexpr), connective)
# Binary expression
left, right, connective = args
return BinaryExpression(unpack(left), unpack(right), connective)
def _combine_post(self, postfix):
if self.__expr_tree is None:
return None
return AttrExpression(self.__expr_tree, postfix)
class MessagePassing(torch.nn.Module):
r"""Base class for creating message passing layers
.. math::
\mathbf{x}_i^{\prime} = \gamma_{\mathbf{\Theta}} \left( \mathbf{x}_i,
\square_{j \in \mathcal{N}(i)} \, \phi_{\mathbf{\Theta}}
\left(\mathbf{x}_i, \mathbf{x}_j,\mathbf{e}_{i,j}\right) \right),
where :math:`\square` denotes a differentiable, permutation invariant
function, *e.g.*, sum, mean or max, and :math:`\gamma_{\mathbf{\Theta}}`
and :math:`\phi_{\mathbf{\Theta}}` denote differentiable functions such as
MLPs.
See `here <https://pytorch-geometric.readthedocs.io/en/latest/notes/
create_gnn.html>`__ for the accompanying tutorial.
Args:
aggr (string, optional): The aggregation scheme to use
(:obj:`"add"`, :obj:`"mean"` or :obj:`"max"`).
(default: :obj:`"add"`)
flow (string, optional): The flow direction of message passing
(:obj:`"source_to_target"` or :obj:`"target_to_source"`).
(default: :obj:`"source_to_target"`)
node_dim (int, optional): The axis along which to propagate.
(default: :obj:`0`)
"""
def __init__(self, aggr='add', flow='source_to_target', node_dim=0):
super(MessagePassing, self).__init__()
self.aggr = aggr
assert self.aggr in ['add', 'mean', 'max']
self.flow = flow
assert self.flow in ['source_to_target', 'target_to_source']
self.node_dim = node_dim
assert self.node_dim >= 0
self.__msg_params__ = inspect.signature(self.message).parameters
self.__aggr_params__ = inspect.signature(self.aggregate).parameters
self.__aggr_params__ = OrderedDict(self.__aggr_params__)
self.__aggr_params__.popitem(last=False)
self.__aggr_params__ = MappingProxyType(self.__aggr_params__)
self.__update_params__ = inspect.signature(self.update).parameters
self.__update_params__ = OrderedDict(self.__update_params__)
self.__update_params__.popitem(last=False)
self.__update_params__ = MappingProxyType(self.__update_params__)
msg_args = set(self.__msg_params__.keys()) - msg_special_args
aggr_args = set(self.__aggr_params__.keys()) - aggr_special_args
update_args = set(self.__update_params__.keys()) - update_special_args
self.__args__ = set().union(msg_args, aggr_args, update_args)
def __set_size__(self, size, index, tensor):
if not torch.is_tensor(tensor):
pass
elif size[index] is None:
size[index] = tensor.size(self.node_dim)
elif size[index] != tensor.size(self.node_dim):
raise ValueError(
(f'Encountered node tensor with size '
f'{tensor.size(self.node_dim)} in dimension {self.node_dim}, '
f'but expected size {size[index]}.'))
def __collect__(self, edge_index, size, kwargs):
i, j = (0, 1) if self.flow == "target_to_source" else (1, 0)
ij = {"_i": i, "_j": j}
out = {}
for arg in self.__args__:
if arg[-2:] not in ij.keys():
out[arg] = kwargs.get(arg, inspect.Parameter.empty)
else:
idx = ij[arg[-2:]]
data = kwargs.get(arg[:-2], inspect.Parameter.empty)
if data is inspect.Parameter.empty:
out[arg] = data
continue
if isinstance(data, tuple) or isinstance(data, list):
assert len(data) == 2
self.__set_size__(size, 1 - idx, data[1 - idx])
data = data[idx]
if not torch.is_tensor(data):
out[arg] = data
continue
self.__set_size__(size, idx, data)
out[arg] = data.index_select(self.node_dim, edge_index[idx])
size[0] = size[1] if size[0] is None else size[0]
size[1] = size[0] if size[1] is None else size[1]
# Add special message arguments.
out['edge_index'] = edge_index
out['edge_index_i'] = edge_index[i]
out['edge_index_j'] = edge_index[j]
out['size'] = size
out['size_i'] = size[i]
out['size_j'] = size[j]
# Add special aggregate arguments.
out['index'] = out['edge_index_i']
out['dim_size'] = out['size_i']
return out
def __distribute__(self, params, kwargs):
out = {}
for key, param in params.items():
data = kwargs[key]
if data is inspect.Parameter.empty:
if param.default is inspect.Parameter.empty:
raise TypeError(f'Required parameter {key} is empty.')
data = param.default
out[key] = data
return out
def propagate(self, edge_index, size=None, **kwargs):
r"""The initial call to start propagating messages.
Args:
edge_index (Tensor): The indices of a general (sparse) assignment
matrix with shape :obj:`[N, M]` (can be directed or
undirected).
size (list or tuple, optional): The size :obj:`[N, M]` of the
assignment matrix. If set to :obj:`None`, the size will be
automatically inferred and assumed to be quadratic.
(default: :obj:`None`)
**kwargs: Any additional data which is needed to construct and
aggregate messages, and to update node embeddings.
"""
size = [None, None] if size is None else size
size = [size, size] if isinstance(size, int) else size
size = size.tolist() if torch.is_tensor(size) else size
size = list(size) if isinstance(size, tuple) else size
assert isinstance(size, list)
assert len(size) == 2
kwargs = self.__collect__(edge_index, size, kwargs)
msg_kwargs = self.__distribute__(self.__msg_params__, kwargs)
out = self.message(**msg_kwargs)
aggr_kwargs = self.__distribute__(self.__aggr_params__, kwargs)
out = self.aggregate(out, **aggr_kwargs)
update_kwargs = self.__distribute__(self.__update_params__, kwargs)
out = self.update(out, **update_kwargs)
return out
def message(self, x_j): # pragma: no cover
r"""Constructs messages to node :math:`i` in analogy to
:math:`\phi_{\mathbf{\Theta}}` for each edge in
:math:`(j,i) \in \mathcal{E}` if :obj:`flow="source_to_target"` and
:math:`(i,j) \in \mathcal{E}` if :obj:`flow="target_to_source"`.
Can take any argument which was initially passed to :meth:`propagate`.
In addition, tensors passed to :meth:`propagate` can be mapped to the
respective nodes :math:`i` and :math:`j` by appending :obj:`_i` or
:obj:`_j` to the variable name, *.e.g.* :obj:`x_i` and :obj:`x_j`.
"""
return x_j
def aggregate(self, inputs, index, dim_size): # pragma: no cover
r"""Aggregates messages from neighbors as
:math:`\square_{j \in \mathcal{N}(i)}`.
By default, delegates call to scatter functions that support
"add", "mean" and "max" operations specified in :meth:`__init__` by
the :obj:`aggr` argument.
"""
return scatter_(self.aggr, inputs, index, self.node_dim, dim_size)
def update(self, inputs): # pragma: no cover
r"""Updates node embeddings in analogy to
:math:`\gamma_{\mathbf{\Theta}}` for each node
:math:`i \in \mathcal{V}`.
Takes in the output of aggregation as first argument and any argument
which was initially passed to :meth:`propagate`.
"""
return inputs
"""
>>> from types import MappingProxyType
>>> d = {1:'A'}
>>> d_proxy = MappingProxyType(d)
>>> d_proxy
mappingproxy({1: 'A'})
>>> d_proxy[1]
'A'
>>> d_proxy[2] = 'x'
Traceback (most recent call last):
File '<stdin>', line 1, in <module>
TypeError: 'mappingproxy' object does not support item assignment
>>> d[2] = 'B'
>>> d_proxy
mappingproxy({1: 'A', 2: 'B'})
>>> d_proxy[2]
'B'
>>>
"""
from types import MappingProxyType
origin_dict = {"a": 100, "b": 200}
use_dict = MappingProxyType(origin_dict)
print(use_dict)
print(use_dict.keys())
print(type(use_dict))
'orc8r',
'lte',
'feg',
'cwf',
'dp',
)
DEPLOYMENT_TO_MODULES = MappingProxyType({
'all': MODULES,
'orc8r': ('orc8r'),
'fwa': ('orc8r', 'lte'),
'ffwa': ('orc8r', 'lte', 'feg'),
'cwf': ('orc8r', 'lte', 'feg', 'cwf'),
})
DEPLOYMENTS = DEPLOYMENT_TO_MODULES.keys()
EXTRA_COMPOSE_FILES = (
'docker-compose.metrics.yml',
# For now, logging is left out of the build because the fluentd daemonset
# and forwarder pod shouldn't change very frequently - we can build and
# push locally when they need to be updated.
# We can integrate this into the CI pipeline if/when we see the need for it
# 'docker-compose.logging.yml',
)
MagmaModule = namedtuple('MagmaModule', ['name', 'host_path'])
def main() -> None:
"""
class ProjectVersions():
r"""
make_ver2
read in a project and have all permutations of that project in an object
We can lazily evaluate each version
>>> from contextlib import contextmanager
>>> @contextmanager
... def testfiles():
... import os
... import tempfile
... td = tempfile.TemporaryDirectory()
... filenames = set()
... def write_file(filename, data):
... filename = os.path.join(td.name, filename)
... with open(filename, 'wt') as filehandle:
... _ = filehandle.write(data)
... filenames.add(filename)
... write_file('test.py', '''
... print('Hello World')
... ''')
... write_file('test.js', '''
... console.log("Hello World")
... ''')
... write_file('Test.java', '''
... public class Test {
... public Test() {
... System.out.println("Hello World");
... }
... public static void main(String[] args) {new Test();}
... }
... ''')
... write_file('test.ver', '''
... VERNAME: base base
... ''')
... yield filenames
... td.cleanup()
>>> with testfiles() as filenames:
... p = ProjectVersions(filenames)
>>> p.versions['base']
('base',)
>>> p.data['py']['base']
"""
def __init__(self, filenames):
self.files = MappingProxyType({
f.suffix.strip('.'): f.open(encoding='utf8').read()
for f in map(Path, filenames)
})
self.data # generate cache on object creation
@cached_property
def langauges(self):
return frozenset(self.files.keys()) - frozenset({'ver', 'yaml', 'yml', 'txt', 'md'})
@cached_property
def versions(self):
"""
Parse versions from .ver or .yaml file
"""
file_exts = set(self.files.keys())
if {'yaml', 'yml'} & file_exts:
raise NotImplementedError('yaml version file format not implemented')
if {'ver',} & file_exts:
return parse_legacy_version_data(self.files['ver'])
raise Exception()
#@lru_cache?
def langauge(self, language):
return MappingProxyType({
ver_name: '\n'.join(make_ver(io.StringIO(self.files[language]), ver_path=ver_path, lang=language, process_additional_metafiles=False))
for ver_name, ver_path in self.versions.items()
})
@cached_property
def data(self):
return MappingProxyType({l: self.langauge(l) for l in self.langauges})
"""MappingProxyType"""
from types import MappingProxyType
d = {'a': 1, 'b': 2}
mp = MappingProxyType(d)
# Read-only
print(list(mp.keys()))
print(list(mp.values()))
print(mp.get('a'))
print(mp.get('c', 'not found'))
# Immutable
# del mp['a'] # TypeError
# mp['a'] = 100 # TypeError
# Mutate the original dictionary
d['a'] = 100
d['c'] = 'new item'
del d['b']
print(mp) # Reflect the changes in original dictionary
"application/vnd.visio",
".vsdx":
"application/octet-stream",
".xls":
"application/vnd.ms-excel",
".xlsm":
"application/vnd.ms-excel.sheet.macroenabled.12",
".xlsx":
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
".xml":
"text/xml",
".zip":
"application/zip",
}, )
#: file extensions that can be proceed by BotX API.
BOTX_API_ACCEPTED_EXTENSIONS = EXTENSIONS_TO_MIMETYPES.keys()
class NamedAsyncIterable(AsyncIterable):
"""AsyncIterable with `name` protocol."""
name: str
class File(BotXBaseModel): # noqa: WPS214
"""Object that represents file in RFC 2397 format."""
#: name of file.
file_name: str
#: file content in RFC 2397 format.
class Composition:
"""
Defines a composition of a compound.
To create a composition, use the class methods:
- :meth:`from_pure`
- :meth:`from_formula`
- :meth:`from_mass_fractions`
- :meth:`from_atomic_fractions`
Use the following attributes to access the composition values:
- :attr:`mass_fractions`: :class:`dict` where the keys are atomic numbers and the values weight fractions.
- :attr:`atomic_fractions`: :class:`dict` where the keys are atomic numbers and the values atomic fractions.
- :attr:`formula`: chemical formula
The composition object is immutable, i.e. it cannot be modified once created.
Equality can be checked.
It is hashable.
It can be pickled or copied.
"""
_key = object()
PRECISION = 0.000000001 # 1ppb
def __init__(self, key, mass_fractions, atomic_fractions, formula):
"""
Private constructor. It should never be used.
"""
if key != Composition._key:
raise TypeError('Composition cannot be created using constructor')
if set(mass_fractions.keys()) != set(atomic_fractions.keys()):
raise ValueError('Mass and atomic fractions must have the same elements')
self.mass_fractions = MappingProxyType(mass_fractions)
self.atomic_fractions = MappingProxyType(atomic_fractions)
self._formula = formula
@classmethod
def from_pure(cls, z):
"""
Creates a pure composition.
Args:
z (int): atomic number
"""
return cls(cls._key, {z: 1.0}, {z: 1.0}, pyxray.element_symbol(z))
@classmethod
def from_formula(cls, formula):
"""
Creates a composition from a chemical formula.
Args:
formula (str): chemical formula
"""
atomic_fractions = convert_formula_to_atomic_fractions(formula)
return cls.from_atomic_fractions(atomic_fractions)
@classmethod
def from_mass_fractions(cls, mass_fractions, formula=None):
"""
Creates a composition from a mass fraction :class:`dict`.
Args:
mass_fractions (dict): mass fraction :class:`dict`.
The keys are atomic numbers and the values weight fractions.
Wildcard are accepted, e.g. ``{5: '?', 25: 0.4}`` where boron
will get a mass fraction of 0.6.
formula (str): optional chemical formula for the composition.
If ``None``, a formula will be generated for the composition.
"""
mass_fractions = process_wildcard(mass_fractions)
atomic_fractions = convert_mass_to_atomic_fractions(mass_fractions)
if not formula:
formula = generate_name(atomic_fractions)
return cls(cls._key, mass_fractions, atomic_fractions, formula)
@classmethod
def from_atomic_fractions(cls, atomic_fractions, formula=None):
"""
Creates a composition from an atomic fraction :class:`dict`.
Args:
atomic_fractions (dict): atomic fraction :class:`dict`.
The keys are atomic numbers and the values atomic fractions.
Wildcard are accepted, e.g. ``{5: '?', 25: 0.4}`` where boron
will get a atomic fraction of 0.6.
formula (str): optional chemical formula for the composition.
If ``None``, a formula will be generated for the composition.
"""
atomic_fractions = process_wildcard(atomic_fractions)
mass_fractions = convert_atomic_to_mass_fractions(atomic_fractions)
if not formula:
formula = generate_name(atomic_fractions)
return cls(cls._key, mass_fractions, atomic_fractions, formula)
def __len__(self):
return len(self.mass_fractions)
def __contains__(self, z):
return z in self.mass_fractions
def __iter__(self):
return iter(self.mass_fractions.keys())
def __repr__(self):
return '<{}({})>'.format(self.__class__.__name__, self.inner_repr())
def __eq__(self, other):
if not isinstance(other, self.__class__):
return False
if len(self) != len(other):
return False
for z in self.mass_fractions:
if z not in other.mass_fractions:
return False
fraction = self.mass_fractions[z]
other_fraction = other.mass_fractions[z]
if not math.isclose(fraction, other_fraction, abs_tol=self.PRECISION):
return False
return True
def __ne__(self, other):
return not self == other
def __hash__(self):
out = []
for z in sorted(self.mass_fractions):
out.append(z)
out.append(int(self.mass_fractions[z] / self.PRECISION))
return hash(tuple(out))
def __getstate__(self):
return {'mass_fractions': dict(self.mass_fractions),
'atomic_fractions': dict(self.atomic_fractions),
'formula': self.formula}
def __setstate__(self, state):
self.mass_fractions = MappingProxyType(state.get('mass_fractions', {}))
self.atomic_fractions = MappingProxyType(state.get('atomic_fractions', {}))
self._formula = state.get('formula', '')
def is_normalized(self):
return math.isclose(sum(self.mass_fractions.values()), 1.0, abs_tol=self.PRECISION)
def inner_repr(self):
return ', '.join('{}: {:.4f}'.format(pyxray.element_symbol(z), mass_fraction) for z, mass_fraction in self.mass_fractions.items())
@property
def formula(self):
return self._formula
LIST = 2
W_STR_RENDERING_TYPE_2_VALUE_TYPE = MappingProxyType(
collections.OrderedDict([
(WithStrRenderingType.STRING, ValueType.STRING),
(WithStrRenderingType.PATH, ValueType.PATH),
(WithStrRenderingType.LIST, ValueType.LIST),
]))
VALUE_TYPE_2_W_STR_RENDERING_TYPE = MappingProxyType(
collections.OrderedDict([
(item[1], item[0])
for item in W_STR_RENDERING_TYPE_2_VALUE_TYPE.items()
]
))
def sorted_types(values: Iterable[ValueType]) -> Tuple[ValueType, ...]:
return tuple(
sorted(values,
key=_value_type_sorting_key)
)
def _value_type_sorting_key(x: ValueType) -> int:
return x.value
VALUE_TYPES_W_STR_RENDERING = sorted_types(VALUE_TYPE_2_W_STR_RENDERING_TYPE.keys())
class Composition:
"""
Defines a composition of a compound.
To create a composition, use the class methods:
- :meth:`from_pure`
- :meth:`from_formula`
- :meth:`from_mass_fractions`
- :meth:`from_atomic_fractions`
Use the following attributes to access the composition values:
- :attr:`mass_fractions`: :class:`dict` where the keys are atomic numbers and the values weight fractions.
- :attr:`atomic_fractions`: :class:`dict` where the keys are atomic numbers and the values atomic fractions.
- :attr:`formula`: chemical formula
The composition object is immutable, i.e. it cannot be modified once created.
Equality can be checked.
It is hashable.
It can be pickled or copied.
"""
_key = object()
PRECISION = 0.000000001 # 1ppb
def __init__(self, key, mass_fractions, atomic_fractions, formula):
"""
Private constructor. It should never be used.
"""
if key != Composition._key:
raise TypeError("Composition cannot be created using constructor")
if set(mass_fractions.keys()) != set(atomic_fractions.keys()):
raise ValueError(
"Mass and atomic fractions must have the same elements")
self.mass_fractions = MappingProxyType(mass_fractions)
self.atomic_fractions = MappingProxyType(atomic_fractions)
self._formula = formula
@classmethod
def from_pure(cls, z):
"""
Creates a pure composition.
Args:
z (int): atomic number
"""
return cls(cls._key, {z: 1.0}, {z: 1.0}, pyxray.element_symbol(z))
@classmethod
def from_formula(cls, formula):
"""
Creates a composition from a chemical formula.
Args:
formula (str): chemical formula
"""
atomic_fractions = convert_formula_to_atomic_fractions(formula)
return cls.from_atomic_fractions(atomic_fractions)
@classmethod
def from_mass_fractions(cls, mass_fractions, formula=None):
"""
Creates a composition from a mass fraction :class:`dict`.
Args:
mass_fractions (dict): mass fraction :class:`dict`.
The keys are atomic numbers and the values weight fractions.
Wildcard are accepted, e.g. ``{5: '?', 25: 0.4}`` where boron
will get a mass fraction of 0.6.
formula (str): optional chemical formula for the composition.
If ``None``, a formula will be generated for the composition.
"""
mass_fractions = process_wildcard(mass_fractions)
atomic_fractions = convert_mass_to_atomic_fractions(mass_fractions)
if not formula:
formula = generate_name(atomic_fractions)
return cls(cls._key, mass_fractions, atomic_fractions, formula)
@classmethod
def from_atomic_fractions(cls, atomic_fractions, formula=None):
"""
Creates a composition from an atomic fraction :class:`dict`.
Args:
atomic_fractions (dict): atomic fraction :class:`dict`.
The keys are atomic numbers and the values atomic fractions.
Wildcard are accepted, e.g. ``{5: '?', 25: 0.4}`` where boron
will get a atomic fraction of 0.6.
formula (str): optional chemical formula for the composition.
If ``None``, a formula will be generated for the composition.
"""
atomic_fractions = process_wildcard(atomic_fractions)
mass_fractions = convert_atomic_to_mass_fractions(atomic_fractions)
if not formula:
formula = generate_name(atomic_fractions)
return cls(cls._key, mass_fractions, atomic_fractions, formula)
def __len__(self):
return len(self.mass_fractions)
def __contains__(self, z):
return z in self.mass_fractions
def __iter__(self):
return iter(self.mass_fractions.keys())
def __repr__(self):
return "<{}({})>".format(self.__class__.__name__, self.inner_repr())
def __eq__(self, other):
if not isinstance(other, self.__class__):
return False
if len(self) != len(other):
return False
for z in self.mass_fractions:
if z not in other.mass_fractions:
return False
fraction = self.mass_fractions[z]
other_fraction = other.mass_fractions[z]
if not math.isclose(
fraction, other_fraction, abs_tol=self.PRECISION):
return False
return True
def __ne__(self, other):
return not self == other
def __hash__(self):
out = []
for z in sorted(self.mass_fractions):
out.append(z)
out.append(int(self.mass_fractions[z] / self.PRECISION))
return hash(tuple(out))
def __getstate__(self):
return {
"mass_fractions": dict(self.mass_fractions),
"atomic_fractions": dict(self.atomic_fractions),
"formula": self.formula,
}
def __setstate__(self, state):
self.mass_fractions = MappingProxyType(state.get("mass_fractions", {}))
self.atomic_fractions = MappingProxyType(
state.get("atomic_fractions", {}))
self._formula = state.get("formula", "")
def is_normalized(self):
return math.isclose(sum(self.mass_fractions.values()),
1.0,
abs_tol=self.PRECISION)
def inner_repr(self):
return ", ".join(
"{}: {:.4f}".format(pyxray.element_symbol(z), mass_fraction)
for z, mass_fraction in self.mass_fractions.items())
@property
def formula(self):
return self._formula
