def render_mpl(self, act_state: state.ProjectedState):
"""Returns a matplotlib figure with the state on it"""
fig = plt.figure(figsize=self.frame_size_in)
ax = fig.add_subplot(111, projection='3d')
axtitle = ax.set_title(act_state.title)
font_size = int((80 / 1920) * self._frame_size[0])
axtitle.set_fontsize(font_size)
renderer = fig.canvas.get_renderer()
bb = axtitle.get_window_extent(renderer=renderer)
while bb.width >= self._frame_size[0] * 0.9:
font_size -= 5
axtitle.set_fontsize(font_size)
bb = axtitle.get_window_extent(renderer=renderer)
for pts, mask, skwargs in act_state.visible_points:
tus.check(skwargs=(skwargs, dict))
self._scatter(act_state, ax, pts, mask, skwargs)
ax.set_xlim(*[float(i) for i in act_state.zoom[0]])
ax.set_ylim(*[float(i) for i in act_state.zoom[1]])
ax.set_zlim(*[float(i) for i in act_state.zoom[2]])
ax.view_init(*act_state.rotation)
return fig
def __init__(self, samples: np.ndarray, centers: np.ndarray,
labels: np.ndarray,
calculate_params: typing.Dict[str, typing.Any]):
tus.check(samples=(samples, np.ndarray),
centers=(centers, np.ndarray),
labels=(labels, np.ndarray),
calculate_params=(calculate_params, dict))
tus.check_ndarrays(
samples=(samples, ('n_samples', 'n_features'),
(np.dtype('float32'), np.dtype('float64'))),
centers=(centers,
('n_clusters',
('n_features',
samples.shape[1] if len(samples.shape) > 1 else None)),
samples.dtype),
labels=(labels,
(('n_samples',
samples.shape[0] if bool(samples.shape) else None), ),
(np.dtype('int32'), np.dtype('int64'))))
self.samples = samples
self.centers = centers
self.labels = labels
self.calculate_params = calculate_params
self._bounds = None
def __init__(self, base: Scene):
tus.check(base=(base, Scene))
self.base = base
self.followed_by = []
self.joined_with = []
self.join_is_sep = False
self.pushed = []
def __init__(self, username, password):
tus.check(username=(username, str), password=(password, str))
self.username = username
self.password = password
self._header = 'Basic ' + base64.b64encode(
(self.username + ':' +
self.password).encode('ascii')).decode('ascii')
def __init__(self, child: FrameGenerator, new_duration: float):
tus.check(new_duration=(new_duration, (int, float)),
child=(child, FrameGenerator))
if new_duration <= 0:
raise ValueError(f'new_duration={new_duration} must be positive')
self.new_duration = float(new_duration)
self.child = child
def then(self, scene: Scene) -> 'FluentScene':
"""Has the given scene follow the currently described scene"""
tus.check(scene=(scene, Scene))
if self.joined_with:
self.apply(lambda x: x)
self.followed_by.append(scene)
return self
def reshape(self, shape: typing.Tuple[int]) -> 'FluentModule':
"""Reshapes the data to the specified shape. Must correspond to the
same total number of features.
.. note::
The batch dimension is preserved.
:param shape: the new shape for the data
:type shape: tuple[int]
:returns: self
:rtype: FluentModule
"""
tus.check(shape=(shape, (list, tuple)))
tus.check_listlike(shape=(shape, int, (1, None)))
for features in shape:
if features <= 0:
raise ValueError(f'shape={shape} must be positive')
old_num_features = reduce(operator.mul, self.shape)
new_num_features = reduce(operator.mul, shape)
if old_num_features != new_num_features:
raise ValueError(
f'cannot view {self.shape} as {shape}: expected ' +
f'{old_num_features} but got {new_num_features}')
self.sequence.append(self._wrap(Reshape(*shape)))
self.shape = tuple(shape)
if self.is_verbose:
print(f' Reshape -> {self.shape}')
return self
def __init__(self,
dirname: str,
perf: typing.Optional[perf_stats.PerfStats] = None) -> None:
tus.check(dirname=(dirname, str),
perf=(perf, (type(None), perf_stats.PerfStats)))
if not perf:
perf = perf_stats.NoopPerfStats()
if not os.path.exists(dirname):
raise FileNotFoundError(dirname)
if not os.path.isdir(dirname):
raise ValueError(f'{dirname} is not a folder')
self.dirname = dirname
self.perf: perf_stats.PerfStats = perf
with open(os.path.join(dirname, META_FILE), 'r') as infile:
meta = json.load(infile)
self._len = meta['length']
largest_nbytes = meta['largest_state_nbytes']
self.padded_size = ((largest_nbytes + 4 + 4095) //
4096) * 4096 # need 4 bytes for length
self._shuffle(1)
self.handle = open(self._shuffle_path(1), 'rb', buffering=0)
self.shuffle_counter = 1
self.marks = []
self._block = bytearray(self.padded_size)
self._blockmv = memoryview(self._block)
def __init__(self, projection_vectors: np.ndarray,
projected_samples: np.ndarray,
projected_sample_labels: np.ndarray):
tus.check_ndarrays(
projection_vectors=(
projection_vectors,
('og_size', 'proj_size'),
('float32', 'float64')
),
projected_samples=(
projected_samples,
('samples', ('proj_size', projection_vectors.shape[1])),
projection_vectors.dtype
)
)
tus.check(projected_sample_labels=(
projected_sample_labels, np.ndarray))
if projected_sample_labels.shape[0] != projected_samples.shape[0]:
raise ValueError(
'projected_sample_labels should have shape (samples, ...)'
+ f'where samples={projected_samples.shape[0]}'
+ '=projected_samples.shape[0], but has shape '
+ str(projected_sample_labels.shape))
self.projection_vectors = projection_vectors
self.projected_samples = projected_samples
self.projected_sample_labels = projected_sample_labels
def dense(self, out_features: int, bias: bool = True) -> 'FluentModule':
"""A dense layer, also known as a linear layer or a fully connected
layer. A dense layer requires that this already be in flattened
form, i.e., len(self.shape) == 1.
:param out_features: the number of neurons to project to
:param bias: determines if a bias (additive) term is applied to each
of the output features
:type out_features: int
:type bias: bool
:returns: self
:rtype: FluentModule
"""
tus.check(out_features=(out_features, int), bias=(bias, bool))
if out_features <= 0:
raise ValueError(f'out_features={out_features} must be positive')
if len(self.shape) != 1:
raise ValueError(
f'cannot perform operation {self.shape} -> dense -> ' +
f'{out_features} (current shape is not flat). consider ' +
'calling flatten() first')
self.sequence.append(
self._wrap(nn.Linear(self.shape[0], out_features, bias)))
self.shape = (out_features, )
if self.is_verbose:
print(f' Linear -> {self.shape}')
return self
def __init__(self, frequency: float, indeps: np.ndarray):
tus.check(frequency=(frequency, (int, float)))
tus.check_ndarrays(
indeps=(indeps, ('n_samples',), ('float32', 'float64'))
)
self.frequency = frequency
self.indeps = indeps
def transfer_up(self, new_hidden_size, new_output_dim) -> 'EncoderRNN':
"""Returns a new encoder rnn with the specified hidden size and this network embedded
into it"""
tus.check(new_hidden_size=(new_hidden_size, int), new_output_dim=(new_output_dim, int))
if new_hidden_size < self.hidden_size:
raise ValueError(f'cannot transfer to hidden size {new_hidden_size} from {self.hidden_size}')
if new_output_dim < self.output_dim:
raise ValueError(f'cannot transfer to output dim {new_output_dim} from {self.output_dim}')
copy = EncoderRNN(self.input_dim, new_hidden_size, new_output_dim, self.num_layers)
copy.in_interpreter.weight.data[:self.hidden_size, :] = self.in_interpreter.weight.data
copy.in_interpreter.bias.data[:self.hidden_size] = self.in_interpreter.bias.data
copy.gru.weight_ih_l0.data[:3*self.hidden_size, :self.hidden_size] = self.gru.weight_ih_l0.data
copy.gru.bias_ih_l0.data[:3*self.hidden_size] = self.gru.bias_ih_l0.data
new_out_w = copy.out_interpreter.weight.data
old_out_w = self.out_interpreter.weight.data
new_out_w = new_out_w.view(new_output_dim, self.num_layers + 1, new_hidden_size)
old_out_w = old_out_w.view(self.output_dim, self.num_layers + 1, self.hidden_size)
new_out_w[:self.output_dim, :, :self.hidden_size] = old_out_w
copy.out_interpreter.bias.data[:self.output_dim] = self.out_interpreter.bias.data
return copy
def format_loan_table(loans: List[Loan], include_id=False):
"""Format the given list of loans into a markdown table.
Arguments:
loans (list[Loan]): The list of loans to format into a table.
include_id (bool): True if the id of the loan should be included in
the table, false otherwise
Returns:
(str) The markdown formatted table
"""
tus.check(loans=(loans, (tuple, list)), include_id=(include_id, bool))
tus.check_listlike(loans=(loans, Loan))
result_lines = [
'Lender|Borrower|Amount Given|Amount Repaid|Unpaid?|Original Thread'
+ '|Date Given|Date Paid Back' + ('|id' if include_id else ''),
':--|:--|:--|:--|:--|:--|:--|:--' + ('|:--' if include_id else '')
]
line_fmt = '|'.join('{' + a + '}' for a in (
'lender', 'borrower', 'principal', 'principal_repayment',
'unpaid_bool', 'permalink', 'created_at_pretty', 'repaid_at_pretty',
*(['id'] if include_id else [])
))
for loan in loans:
loan_dict = loan.dict().copy()
loan_dict['permalink'] = loan_dict.get('permalink', '')
loan_dict['unpaid_bool'] = '***UNPAID***' if loan.unpaid_at is not None else ''
loan_dict['created_at_pretty'] = loan.created_at.strftime('%b %d, %Y')
loan_dict['repaid_at_pretty'] = (
loan.repaid_at.strftime('%b %d, %Y') if loan.repaid_at is not None else ''
)
result_lines.append(line_fmt.format(**loan_dict))
return '\n'.join(result_lines)
def __init__(self,
trajectory: ProjectedTrajectory,
default_styling: typing.Tuple[typing.Tuple[np.ndarray, dict]],
title: str = 'Projected Trajectory',
zoom: np.ndarray = None,
rotation: typing.Tuple[float, float] = (30, 45),
visible_points: typing.Tuple[typing.Tuple[np.ndarray,
np.ndarray,
dict]] = None):
if visible_points is None:
visible_points = ((trajectory.snapshots[0].projected_samples,
np.ones(trajectory.num_samples,
dtype='bool'), dict()), )
if zoom is None:
zoom = get_square_bounds_for_all(pts for (pts, mask,
d) in visible_points)
tus.check(trajectory=(trajectory, ProjectedTrajectory),
default_styling=(default_styling, (list, tuple)),
title=(title, str),
rotation=(rotation, (list, tuple)),
visible_points=(visible_points, (list, tuple)))
tus.check_ndarrays(zoom=(zoom, (3, 2), ('float32', 'float64')))
tus.check_listlike(rotation=(rotation, (int, float), 2))
self.trajectory = trajectory
self.default_styling = default_styling
self.title = title
self.zoom = zoom
self.rotation = rotation
self.visible_points = visible_points
def transpose(self, dim1: int, dim2: int) -> 'FluentModule':
"""Transposes the two specified dimensions, where dimension 0 is the
first dimension after the batch dimension (i.e., really index 0
in self.shape).
:Example:
from torchluent import FluentModule
import torch
net = FluentModule((1, 12, 24)).transpose(0, 2).build()
inp = torch.randn((5, 1, 12, 24))
out = net(inp)
print(out.shape) # torch.Size[5, 12, 24, 1]
:returns: self
:rtype: FluentModule
"""
tus.check(dim1=(dim1, int), dim2=(dim2, int))
if not 0 <= dim1 < len(self.shape) or not 0 <= dim2 < len(self.shape):
raise ValueError(f'cannot transpose {dim1} and {dim2} for ' +
f'shape {self.shape}')
self.sequence.append(self._wrap(Transpose(dim1, dim2)))
newshape = list(self.shape)
tmp = newshape[dim1]
newshape[dim1] = newshape[dim2]
newshape[dim2] = tmp
self.shape = list(newshape)
if self.is_verbose:
print(f' Transpose[{dim1}, {dim2}] -> {self.shape}')
return self
def crop(self, start: float, end: float, unit: str) -> 'FluentFG':
"""Crops this to be in the given subsection of time, where time
is given in the specific unit.
Args:
start (float): the start time of the currently described video
end (float): the end time of the currently described video
unit (str): the unit of time, i.e. 'ms', 's', 'h', 'd'. For
all possible keys, see list(pympanim.utils.UNITS)
"""
tus.check(start=(start, (int, float)),
end=(end, (int, float)),
unit=(unit, str))
if unit not in mutils.UNITS_LOOKUP:
raise ValueError(
f'unit={unit} must be one of {list(mutils.UNITS_LOOKUP)}')
ms_per_unit = mutils.UNITS_LOOKUP[unit]
start *= ms_per_unit
end *= ms_per_unit
if start < 0:
raise ValueError(f'start={start} must be positive')
if start >= end:
raise ValueError(f'start={start} must be before end={end}')
if end >= self.base.duration:
raise ValueError(
f'end={end} is after current duration={self.base.duration}')
return self.apply(CroppedFrameGenerator, start, end)
def operator(self, oper, *args, **kwargs) -> 'FluentModule':
"""An operator is some operation which does not change the shape of the
data. The operator may be specified as a string, in which it should be
a module in torch.nn, or it may be the module itself which has not yet
be initialized (i.e. 'ReLU' or nn.ReLU but not nn.ReLU())
:Example:
.. code-block:: python
from torchluent import FluentModule
net = (
FluentModule(28*28)
.dense(10)
.operator('LeakyReLU', negative_slope=0.05)
.build()
)
:param oper: the name of the operator or a callable which returns one
:param args: passed to the operator
:param kwargs: passed to the operator
:returns: self
:rtype: FluentModule
"""
if isinstance(oper, str):
if not hasattr(nn, oper):
raise ValueError(f'torch.nn has no attribute {oper}')
oper = getattr(nn, oper)
mod = oper(*args, **kwargs)
if self.is_verbose:
print(f' {type(mod).__name__}')
tus.check(**{'oper(*args, **kwargs)': (mod, nn.Module)})
self.sequence.append(self._wrap(mod))
return self
def __init__(self, child: FrameGenerator, playback_rate: float):
tus.check(playback_rate=(playback_rate, (int, float)),
child=(child, FrameGenerator))
if playback_rate <= 0:
raise ValueError(f'playback_rate={playback_rate} must be positive')
self.playback_rate = float(playback_rate)
self.child = child
def get_back_off(self, num_failed_requests):
tus.check(num_failed_requests=(num_failed_requests, int))
if num_failed_requests <= 0:
raise ValueError('Backoff only makes sense after failed requests!')
if num_failed_requests <= len(self.steps):
return self.steps[num_failed_requests - 1]
return None
def consume(self, text, offset):
tus.check(text=(text, str), offset=(offset, int))
for child in self.children:
num_consumed, val = child.consume(text, offset)
if num_consumed is not None:
return (num_consumed, val)
return None, None
def test_check_torchintisint_fail(self):
try:
import torch
except ImportError:
return
with self.assertRaises(ValueError):
tus.check(a=(torch.tensor([5], dtype=torch.int32), float))
def test_check_npfloatisfloat_fail(self):
try:
import numpy as np
except ImportError:
return
with self.assertRaises(ValueError):
tus.check(a=(np.float32(5), float))
def time_rescale(self, playback_rate: float) -> 'FluentFG':
"""Rescales time to play back at the given rate. For example,
a playback_rate of 2 means that the final video will complete in
50% of the time.
"""
tus.check(playback_rate=(playback_rate, (int, float)))
return self.apply(TimeRescaleFrameGenerator, playback_rate)
def __init__(self, features: int, batch_size: int = 1024):
super().__init__()
tus.check(features=(features, int), batch_size=(batch_size, int))
self.features = features
self.batch_size = batch_size
self.history = []
self.current = None
self.current_len = 0
def __init__(self, dim1: int, dim2: int):
super().__init__()
tus.check(dim1=(dim1, int), dim2=(dim2, int))
if dim1 < 0:
raise ValueError(f'dim1={dim1} must be nonnegative')
if dim2 < 0:
raise ValueError(f'dim2={dim2} must be nonnegative')
self.dim1 = dim1
self.dim2 = dim2
def __init__(self, features: int, mult: torch.tensor, add: torch.tensor):
super().__init__()
tus.check(features=(features, int))
tus.check_tensors(mult=(mult, (('features', features), ),
(torch.float, torch.double)),
add=(add, (('features', features), ), mult.dtype))
self.features = features
self.mult = torch.nn.Parameter(mult)
self.add = torch.nn.Parameter(add)
def __init__(self, child: Scene, playback_rate: float) -> None:
tus.check(child=(child, Scene),
playback_rate=(playback_rate, (int, float)))
if playback_rate <= 0:
raise ValueError(
f'playback_rate={playback_rate} should be positive')
self.child = child
self.playback_rate = playback_rate
def overlay(self, overlay: FrameGenerator, pos: typing.Tuple[int, int]):
"""Overlays the current frame generator with the given one at the given
position. Note that the overlayed frame generator must have the same
duration as the current one and must fit entirely within the frame
"""
tus.check(overlay=(overlay, FrameGenerator), pos=(pos, (list, tuple)))
tus.check_listlike(pos=(pos, int, 2))
return self.apply(OverlayFrameGenerator, overlay, pos)
def __init__(self, dirname: str, exist_ok: bool = False):
tus.check(dirname=(dirname, str))
self.dirname = dirname
if not os.path.exists(dirname):
os.makedirs(dirname, exist_ok=True)
self.handle = open(os.path.join(self.dirname, EXPERIENCES_FILE),
'wb')
self._len = 0
self._flen = 0
self._largest_state_nbytes = 0
self._write_meta()
elif not os.path.isdir(dirname):
raise ValueError(
f'dirname must be a path to a directory, but {dirname} ' +
f'exists and is not a directory')
else:
with open(os.path.join(self.dirname, META_FILE), 'r') as infile:
meta = json.load(infile)
self._len = meta['length']
self._flen = meta['file_length']
self._largest_state_nbytes = meta['largest_state_nbytes']
expfile = os.path.join(self.dirname, EXPERIENCES_FILE)
exp_len = os.path.getsize(expfile)
if self._flen != exp_len:
import warnings
warnings.warn(
f'experiences file corrupted - written up to {exp_len}, ' +
f'valid up to {self._flen}. will discard bad data')
if exp_len < self._flen:
raise ValueError(
f'cannot recover experience file smaller than ' +
f'expected (got {exp_len}, expected {self._flen})')
counter = 1
cp_loc = os.path.join(
self.dirname,
EXPERIENCES_FILE + '.' + str(counter) + '.corrupted')
while os.path.exists(cp_loc):
counter += 1
cp_loc = os.path.join(
self.dirname,
EXPERIENCES_FILE + '.' + str(counter) + '.corrupted')
os.rename(expfile, cp_loc)
with open(cp_loc, 'rb') as infile:
with open(expfile, 'wb') as outfile:
bytes_rem = self._flen
while bytes_rem >= 4096:
outfile.write(infile.read(4096))
bytes_rem = bytes_rem - 4096
if bytes_rem > 0:
outfile.write(infile.read(bytes_rem))
self._write_meta()
self.handle = open(expfile, 'ab')
def __init__(self, epoch_size: int, input_dim: int, output_dim: int):
tus.check(epoch_size=(epoch_size, int),
input_dim=(input_dim, int),
output_dim=(output_dim, int))
self.epoch_size = epoch_size
self.input_dim = input_dim
self.output_dim = output_dim
self.__position = 0
self.mark_stack = []
