def render_mpl(self, act_state: state.ProjectedState): """Returns a matplotlib figure with the state on it""" fig = plt.figure(figsize=self.frame_size_in) ax = fig.add_subplot(111, projection='3d') axtitle = ax.set_title(act_state.title) font_size = int((80 / 1920) * self._frame_size[0]) axtitle.set_fontsize(font_size) renderer = fig.canvas.get_renderer() bb = axtitle.get_window_extent(renderer=renderer) while bb.width >= self._frame_size[0] * 0.9: font_size -= 5 axtitle.set_fontsize(font_size) bb = axtitle.get_window_extent(renderer=renderer) for pts, mask, skwargs in act_state.visible_points: tus.check(skwargs=(skwargs, dict)) self._scatter(act_state, ax, pts, mask, skwargs) ax.set_xlim(*[float(i) for i in act_state.zoom[0]]) ax.set_ylim(*[float(i) for i in act_state.zoom[1]]) ax.set_zlim(*[float(i) for i in act_state.zoom[2]]) ax.view_init(*act_state.rotation) return fig
def __init__(self, samples: np.ndarray, centers: np.ndarray, labels: np.ndarray, calculate_params: typing.Dict[str, typing.Any]): tus.check(samples=(samples, np.ndarray), centers=(centers, np.ndarray), labels=(labels, np.ndarray), calculate_params=(calculate_params, dict)) tus.check_ndarrays( samples=(samples, ('n_samples', 'n_features'), (np.dtype('float32'), np.dtype('float64'))), centers=(centers, ('n_clusters', ('n_features', samples.shape[1] if len(samples.shape) > 1 else None)), samples.dtype), labels=(labels, (('n_samples', samples.shape[0] if bool(samples.shape) else None), ), (np.dtype('int32'), np.dtype('int64')))) self.samples = samples self.centers = centers self.labels = labels self.calculate_params = calculate_params self._bounds = None
def __init__(self, base: Scene): tus.check(base=(base, Scene)) self.base = base self.followed_by = [] self.joined_with = [] self.join_is_sep = False self.pushed = []
def __init__(self, username, password): tus.check(username=(username, str), password=(password, str)) self.username = username self.password = password self._header = 'Basic ' + base64.b64encode( (self.username + ':' + self.password).encode('ascii')).decode('ascii')
def __init__(self, child: FrameGenerator, new_duration: float): tus.check(new_duration=(new_duration, (int, float)), child=(child, FrameGenerator)) if new_duration <= 0: raise ValueError(f'new_duration={new_duration} must be positive') self.new_duration = float(new_duration) self.child = child
def then(self, scene: Scene) -> 'FluentScene': """Has the given scene follow the currently described scene""" tus.check(scene=(scene, Scene)) if self.joined_with: self.apply(lambda x: x) self.followed_by.append(scene) return self
def reshape(self, shape: typing.Tuple[int]) -> 'FluentModule': """Reshapes the data to the specified shape. Must correspond to the same total number of features. .. note:: The batch dimension is preserved. :param shape: the new shape for the data :type shape: tuple[int] :returns: self :rtype: FluentModule """ tus.check(shape=(shape, (list, tuple))) tus.check_listlike(shape=(shape, int, (1, None))) for features in shape: if features <= 0: raise ValueError(f'shape={shape} must be positive') old_num_features = reduce(operator.mul, self.shape) new_num_features = reduce(operator.mul, shape) if old_num_features != new_num_features: raise ValueError( f'cannot view {self.shape} as {shape}: expected ' + f'{old_num_features} but got {new_num_features}') self.sequence.append(self._wrap(Reshape(*shape))) self.shape = tuple(shape) if self.is_verbose: print(f' Reshape -> {self.shape}') return self
def __init__(self, dirname: str, perf: typing.Optional[perf_stats.PerfStats] = None) -> None: tus.check(dirname=(dirname, str), perf=(perf, (type(None), perf_stats.PerfStats))) if not perf: perf = perf_stats.NoopPerfStats() if not os.path.exists(dirname): raise FileNotFoundError(dirname) if not os.path.isdir(dirname): raise ValueError(f'{dirname} is not a folder') self.dirname = dirname self.perf: perf_stats.PerfStats = perf with open(os.path.join(dirname, META_FILE), 'r') as infile: meta = json.load(infile) self._len = meta['length'] largest_nbytes = meta['largest_state_nbytes'] self.padded_size = ((largest_nbytes + 4 + 4095) // 4096) * 4096 # need 4 bytes for length self._shuffle(1) self.handle = open(self._shuffle_path(1), 'rb', buffering=0) self.shuffle_counter = 1 self.marks = [] self._block = bytearray(self.padded_size) self._blockmv = memoryview(self._block)
def __init__(self, projection_vectors: np.ndarray, projected_samples: np.ndarray, projected_sample_labels: np.ndarray): tus.check_ndarrays( projection_vectors=( projection_vectors, ('og_size', 'proj_size'), ('float32', 'float64') ), projected_samples=( projected_samples, ('samples', ('proj_size', projection_vectors.shape[1])), projection_vectors.dtype ) ) tus.check(projected_sample_labels=( projected_sample_labels, np.ndarray)) if projected_sample_labels.shape[0] != projected_samples.shape[0]: raise ValueError( 'projected_sample_labels should have shape (samples, ...)' + f'where samples={projected_samples.shape[0]}' + '=projected_samples.shape[0], but has shape ' + str(projected_sample_labels.shape)) self.projection_vectors = projection_vectors self.projected_samples = projected_samples self.projected_sample_labels = projected_sample_labels
def dense(self, out_features: int, bias: bool = True) -> 'FluentModule': """A dense layer, also known as a linear layer or a fully connected layer. A dense layer requires that this already be in flattened form, i.e., len(self.shape) == 1. :param out_features: the number of neurons to project to :param bias: determines if a bias (additive) term is applied to each of the output features :type out_features: int :type bias: bool :returns: self :rtype: FluentModule """ tus.check(out_features=(out_features, int), bias=(bias, bool)) if out_features <= 0: raise ValueError(f'out_features={out_features} must be positive') if len(self.shape) != 1: raise ValueError( f'cannot perform operation {self.shape} -> dense -> ' + f'{out_features} (current shape is not flat). consider ' + 'calling flatten() first') self.sequence.append( self._wrap(nn.Linear(self.shape[0], out_features, bias))) self.shape = (out_features, ) if self.is_verbose: print(f' Linear -> {self.shape}') return self
def __init__(self, frequency: float, indeps: np.ndarray): tus.check(frequency=(frequency, (int, float))) tus.check_ndarrays( indeps=(indeps, ('n_samples',), ('float32', 'float64')) ) self.frequency = frequency self.indeps = indeps
def transfer_up(self, new_hidden_size, new_output_dim) -> 'EncoderRNN': """Returns a new encoder rnn with the specified hidden size and this network embedded into it""" tus.check(new_hidden_size=(new_hidden_size, int), new_output_dim=(new_output_dim, int)) if new_hidden_size < self.hidden_size: raise ValueError(f'cannot transfer to hidden size {new_hidden_size} from {self.hidden_size}') if new_output_dim < self.output_dim: raise ValueError(f'cannot transfer to output dim {new_output_dim} from {self.output_dim}') copy = EncoderRNN(self.input_dim, new_hidden_size, new_output_dim, self.num_layers) copy.in_interpreter.weight.data[:self.hidden_size, :] = self.in_interpreter.weight.data copy.in_interpreter.bias.data[:self.hidden_size] = self.in_interpreter.bias.data copy.gru.weight_ih_l0.data[:3*self.hidden_size, :self.hidden_size] = self.gru.weight_ih_l0.data copy.gru.bias_ih_l0.data[:3*self.hidden_size] = self.gru.bias_ih_l0.data new_out_w = copy.out_interpreter.weight.data old_out_w = self.out_interpreter.weight.data new_out_w = new_out_w.view(new_output_dim, self.num_layers + 1, new_hidden_size) old_out_w = old_out_w.view(self.output_dim, self.num_layers + 1, self.hidden_size) new_out_w[:self.output_dim, :, :self.hidden_size] = old_out_w copy.out_interpreter.bias.data[:self.output_dim] = self.out_interpreter.bias.data return copy
def format_loan_table(loans: List[Loan], include_id=False): """Format the given list of loans into a markdown table. Arguments: loans (list[Loan]): The list of loans to format into a table. include_id (bool): True if the id of the loan should be included in the table, false otherwise Returns: (str) The markdown formatted table """ tus.check(loans=(loans, (tuple, list)), include_id=(include_id, bool)) tus.check_listlike(loans=(loans, Loan)) result_lines = [ 'Lender|Borrower|Amount Given|Amount Repaid|Unpaid?|Original Thread' + '|Date Given|Date Paid Back' + ('|id' if include_id else ''), ':--|:--|:--|:--|:--|:--|:--|:--' + ('|:--' if include_id else '') ] line_fmt = '|'.join('{' + a + '}' for a in ( 'lender', 'borrower', 'principal', 'principal_repayment', 'unpaid_bool', 'permalink', 'created_at_pretty', 'repaid_at_pretty', *(['id'] if include_id else []) )) for loan in loans: loan_dict = loan.dict().copy() loan_dict['permalink'] = loan_dict.get('permalink', '') loan_dict['unpaid_bool'] = '***UNPAID***' if loan.unpaid_at is not None else '' loan_dict['created_at_pretty'] = loan.created_at.strftime('%b %d, %Y') loan_dict['repaid_at_pretty'] = ( loan.repaid_at.strftime('%b %d, %Y') if loan.repaid_at is not None else '' ) result_lines.append(line_fmt.format(**loan_dict)) return '\n'.join(result_lines)
def __init__(self, trajectory: ProjectedTrajectory, default_styling: typing.Tuple[typing.Tuple[np.ndarray, dict]], title: str = 'Projected Trajectory', zoom: np.ndarray = None, rotation: typing.Tuple[float, float] = (30, 45), visible_points: typing.Tuple[typing.Tuple[np.ndarray, np.ndarray, dict]] = None): if visible_points is None: visible_points = ((trajectory.snapshots[0].projected_samples, np.ones(trajectory.num_samples, dtype='bool'), dict()), ) if zoom is None: zoom = get_square_bounds_for_all(pts for (pts, mask, d) in visible_points) tus.check(trajectory=(trajectory, ProjectedTrajectory), default_styling=(default_styling, (list, tuple)), title=(title, str), rotation=(rotation, (list, tuple)), visible_points=(visible_points, (list, tuple))) tus.check_ndarrays(zoom=(zoom, (3, 2), ('float32', 'float64'))) tus.check_listlike(rotation=(rotation, (int, float), 2)) self.trajectory = trajectory self.default_styling = default_styling self.title = title self.zoom = zoom self.rotation = rotation self.visible_points = visible_points
def transpose(self, dim1: int, dim2: int) -> 'FluentModule': """Transposes the two specified dimensions, where dimension 0 is the first dimension after the batch dimension (i.e., really index 0 in self.shape). :Example: from torchluent import FluentModule import torch net = FluentModule((1, 12, 24)).transpose(0, 2).build() inp = torch.randn((5, 1, 12, 24)) out = net(inp) print(out.shape) # torch.Size[5, 12, 24, 1] :returns: self :rtype: FluentModule """ tus.check(dim1=(dim1, int), dim2=(dim2, int)) if not 0 <= dim1 < len(self.shape) or not 0 <= dim2 < len(self.shape): raise ValueError(f'cannot transpose {dim1} and {dim2} for ' + f'shape {self.shape}') self.sequence.append(self._wrap(Transpose(dim1, dim2))) newshape = list(self.shape) tmp = newshape[dim1] newshape[dim1] = newshape[dim2] newshape[dim2] = tmp self.shape = list(newshape) if self.is_verbose: print(f' Transpose[{dim1}, {dim2}] -> {self.shape}') return self
def crop(self, start: float, end: float, unit: str) -> 'FluentFG': """Crops this to be in the given subsection of time, where time is given in the specific unit. Args: start (float): the start time of the currently described video end (float): the end time of the currently described video unit (str): the unit of time, i.e. 'ms', 's', 'h', 'd'. For all possible keys, see list(pympanim.utils.UNITS) """ tus.check(start=(start, (int, float)), end=(end, (int, float)), unit=(unit, str)) if unit not in mutils.UNITS_LOOKUP: raise ValueError( f'unit={unit} must be one of {list(mutils.UNITS_LOOKUP)}') ms_per_unit = mutils.UNITS_LOOKUP[unit] start *= ms_per_unit end *= ms_per_unit if start < 0: raise ValueError(f'start={start} must be positive') if start >= end: raise ValueError(f'start={start} must be before end={end}') if end >= self.base.duration: raise ValueError( f'end={end} is after current duration={self.base.duration}') return self.apply(CroppedFrameGenerator, start, end)
def operator(self, oper, *args, **kwargs) -> 'FluentModule': """An operator is some operation which does not change the shape of the data. The operator may be specified as a string, in which it should be a module in torch.nn, or it may be the module itself which has not yet be initialized (i.e. 'ReLU' or nn.ReLU but not nn.ReLU()) :Example: .. code-block:: python from torchluent import FluentModule net = ( FluentModule(28*28) .dense(10) .operator('LeakyReLU', negative_slope=0.05) .build() ) :param oper: the name of the operator or a callable which returns one :param args: passed to the operator :param kwargs: passed to the operator :returns: self :rtype: FluentModule """ if isinstance(oper, str): if not hasattr(nn, oper): raise ValueError(f'torch.nn has no attribute {oper}') oper = getattr(nn, oper) mod = oper(*args, **kwargs) if self.is_verbose: print(f' {type(mod).__name__}') tus.check(**{'oper(*args, **kwargs)': (mod, nn.Module)}) self.sequence.append(self._wrap(mod)) return self
def __init__(self, child: FrameGenerator, playback_rate: float): tus.check(playback_rate=(playback_rate, (int, float)), child=(child, FrameGenerator)) if playback_rate <= 0: raise ValueError(f'playback_rate={playback_rate} must be positive') self.playback_rate = float(playback_rate) self.child = child
def get_back_off(self, num_failed_requests): tus.check(num_failed_requests=(num_failed_requests, int)) if num_failed_requests <= 0: raise ValueError('Backoff only makes sense after failed requests!') if num_failed_requests <= len(self.steps): return self.steps[num_failed_requests - 1] return None
def consume(self, text, offset): tus.check(text=(text, str), offset=(offset, int)) for child in self.children: num_consumed, val = child.consume(text, offset) if num_consumed is not None: return (num_consumed, val) return None, None
def test_check_torchintisint_fail(self): try: import torch except ImportError: return with self.assertRaises(ValueError): tus.check(a=(torch.tensor([5], dtype=torch.int32), float))
def test_check_npfloatisfloat_fail(self): try: import numpy as np except ImportError: return with self.assertRaises(ValueError): tus.check(a=(np.float32(5), float))
def time_rescale(self, playback_rate: float) -> 'FluentFG': """Rescales time to play back at the given rate. For example, a playback_rate of 2 means that the final video will complete in 50% of the time. """ tus.check(playback_rate=(playback_rate, (int, float))) return self.apply(TimeRescaleFrameGenerator, playback_rate)
def __init__(self, features: int, batch_size: int = 1024): super().__init__() tus.check(features=(features, int), batch_size=(batch_size, int)) self.features = features self.batch_size = batch_size self.history = [] self.current = None self.current_len = 0
def __init__(self, dim1: int, dim2: int): super().__init__() tus.check(dim1=(dim1, int), dim2=(dim2, int)) if dim1 < 0: raise ValueError(f'dim1={dim1} must be nonnegative') if dim2 < 0: raise ValueError(f'dim2={dim2} must be nonnegative') self.dim1 = dim1 self.dim2 = dim2
def __init__(self, features: int, mult: torch.tensor, add: torch.tensor): super().__init__() tus.check(features=(features, int)) tus.check_tensors(mult=(mult, (('features', features), ), (torch.float, torch.double)), add=(add, (('features', features), ), mult.dtype)) self.features = features self.mult = torch.nn.Parameter(mult) self.add = torch.nn.Parameter(add)
def __init__(self, child: Scene, playback_rate: float) -> None: tus.check(child=(child, Scene), playback_rate=(playback_rate, (int, float))) if playback_rate <= 0: raise ValueError( f'playback_rate={playback_rate} should be positive') self.child = child self.playback_rate = playback_rate
def overlay(self, overlay: FrameGenerator, pos: typing.Tuple[int, int]): """Overlays the current frame generator with the given one at the given position. Note that the overlayed frame generator must have the same duration as the current one and must fit entirely within the frame """ tus.check(overlay=(overlay, FrameGenerator), pos=(pos, (list, tuple))) tus.check_listlike(pos=(pos, int, 2)) return self.apply(OverlayFrameGenerator, overlay, pos)
def __init__(self, dirname: str, exist_ok: bool = False): tus.check(dirname=(dirname, str)) self.dirname = dirname if not os.path.exists(dirname): os.makedirs(dirname, exist_ok=True) self.handle = open(os.path.join(self.dirname, EXPERIENCES_FILE), 'wb') self._len = 0 self._flen = 0 self._largest_state_nbytes = 0 self._write_meta() elif not os.path.isdir(dirname): raise ValueError( f'dirname must be a path to a directory, but {dirname} ' + f'exists and is not a directory') else: with open(os.path.join(self.dirname, META_FILE), 'r') as infile: meta = json.load(infile) self._len = meta['length'] self._flen = meta['file_length'] self._largest_state_nbytes = meta['largest_state_nbytes'] expfile = os.path.join(self.dirname, EXPERIENCES_FILE) exp_len = os.path.getsize(expfile) if self._flen != exp_len: import warnings warnings.warn( f'experiences file corrupted - written up to {exp_len}, ' + f'valid up to {self._flen}. will discard bad data') if exp_len < self._flen: raise ValueError( f'cannot recover experience file smaller than ' + f'expected (got {exp_len}, expected {self._flen})') counter = 1 cp_loc = os.path.join( self.dirname, EXPERIENCES_FILE + '.' + str(counter) + '.corrupted') while os.path.exists(cp_loc): counter += 1 cp_loc = os.path.join( self.dirname, EXPERIENCES_FILE + '.' + str(counter) + '.corrupted') os.rename(expfile, cp_loc) with open(cp_loc, 'rb') as infile: with open(expfile, 'wb') as outfile: bytes_rem = self._flen while bytes_rem >= 4096: outfile.write(infile.read(4096)) bytes_rem = bytes_rem - 4096 if bytes_rem > 0: outfile.write(infile.read(bytes_rem)) self._write_meta() self.handle = open(expfile, 'ab')
def __init__(self, epoch_size: int, input_dim: int, output_dim: int): tus.check(epoch_size=(epoch_size, int), input_dim=(input_dim, int), output_dim=(output_dim, int)) self.epoch_size = epoch_size self.input_dim = input_dim self.output_dim = output_dim self.__position = 0 self.mark_stack = []