def __exit__(self, exc_type, exc_value, exc_traceback): """ Defines the exit protocol from asyncio task. Args: exc_type (Type): The type of the exception. exc_value (RuntimeError): The value of the exception, typically RuntimeError. exc_traceback (traceback): The traceback that can be printed for this exception, detailing where error actually happend. Returns: Neuron: present instance of Neuron. """ self.stop() if exc_value: top_stack = StringIO() tb.print_stack(file=top_stack) top_lines = top_stack.getvalue().strip('\n').split('\n')[:-4] top_stack.close() full_stack = StringIO() full_stack.write('Traceback (most recent call last):\n') full_stack.write('\n'.join(top_lines)) full_stack.write('\n') tb.print_tb(exc_traceback, file=full_stack) full_stack.write('{}: {}'.format(exc_type.__name__, str(exc_value))) sinfo = full_stack.getvalue() full_stack.close() # Log the combined stack logger.error('Exception:{}'.format(sinfo)) if rollbar.is_enabled(): rollbar.send_exception() return self
def backward(ctx, grads: torch.FloatTensor, code: torch.FloatTensor) -> Optional[torch.Tensor]: """ Internal autograd-friendly Backward RPC call to a remote neuron (calls the Backward method on an remote Axon terminal.) Args: ctx: (:obj:`torch.autograd.ctx`, `required`): Autograd context, saves state information between forward and backward calls. i.e. inputs for gradient computation. grads (:obj:`List[torch.Tensor]` of shape :obj:`(shape)`, `required`): Gradients of this function's outputs computed during the loss.backward() call. code (:obj:`bittensor.proto.Modality` of shape :obj:`(1)`, `required`): Code output from the forward call. Returns: output (:obj:`Tuple[torch.FloatTensor`, torch.LongTensor]`, `optional`): Gradients of the inputs with respect to the inputs and grads of the outputs. """ # ---- Zeros response in the case of failure ---- zeros = nill_response_for(ctx.inputs) # ---- Check if are passing gradients ---- if not ctx.caller.config.receptor.pass_gradients: return (None, None, zeros, None) # ---- Check that forward query was a success ---- if code.item() != bittensor.proto.ReturnCode.Success: return (None, None, zeros, None) # ---- Try to pass gradients ---- else: try: # ---- Get forward call serialzied inputs ---- try: serialized_inputs = ctx.serialized_inputs except: logger.trace( 'backward failed because forward previously failed.') return (None, None, zeros, None) # ---- Serialization ---- try: # ---- Get serializer ---- serializer = serialization.get_serializer( bittensor.proto.Serializer.MSGPACK) # ---- Serialize grads to bitensor_pb2.Tensors ---- serialized_grads = serializer.serialize( grads, modality=bittensor.proto.Modality.TENSOR, from_type=bittensor.proto.TensorType.TORCH) except Exception as e: logger.trace( 'backward failed during serialization of gradients.') return (None, None, zeros, None) # ---- Build request for backward ---- request = bittensor.proto.TensorMessage( version=bittensor.__version__, public_key=ctx.caller.wallet.hotkey.public_key, nounce=ctx.caller.nounce, signature=ctx.caller.signature, tensors=[serialized_inputs, serialized_grads]) # --- Send non blocking grad request ---- # NOTE(const): we dont care about the response. try: ctx.caller.stats.backward_qps.update(1) ctx.caller.stats.backwar_bytes_out.update( sys.getsizeof(request)) ctx.caller.stub.Backward.future( request, timeout=ctx.caller.config.receptor.timeout) ctx.caller.stats.backwar_bytes_in.update( 0.0) # responses are dropped. except: logger.trace( 'backward failed during backward call. Do not care.') return (None, None, zeros, None) # ---- Always return zeros ---- # NOTE(const): We can return non zeros but a remote host could mess with your training # without you knowing about it. i.e. by passing you malicious gradients. return (None, None, zeros, None) except: # ---- Catch all exceptions in Backward ---- rollbar.send_exception() return (None, None, zeros, None)