Example #1
0
def test_axon_receptor_connection_backward_timeout():
    def backward(inputs_x: torch.FloatTensor, grads):
        if inputs_x.size() == (1, 1, 1):
            return None
        else:
            raise TimeoutError('Timeout')

    axon = bittensor.axon(
        backward_tensor=backward,
        port=8088,
        ip='127.0.0.1',
        wallet=wallet,
    )
    axon.start()

    endpoint = bittensor.endpoint(version=bittensor.__version_as_int__,
                                  uid=0,
                                  ip='127.0.0.1',
                                  ip_type=4,
                                  port=8088,
                                  hotkey=wallet.hotkey.ss58_address,
                                  coldkey=wallet.coldkey.ss58_address,
                                  modality=2)

    receptor = bittensor.receptor(
        endpoint=endpoint,
        wallet=wallet,
    )
    x = torch.rand(3, 3, bittensor.__network_dim__)
    out, ops, time = receptor.backward(x,
                                       x,
                                       bittensor.proto.Modality.TENSOR,
                                       timeout=1)
    assert ops == bittensor.proto.ReturnCode.Timeout
    axon.stop()
Example #2
0
def test_axon_receptor_connection_backward_unauthenticated():
    def backward(inputs_x: torch.FloatTensor, grads):
        return torch.zeros([3, 3, bittensor.__network_dim__])

    axon = bittensor.axon(
        backward_tensor=backward,
        port=8090,
        ip='127.0.0.1',
        wallet=wallet,
    )
    axon.start()

    endpoint = bittensor.endpoint(version=bittensor.__version_as_int__,
                                  uid=0,
                                  ip='127.0.0.1',
                                  ip_type=4,
                                  port=8090,
                                  hotkey=wallet.hotkey.ss58_address,
                                  coldkey=wallet.coldkey.ss58_address,
                                  modality=2)

    receptor = bittensor.receptor(
        endpoint=endpoint,
        wallet=wallet,
    )

    x = torch.rand(3, 3, bittensor.__network_dim__)
    receptor.sign = MagicMock(return_value='mock')
    out, ops, time = receptor.backward(x,
                                       x,
                                       bittensor.proto.Modality.TENSOR,
                                       timeout=1)
    assert ops == bittensor.proto.ReturnCode.Unauthenticated
    axon.stop()
Example #3
0
def test_axon_receptor_connection_backward_unimplemented():
    def backward(inputs_x: torch.FloatTensor, grads):
        return torch.zeros([3, 3, bittensor.__network_dim__])

    axon = bittensor.axon(
        backward_tensor=backward,
        port=8086,
        ip='127.0.0.1',
        wallet=wallet,
    )
    axon.start()
    endpoint = bittensor.endpoint(version=bittensor.__version_as_int__,
                                  uid=0,
                                  ip='127.0.0.1',
                                  ip_type=4,
                                  port=8086,
                                  hotkey=wallet.hotkey.ss58_address,
                                  coldkey=wallet.coldkey.ss58_address,
                                  modality=2)

    receptor = bittensor.receptor(
        endpoint=endpoint,
        wallet=wallet,
    )

    x = torch.rand(3, 3)
    grads = torch.rand(3, 3, bittensor.__network_dim__)
    out, ops, time = receptor.backward(x,
                                       grads,
                                       bittensor.proto.Modality.TEXT,
                                       timeout=1)
    assert ops == bittensor.proto.ReturnCode.NotImplemented
    axon.stop()
Example #4
0
def test_axon_receptor_forward_works():
    def forward(inputs_x: torch.FloatTensor):
        time.sleep(0.2)
        return torch.zeros([3, 3, bittensor.__network_dim__])

    axon = bittensor.axon(
        port=8080,
        ip='0.0.0.0',
        wallet=wallet,
    )
    axon.attach_forward_callback(forward,
                                 modality=bittensor.proto.Modality.TENSOR)
    axon.start()
    endpoints = []
    for i in range(5):
        endpoint = bittensor.endpoint(version=bittensor.__version_as_int__,
                                      uid=1,
                                      hotkey=str(i),
                                      ip='0.0.0.0',
                                      ip_type=4,
                                      port=8080,
                                      modality=0,
                                      coldkey='')
        endpoints += [endpoint]
    dendrite = bittensor.dendrite(max_active_receptors=500)
    x = torch.rand(3, 3, bittensor.__network_dim__, dtype=torch.float32)
    tensors, codes, times = dendrite.forward_tensor(
        endpoints=endpoints, inputs=[x for i in endpoints])
    for i in dendrite.receptor_pool.receptors:
        assert (dendrite.receptor_pool.receptors[i].state() ==
                dendrite.receptor_pool.receptors[i].state().READY)
    assert codes[0].item() == bittensor.proto.ReturnCode.Success
    assert list(tensors[0].shape) == [3, 3, bittensor.__network_dim__]
Example #5
0
def test_axon_receptor_connection_forward_works():
    def forward(inputs_x: torch.FloatTensor):
        return torch.zeros([3, 3, bittensor.__network_dim__])

    axon = bittensor.axon(
        forward_tensor=forward,
        port=8081,
        ip='127.0.0.1',
        wallet=wallet,
    )
    axon.start()

    endpoint = bittensor.endpoint(version=bittensor.__version_as_int__,
                                  uid=0,
                                  ip='127.0.0.1',
                                  ip_type=4,
                                  port=8081,
                                  hotkey=wallet.hotkey.ss58_address,
                                  coldkey=wallet.coldkey.ss58_address,
                                  modality=2)

    receptor = bittensor.receptor(
        endpoint=endpoint,
        wallet=wallet,
    )

    x = torch.rand(3, 3, bittensor.__network_dim__)
    out, ops, time = receptor.forward(x,
                                      bittensor.proto.Modality.TENSOR,
                                      timeout=1)
    assert ops == bittensor.proto.ReturnCode.Success
    axon.stop()
Example #6
0
def test_backward_response_success_text_priority():
    def priority(pubkey: str, request_type: str, inputs_x):
        return 100

    axon = bittensor.axon(wallet=wallet, priority=priority)

    def backward(inputs_x: torch.FloatTensor, grads_dy: torch.FloatTensor):
        return torch.zeros([1, 1])

    axon.attach_backward_callback(backward,
                                  modality=bittensor.proto.Modality.TEXT)
    inputs_raw = torch.ones((1, 1))
    grads_raw = torch.zeros((1, 1, bittensor.__network_dim__))
    serializer = bittensor.serializer(
        serialzer_type=bittensor.proto.Serializer.MSGPACK)
    inputs_serialized = serializer.serialize(
        inputs_raw,
        modality=bittensor.proto.Modality.TEXT,
        from_type=bittensor.proto.TensorType.TORCH)
    grads_serialized = serializer.serialize(
        grads_raw,
        modality=bittensor.proto.Modality.TEXT,
        from_type=bittensor.proto.TensorType.TORCH)
    request = bittensor.proto.TensorMessage(
        version=bittensor.__version_as_int__,
        hotkey=axon.wallet.hotkey.ss58_address,
        tensors=[inputs_serialized, grads_serialized])
    response, code, call_time, message = axon._backward(request)
    assert code == bittensor.proto.ReturnCode.Success
Example #7
0
def test_axon_is_destroyed():
    port = 8081
    assert is_port_in_use(port) == False
    axon = bittensor.axon(port=port)
    assert is_port_in_use(port) == True
    axon.start()
    assert is_port_in_use(port) == True
    axon.stop()
    assert is_port_in_use(port) == False
    axon.__del__()
    assert is_port_in_use(port) == False

    port = 8082
    assert is_port_in_use(port) == False
    axon2 = bittensor.axon(port=port)
    assert is_port_in_use(port) == True
    axon2.start()
    assert is_port_in_use(port) == True
    axon2.__del__()
    assert is_port_in_use(port) == False

    port_3 = 8086
    assert is_port_in_use(port_3) == False
    axonA = bittensor.axon(port=port_3)
    assert is_port_in_use(port_3) == True
    axonB = bittensor.axon(port=port_3)
    assert axonA.server != axonB.server
    assert is_port_in_use(port_3) == True
    axonA.start()
    assert is_port_in_use(port_3) == True
    axonB.start()
    assert is_port_in_use(port_3) == True
    axonA.__del__()
    assert is_port_in_use(port) == False
    axonB.__del__()
    assert is_port_in_use(port) == False
Example #8
0
def test_grpc_backward_works():
    def backward(inputs_x: torch.FloatTensor, grads_dy: torch.FloatTensor):
        return torch.zeros([1, 1, 1])

    axon = bittensor.axon(
        port=7081,
        ip='127.0.0.1',
        wallet=wallet,
    )
    axon.attach_backward_callback(backward,
                                  modality=bittensor.proto.Modality.TENSOR)
    axon.start()

    channel = grpc.insecure_channel('127.0.0.1:7081',
                                    options=[
                                        ('grpc.max_send_message_length', -1),
                                        ('grpc.max_receive_message_length', -1)
                                    ])
    stub = bittensor.grpc.BittensorStub(channel)

    inputs_raw = torch.rand(3, 3, bittensor.__network_dim__)
    grads_raw = torch.rand(3, 3, bittensor.__network_dim__)
    serializer = bittensor.serializer(
        serialzer_type=bittensor.proto.Serializer.MSGPACK)
    inputs_serialized = serializer.serialize(
        inputs_raw,
        modality=bittensor.proto.Modality.TENSOR,
        from_type=bittensor.proto.TensorType.TORCH)
    grads_serialized = serializer.serialize(
        grads_raw,
        modality=bittensor.proto.Modality.TENSOR,
        from_type=bittensor.proto.TensorType.TORCH)
    request = bittensor.proto.TensorMessage(
        version=bittensor.__version_as_int__,
        hotkey='1092310312914',
        tensors=[inputs_serialized, grads_serialized])
    response = stub.Backward(request,
                             metadata=(
                                 ('rpc-auth-header', 'Bittensor'),
                                 ('bittensor-signature', sign(axon.wallet)),
                                 ('bittensor-version',
                                  str(bittensor.__version_as_int__)),
                             ))
    outputs = serializer.deserialize(response.tensors[0],
                                     to_type=bittensor.proto.TensorType.TORCH)
    assert outputs.tolist() == [[[0]]]
    axon.stop()
Example #9
0
 def __init__( self, config: 'bittensor.config', nucleus: 'Nucleus'):
     r""" Initializes the neuron with the passed config.
     """
     self.config = config
     self.wallet = bittensor.wallet ( config = self.config )
     self.subtensor = bittensor.subtensor ( config = self.config )
     self.metagraph = bittensor.metagraph ( config = self.config, subtensor = self.subtensor )
     self.dendrite = bittensor.dendrite ( config = self.config, wallet = self.wallet )
     self.dataset = bittensor.dataset ( config = self.config )
     self.axon = bittensor.axon (
         config = self.config,
         wallet = self.wallet,
         forward_text = self.forward_text,
         backward_text = self.backward_text,
         blacklist = self.blacklist,
     )
     self.device = torch.device( device = self.config.neuron.device )
     self.nucleus = nucleus.to(self.device)
     self.nucleus.metagraph = self.metagraph_callback
     self.nucleus.dendrite = self.dendrite
     self.optimizer = torch.optim.SGD(
         [ {'params': self.nucleus.peer_weights, 'lr': self.config.neuron.learning_rate_chain} ],
         lr = self.config.neuron.learning_rate,
         momentum = self.config.neuron.momentum,
     )
     self.scheduler = torch.optim.lr_scheduler.StepLR(self.optimizer,
         step_size = 1.0,
         gamma = 0.95
     )
     self.stats = SimpleNamespace(
         global_step = 0,
         last_sync_block = 0,
         epoch_data_size = 0,
         epoch_sync_count = 0,
         local_target_epoch_loss = math.inf,
         distillation_epoch_loss = math.inf,
         remote_target_epoch_loss = math.inf,
         local_epoch_acc = 0,
         best_epoch_loss = math.inf,
         scores = torch.nn.Parameter(torch.zeros(0), requires_grad = False).to(self.device),
         ema_scores = torch.nn.Parameter(torch.zeros(0), requires_grad = False).to(self.device)
     )
     # ---- Decay factor for fisher ema score 
     self.fisher_ema_decay = 0.995
Example #10
0
def test_grpc_backward_fails():
    def backward(inputs_x: torch.FloatTensor, grads_dy: torch.FloatTensor):
        return torch.zeros([1, 1, 1])

    axon = bittensor.axon(port=7081, ip='127.0.0.1', wallet=wallet)
    axon.attach_backward_callback(backward,
                                  modality=bittensor.proto.Modality.TENSOR)
    axon.start()

    channel = grpc.insecure_channel('127.0.0.1:7081',
                                    options=[
                                        ('grpc.max_send_message_length', -1),
                                        ('grpc.max_receive_message_length', -1)
                                    ])
    stub = bittensor.grpc.BittensorStub(channel)

    inputs_raw = torch.rand(3, 3, bittensor.__network_dim__)
    grads_raw = torch.rand(3, 3, bittensor.__network_dim__)
    serializer = bittensor.serializer(
        serialzer_type=bittensor.proto.Serializer.MSGPACK)
    inputs_serialized = serializer.serialize(
        inputs_raw,
        modality=bittensor.proto.Modality.TENSOR,
        from_type=bittensor.proto.TensorType.TORCH)
    grads_serialized = serializer.serialize(
        grads_raw,
        modality=bittensor.proto.Modality.TENSOR,
        from_type=bittensor.proto.TensorType.TORCH)
    request = bittensor.proto.TensorMessage(
        version=bittensor.__version_as_int__,
        hotkey='1092310312914',
        tensors=[inputs_serialized, grads_serialized])

    try:
        response = stub.Backward(request)
    except grpc.RpcError as rpc_error_call:
        grpc_code = rpc_error_call.code()
        assert grpc_code == grpc.StatusCode.UNAUTHENTICATED

    axon.stop()
Example #11
0
def test_forward_tensor_success_priority():
    def priority(pubkey: str, request_type: str, inputs_x):
        return 100

    axon = bittensor.axon(wallet=wallet, priority=priority)

    def forward(inputs_x: torch.FloatTensor):
        return torch.zeros(
            [inputs_x.shape[0], inputs_x.shape[1], bittensor.__network_dim__])

    axon.attach_forward_callback(forward, modality=2)
    inputs_raw = torch.rand(3, 3, bittensor.__network_dim__)
    serializer = bittensor.serializer(
        serialzer_type=bittensor.proto.Serializer.MSGPACK)
    inputs_serialized = serializer.serialize(
        inputs_raw,
        modality=bittensor.proto.Modality.TENSOR,
        from_type=bittensor.proto.TensorType.TORCH)
    request = bittensor.proto.TensorMessage(
        version=bittensor.__version_as_int__, tensors=[inputs_serialized])
    response, code, call_time, message = axon._forward(request)
    assert code == bittensor.proto.ReturnCode.Success
Example #12
0
def test_axon():
    axon = bittensor.axon()
    axon.to_wandb()
Example #13
0
def serve(config, model):
    config.to_defaults()

    # Create Subtensor connection
    subtensor = bittensor.subtensor(config=config)

    # Load/Create our bittensor wallet.
    wallet = bittensor.wallet(config=config).create().register()

    # Load/Sync/Save our metagraph.
    metagraph = bittensor.metagraph(subtensor=bittensor.subtensor(
        config=config)).load().sync().save()

    # Create our optimizer.
    optimizer = torch.optim.SGD(
        [{
            "params": model.parameters()
        }],
        lr=config.neuron.learning_rate,
        momentum=config.neuron.momentum,
    )
    mutex = Lock()

    def forward_text(inputs_x):
        r""" Single threaded version of the Forward function that is called when the axon recieves a forward request from other peers
        """
        return model.encode_forward(inputs_x)

    def backward_text(inputs_x, grads_dy):
        r"""Single threaded backwards function that is called when the axon recieves a backwards request from other peers.
            Updates the server parameters with gradients through the chain.             
        """
        if config.neuron.training:
            with mutex:
                with torch.enable_grad():
                    with torch.autograd.set_detect_anomaly(True):
                        outputs_y = model.encode_forward(inputs_x)
                        torch.autograd.backward(tensors=[outputs_y],
                                                grad_tensors=[grads_dy])
                        optimizer.step()
                        optimizer.zero_grad()

    # Create our axon server and subscribe it to the network.
    axon = bittensor.axon(
        wallet=wallet,
        forward_text=forward_text,
        backward_text=backward_text,
    ).start().serve(subtensor=subtensor)

    if config.wandb.api_key != 'default':
        # --- Init Wandb.
        bittensor.wandb(config=config,
                        cold_pubkey=wallet.coldkeypub.ss58_address,
                        hot_pubkey=wallet.hotkey.ss58_address,
                        root_dir=config.neuron.full_path)

    last_set_block = subtensor.get_current_block()

    # --- Run Forever.
    while True:

        current_block = subtensor.get_current_block()
        end_block = current_block + config.neuron.blocks_per_epoch
        while end_block >= current_block:
            time.sleep(bittensor.__blocktime__)
            current_block = subtensor.get_current_block()

        nn = subtensor.neuron_for_pubkey(wallet.hotkey.ss58_address)
        uid = metagraph.hotkeys.index(wallet.hotkey.ss58_address)
        wandb_data = {
            'stake': nn.stake,
            'rank': nn.rank,
            'trust': nn.trust,
            'consensus': nn.consensus,
            'incentive': nn.incentive,
            'emission': nn.emission,
        }
        bittensor.__console__.print('[green]Current Status:[/green]',
                                    wandb_data)
        if config.wandb.api_key != 'default':

            df = pandas.concat([
                bittensor.utils.indexed_values_to_dataframe(
                    prefix='w_i_{}'.format(nn.uid),
                    index=metagraph.uids,
                    values=metagraph.W[:, uid]),
                axon.to_dataframe(metagraph=metagraph),
            ],
                               axis=1)
            df['uid'] = df.index
            wandb_info_axon = axon.to_wandb()
            wandb.log({**wandb_data, **wandb_info_axon}, step=current_block)
            wandb.log({'stats': wandb.Table(dataframe=df)}, step=current_block)

        if current_block - last_set_block > config.neuron.blocks_per_set_weights:
            try:
                last_set_block = current_block
                # Set self weights to maintain activity.
                chain_weights = torch.zeros(metagraph.n)
                chain_weights[uid] = 1
                did_set = subtensor.set_weights(
                    uids=metagraph.uids,
                    weights=chain_weights,
                    wait_for_inclusion=False,
                    wallet=wallet,
                )

                if did_set:
                    logger.success('Successfully set weights on the chain')
                else:
                    logger.error('Failed to set weights on chain. (Timeout)')
            except Exception as e:
                logger.error('Failure setting weights on chain with error: {}',
                             e)
Example #14
0
def serve(config, gp_server):
    config.to_defaults()

    # Create Subtensor connection
    subtensor = bittensor.subtensor(config=config)

    # Load/Create our bittensor wallet.
    wallet = bittensor.wallet(config=config).create().register()

    # Load/Sync/Save our metagraph.
    metagraph = bittensor.metagraph(subtensor=subtensor).load().sync().save()

    # Instantiate the model we are going to serve on the network.
    # Creating a threading lock for updates to the model
    mutex = Lock()
    gp_server = gp_server.to(gp_server.device)

    # Create our optimizer.
    optimizer = torch.optim.SGD(
        [{
            "params": gp_server.parameters()
        }],
        lr=config.neuron.learning_rate,
        momentum=config.neuron.momentum,
    )

    timecheck = {}

    # Define our forward function.
    def forward_text(inputs_x):
        r""" Forward function that is called when the axon recieves a forward request from other peers
            Args:
                inputs_x ( :obj:`torch.Tensor`, `required`):
                    torch inputs to be forward processed.

            Returns:
                outputs (:obj:`torch.FloatTensor`):
                    The nucleus's outputs as a torch tensor of shape [batch_size, sequence_len, __network_dim__]
        """
        return gp_server.encode_forward(inputs_x.to(gp_server.device))

    # Define our backward function.
    def backward_text(inputs_x, grads_dy):
        r"""Backwards function that is called when the axon recieves a backwards request from other peers.
            Updates the server parameters with gradients through the chain.

            Args:
                inputs_x ( :obj:`torch.Tensor`, `required`):
                    torch inputs from previous forward call.
                grads_dy ( :obj:`torch.Tensor`, `required`):
                    torch grads of forward output.
                    
        """
        # -- normalized grads --
        grads_dy = grads_dy / (grads_dy.sum() + 0.00001)

        with mutex:
            outputs_y = gp_server.encode_forward(inputs_x.to(gp_server.device))
            with torch.autograd.set_detect_anomaly(True):
                torch.autograd.backward(
                    tensors=[outputs_y],
                    grad_tensors=[grads_dy.to(gp_server.device)],
                    retain_graph=True)
            logger.info('Backwards axon gradient applied')

        gp_server.backward_gradients += inputs_x.size(0)

    def priority(pubkey: str, request_type: bittensor.proto.RequestType,
                 inputs_x) -> float:
        r"""Calculates the priority on requests based on stake and size of input

            Args:
                pubkey ( str, `required`):
                    The public key of the caller.
                inputs_x ( :obj:`torch.Tensor`, `required`):
                    torch inputs to be forward processed.
                request_type ( bittensor.proto.RequestType, `required`):
                    the request type ('FORWARD' or 'BACKWARD').
        """
        uid = metagraph.hotkeys.index(pubkey)
        priority = metagraph.S[uid].item() / sys.getsizeof(inputs_x)

        return priority

    def blacklist(pubkey: str,
                  request_type: bittensor.proto.RequestType) -> bool:
        r"""Axon security blacklisting, used to blacklist message from low stake members
            Args:
                pubkey ( str, `required`):
                    The public key of the caller.
                request_type ( bittensor.proto.RequestType, `required`):
                    the request type ('FORWARD' or 'BACKWARD').
        """

        # Check for stake
        def stake_check() -> bool:
            # If we allow non-registered requests return False = not blacklisted.
            is_registered = pubkey in metagraph.hotkeys
            if not is_registered:
                if config.neuron.blacklist_allow_non_registered:
                    return False
                else:
                    return True

            # Check stake.
            uid = metagraph.hotkeys.index(pubkey)
            if request_type == bittensor.proto.RequestType.FORWARD:
                if metagraph.S[uid].item(
                ) < config.neuron.blacklist.stake.forward:
                    return True
                else:
                    return False

            elif request_type == bittensor.proto.RequestType.BACKWARD:
                if metagraph.S[uid].item(
                ) < config.neuron.blacklist.stake.backward:
                    return True
                else:
                    return False

        # Check for time
        def time_check():
            current_time = datetime.now()
            if pubkey in timecheck.keys():
                prev_time = timecheck[pubkey]
                if current_time - prev_time >= timedelta(
                        seconds=config.neuron.blacklist.time):
                    timecheck[pubkey] = current_time
                    return False
                else:
                    timecheck[pubkey] = current_time
                    return True
            else:
                timecheck[pubkey] = current_time
                return False

        # Black list or not
        if stake_check() or time_check():
            return True
        else:
            return False

    # Create our axon server
    axon = bittensor.axon(wallet=wallet,
                          forward_text=forward_text,
                          backward_text=backward_text,
                          blacklist=blacklist,
                          priority=priority)

    # Training Data
    dataset = bittensor.dataset(config=config)

    # load our old model
    if config.neuron.no_restart != True:
        gp_server.load(config.neuron.full_path)

    if config.wandb.api_key != 'default':
        # --- Init Wandb.
        bittensor.wandb(config=config,
                        cold_pubkey=wallet.coldkeypub.ss58_address,
                        hot_pubkey=wallet.hotkey.ss58_address,
                        root_dir=config.neuron.full_path)

    nn = subtensor.neuron_for_pubkey(wallet.hotkey.ss58_address)

    # --- last sync block
    last_sync_block = subtensor.get_current_block()
    last_set_block = last_sync_block

    # -- Main Training loop --
    try:
        # -- download files from the mountain
        data = next(dataset)

        # --- creating our chain weights
        chain_weights = torch.zeros(metagraph.n)
        uid = nn.uid
        chain_weights[uid] = 1

        # --  serve axon to the network.
        axon.start().serve(subtensor=subtensor)

        while True:
            # --- Run
            current_block = subtensor.get_current_block()
            end_block = current_block + config.neuron.blocks_per_epoch
            interation = 0

            # --- Training step.
            while end_block >= current_block:
                if current_block != subtensor.get_current_block():
                    loss, _ = gp_server(next(dataset).to(gp_server.device))
                    if interation > 0:
                        losses += loss
                    else:
                        losses = loss
                    interation += 1
                    current_block = subtensor.get_current_block()

            #Custom learning rate
            if gp_server.backward_gradients > 0:
                optimizer.param_groups[0]['lr'] = 1 / (
                    gp_server.backward_gradients)
            else:
                optimizer.param_groups[0]['lr'] = 0.1

            # --- Update parameters
            if interation != 0 or gp_server.backward_gradients != 0:
                with mutex:
                    logger.info('Backpropagation Started')
                    if interation != 0:
                        losses.backward()
                    clip_grad_norm_(gp_server.parameters(), 1.0)

                    optimizer.step()
                    optimizer.zero_grad()
                    logger.info('Backpropagation Successful: Model updated')

            nn = subtensor.neuron_for_pubkey(wallet.hotkey.ss58_address)

            gp_server.backward_gradients = 0
            # --- logging data
            wandb_data = {
                'block': end_block,
                'loss': losses.cpu().item() / interation,
                'stake': nn.stake,
                'rank': nn.rank,
                'incentive': nn.incentive,
                'trust': nn.trust,
                'consensus': nn.consensus,
                'incentive': nn.incentive,
                'dividends': nn.dividends,
                'emission': nn.emission,
            }
            bittensor.__console__.print('[green]Current Status:[/green]',
                                        wandb_data)

            # Add additional wandb data for axon, metagraph etc.
            if config.wandb.api_key != 'default':

                df = pandas.concat([
                    bittensor.utils.indexed_values_to_dataframe(
                        prefix='w_i_{}'.format(nn.uid),
                        index=metagraph.uids,
                        values=metagraph.W[:, uid]),
                    bittensor.utils.indexed_values_to_dataframe(
                        prefix='s_i'.format(nn.uid),
                        index=metagraph.uids,
                        values=metagraph.S),
                    axon.to_dataframe(metagraph=metagraph),
                ],
                                   axis=1)
                df['uid'] = df.index
                stats_data_table = wandb.Table(dataframe=df)
                wandb_info_axon = axon.to_wandb()
                wandb.log({
                    **wandb_data,
                    **wandb_info_axon
                },
                          step=current_block)
                wandb.log({'stats': stats_data_table}, step=current_block)
                wandb.log({
                    'axon_query_times':
                    wandb.plot.scatter(stats_data_table,
                                       "uid",
                                       "axon_query_time",
                                       title="Axon Query time by UID")
                })
                wandb.log({
                    'in_weights':
                    wandb.plot.scatter(stats_data_table,
                                       "uid",
                                       'w_i_{}'.format(nn.uid),
                                       title="Inward weights by UID")
                })
                wandb.log({
                    'stake':
                    wandb.plot.scatter(stats_data_table,
                                       "uid",
                                       's_i',
                                       title="Stake by UID")
                })

            # Save the model
            gp_server.save(config.neuron.full_path)

            if current_block - last_set_block > config.neuron.blocks_per_set_weights:

                # --- Setting weights
                try:
                    last_set_block = current_block
                    # Set self weights to maintain activity.
                    chain_weights = torch.zeros(metagraph.n)
                    chain_weights[uid] = 1
                    did_set = subtensor.set_weights(
                        uids=metagraph.uids,
                        weights=chain_weights,
                        wait_for_inclusion=False,
                        wallet=wallet,
                    )

                    if did_set:
                        logger.success('Successfully set weights on the chain')
                    else:
                        logger.error(
                            'Failed to set weights on chain. (Timeout)')
                except Exception as e:
                    logger.error(
                        'Failure setting weights on chain with error: {}', e)

            if current_block - last_sync_block > config.neuron.metagraph_sync:
                metagraph.sync()
                last_sync_block = current_block

    except KeyboardInterrupt:
        # --- User ended session ----
        axon.stop()
    except Exception as e:
        # --- Unknown error ----
        logger.exception('Unknown exception: {} with traceback {}', e,
                         traceback.format_exc())
Example #15
0
import bittensor
import time
import pytest
import uuid
import unittest.mock as mock

wallet = bittensor.wallet(
    path='/tmp/pytest',
    name='pytest',
    hotkey='pytest',
)

wallet.create_new_coldkey(use_password=False, overwrite=True)
wallet.create_new_hotkey(use_password=False, overwrite=True)

axon = bittensor.axon(wallet=wallet)


def sign(wallet):
    nounce = str(int(time.time() * 1000))
    receptor_uid = str(uuid.uuid1())
    message = "{}{}{}".format(nounce, str(wallet.hotkey.ss58_address),
                              receptor_uid)
    spliter = 'bitxx'
    signature = spliter.join([
        nounce,
        str(wallet.hotkey.ss58_address),
        wallet.hotkey.sign(message), receptor_uid
    ])
    return signature