def weights(self): r""" Prints an weights to screen. """ console = bittensor.__console__ subtensor = bittensor.subtensor( config = self.config ) metagraph = bittensor.metagraph( subtensor = subtensor ) wallet = bittensor.wallet( config = self.config ) with console.status(":satellite: Syncing with chain: [white]{}[/white] ...".format(self.config.subtensor.network)): metagraph.load() metagraph.sync() metagraph.save() table = Table() rows = [] table.add_column("[bold white]uid", style='white', no_wrap=False) for uid in metagraph.uids.tolist(): table.add_column("[bold white]{}".format(uid), style='white', no_wrap=False) if self.config.all_weights: rows.append(["[bold white]{}".format(uid) ] + ['{:.3f}'.format(v) for v in metagraph.W[uid].tolist()]) else: if metagraph.coldkeys[uid] == wallet.coldkeypub.ss58_address: if not self.config.all_hotkeys: if metagraph.hotkeys[uid] == wallet.hotkey.ss58_address: rows.append(["[bold white]{}".format(uid) ] + ['{:.3f}'.format(v) for v in metagraph.W[uid].tolist()]) else: rows.append(["[bold white]{}".format(uid) ] + ['{:.3f}'.format(v) for v in metagraph.W[uid].tolist()]) for row in rows: table.add_row(*row) table.box = None table.pad_edge = False table.width = None with console.pager(): console.print(table)
def test_get_last_emit_data_for_uid__success(setup_chain): wallet = generate_wallet() subtensor = setup_subtensor(setup_chain) metagraph = bittensor.metagraph(subtensor=subtensor) subtensor.register(wallet=wallet) uid = subtensor.get_uid_for_hotkey(wallet.hotkey.ss58_address) metagraph.sync() result = metagraph.last_update[uid] current_block = subtensor.get_current_block() assert result.item() <= current_block
def test_get_active(setup_chain): wallet = generate_wallet() subtensor = setup_subtensor(setup_chain) metagraph = bittensor.metagraph(subtensor=subtensor) subtensor.register(wallet=wallet) metagraph.sync() result = metagraph.active assert isinstance(result, Tensor) assert len(result) > 0 result = metagraph.hotkeys elem = result[0] elem = "0x" + ss58_decode(elem) assert isinstance(elem[0], str) assert elem[:2] == "0x" assert len(elem[2:]) == 64
def __init__( self, config: 'bittensor.config', nucleus: 'Nucleus'): r""" Initializes the neuron with the passed config. """ self.config = config self.wallet = bittensor.wallet ( config = self.config ) self.subtensor = bittensor.subtensor ( config = self.config ) self.metagraph = bittensor.metagraph ( config = self.config, subtensor = self.subtensor ) self.dendrite = bittensor.dendrite ( config = self.config, wallet = self.wallet ) self.dataset = bittensor.dataset ( config = self.config ) self.axon = bittensor.axon ( config = self.config, wallet = self.wallet, forward_text = self.forward_text, backward_text = self.backward_text, blacklist = self.blacklist, ) self.device = torch.device( device = self.config.neuron.device ) self.nucleus = nucleus.to(self.device) self.nucleus.metagraph = self.metagraph_callback self.nucleus.dendrite = self.dendrite self.optimizer = torch.optim.SGD( [ {'params': self.nucleus.peer_weights, 'lr': self.config.neuron.learning_rate_chain} ], lr = self.config.neuron.learning_rate, momentum = self.config.neuron.momentum, ) self.scheduler = torch.optim.lr_scheduler.StepLR(self.optimizer, step_size = 1.0, gamma = 0.95 ) self.stats = SimpleNamespace( global_step = 0, last_sync_block = 0, epoch_data_size = 0, epoch_sync_count = 0, local_target_epoch_loss = math.inf, distillation_epoch_loss = math.inf, remote_target_epoch_loss = math.inf, local_epoch_acc = 0, best_epoch_loss = math.inf, scores = torch.nn.Parameter(torch.zeros(0), requires_grad = False).to(self.device), ema_scores = torch.nn.Parameter(torch.zeros(0), requires_grad = False).to(self.device) ) # ---- Decay factor for fisher ema score self.fisher_ema_decay = 0.995
def __init__(self, config: 'bittensor.config' = None): if config == None: config = neuron.config() config = config self.check_config(config) bittensor.logging( config=config, logging_dir=config.neuron.full_path, ) self.config = config print(config) # Load/Create our bittensor wallet. self.wallet = bittensor.wallet(config=config).create().register() # Connect to the chain. self.subtensor = bittensor.subtensor(config=config) # Load/Sync/Save our metagraph. self.metagraph = bittensor.metagraph( subtensor=self.subtensor).load().sync().save() self.uid = self.metagraph.hotkeys.index( self.wallet.hotkey.ss58_address) # Create Dendrite. self.dendrite = bittensor.dendrite(config=config) # Load genesis dataset. self.dataset = bittensor.dataset(config=config) # Build Device. self.device = torch.device(device=config.neuron.device) self.nucleus = Validator(config=config, metagraph=self.metagraph_callback, dendrite=self.dendrite, device=self.device)
def test_indexed_values_to_dataframe(setup_chain): subtensor, port = setup_subtensor(setup_chain) wallet = generate_wallet() wallet.register(subtensor=subtensor) nn = subtensor.neuron_for_pubkey(wallet.hotkey.ss58_address) metagraph = bittensor.metagraph(subtensor=subtensor) metagraph.sync() idx_df = bittensor.utils.indexed_values_to_dataframe( prefix='w_i_{}'.format(nn.uid), index=[nn.uid], values=metagraph.W[:, nn.uid]) assert idx_df.values[0][0] == 1.0 idx_df = bittensor.utils.indexed_values_to_dataframe( prefix=nn.uid, index=[nn.uid], values=metagraph.W[:, nn.uid]) assert idx_df.values[0][0] == 1.0 idx_df = bittensor.utils.indexed_values_to_dataframe( prefix=nn.uid, index=torch.LongTensor([nn.uid]), values=metagraph.W[:, nn.uid]) assert idx_df.values[0][0] == 1.0 # Need to check for errors with pytest.raises(ValueError): idx_df = bittensor.utils.indexed_values_to_dataframe( prefix=b'w_i', index=[nn.uid], values=metagraph.W[:, nn.uid]) with pytest.raises(ValueError): idx_df = bittensor.utils.indexed_values_to_dataframe( prefix='w_i_{}'.format(nn.uid), index=nn.uid, values=metagraph.W[:, nn.uid]) with pytest.raises(ValueError): idx_df = bittensor.utils.indexed_values_to_dataframe( prefix='w_i_{}'.format(nn.uid), index=[nn.uid], values=nn.uid)
def test_create(): global metagraph metagraph = bittensor.metagraph(network='nobunaga')
def test_metagraph(): metagraph = bittensor.metagraph() metagraph.to_wandb()
def serve(config, model): config.to_defaults() # Create Subtensor connection subtensor = bittensor.subtensor(config=config) # Load/Create our bittensor wallet. wallet = bittensor.wallet(config=config).create().register() # Load/Sync/Save our metagraph. metagraph = bittensor.metagraph(subtensor=bittensor.subtensor( config=config)).load().sync().save() # Create our optimizer. optimizer = torch.optim.SGD( [{ "params": model.parameters() }], lr=config.neuron.learning_rate, momentum=config.neuron.momentum, ) mutex = Lock() def forward_text(inputs_x): r""" Single threaded version of the Forward function that is called when the axon recieves a forward request from other peers """ return model.encode_forward(inputs_x) def backward_text(inputs_x, grads_dy): r"""Single threaded backwards function that is called when the axon recieves a backwards request from other peers. Updates the server parameters with gradients through the chain. """ if config.neuron.training: with mutex: with torch.enable_grad(): with torch.autograd.set_detect_anomaly(True): outputs_y = model.encode_forward(inputs_x) torch.autograd.backward(tensors=[outputs_y], grad_tensors=[grads_dy]) optimizer.step() optimizer.zero_grad() # Create our axon server and subscribe it to the network. axon = bittensor.axon( wallet=wallet, forward_text=forward_text, backward_text=backward_text, ).start().serve(subtensor=subtensor) if config.wandb.api_key != 'default': # --- Init Wandb. bittensor.wandb(config=config, cold_pubkey=wallet.coldkeypub.ss58_address, hot_pubkey=wallet.hotkey.ss58_address, root_dir=config.neuron.full_path) last_set_block = subtensor.get_current_block() # --- Run Forever. while True: current_block = subtensor.get_current_block() end_block = current_block + config.neuron.blocks_per_epoch while end_block >= current_block: time.sleep(bittensor.__blocktime__) current_block = subtensor.get_current_block() nn = subtensor.neuron_for_pubkey(wallet.hotkey.ss58_address) uid = metagraph.hotkeys.index(wallet.hotkey.ss58_address) wandb_data = { 'stake': nn.stake, 'rank': nn.rank, 'trust': nn.trust, 'consensus': nn.consensus, 'incentive': nn.incentive, 'emission': nn.emission, } bittensor.__console__.print('[green]Current Status:[/green]', wandb_data) if config.wandb.api_key != 'default': df = pandas.concat([ bittensor.utils.indexed_values_to_dataframe( prefix='w_i_{}'.format(nn.uid), index=metagraph.uids, values=metagraph.W[:, uid]), axon.to_dataframe(metagraph=metagraph), ], axis=1) df['uid'] = df.index wandb_info_axon = axon.to_wandb() wandb.log({**wandb_data, **wandb_info_axon}, step=current_block) wandb.log({'stats': wandb.Table(dataframe=df)}, step=current_block) if current_block - last_set_block > config.neuron.blocks_per_set_weights: try: last_set_block = current_block # Set self weights to maintain activity. chain_weights = torch.zeros(metagraph.n) chain_weights[uid] = 1 did_set = subtensor.set_weights( uids=metagraph.uids, weights=chain_weights, wait_for_inclusion=False, wallet=wallet, ) if did_set: logger.success('Successfully set weights on the chain') else: logger.error('Failed to set weights on chain. (Timeout)') except Exception as e: logger.error('Failure setting weights on chain with error: {}', e)
def serve(config, gp_server): config.to_defaults() # Create Subtensor connection subtensor = bittensor.subtensor(config=config) # Load/Create our bittensor wallet. wallet = bittensor.wallet(config=config).create().register() # Load/Sync/Save our metagraph. metagraph = bittensor.metagraph(subtensor=subtensor).load().sync().save() # Instantiate the model we are going to serve on the network. # Creating a threading lock for updates to the model mutex = Lock() gp_server = gp_server.to(gp_server.device) # Create our optimizer. optimizer = torch.optim.SGD( [{ "params": gp_server.parameters() }], lr=config.neuron.learning_rate, momentum=config.neuron.momentum, ) timecheck = {} # Define our forward function. def forward_text(inputs_x): r""" Forward function that is called when the axon recieves a forward request from other peers Args: inputs_x ( :obj:`torch.Tensor`, `required`): torch inputs to be forward processed. Returns: outputs (:obj:`torch.FloatTensor`): The nucleus's outputs as a torch tensor of shape [batch_size, sequence_len, __network_dim__] """ return gp_server.encode_forward(inputs_x.to(gp_server.device)) # Define our backward function. def backward_text(inputs_x, grads_dy): r"""Backwards function that is called when the axon recieves a backwards request from other peers. Updates the server parameters with gradients through the chain. Args: inputs_x ( :obj:`torch.Tensor`, `required`): torch inputs from previous forward call. grads_dy ( :obj:`torch.Tensor`, `required`): torch grads of forward output. """ # -- normalized grads -- grads_dy = grads_dy / (grads_dy.sum() + 0.00001) with mutex: outputs_y = gp_server.encode_forward(inputs_x.to(gp_server.device)) with torch.autograd.set_detect_anomaly(True): torch.autograd.backward( tensors=[outputs_y], grad_tensors=[grads_dy.to(gp_server.device)], retain_graph=True) logger.info('Backwards axon gradient applied') gp_server.backward_gradients += inputs_x.size(0) def priority(pubkey: str, request_type: bittensor.proto.RequestType, inputs_x) -> float: r"""Calculates the priority on requests based on stake and size of input Args: pubkey ( str, `required`): The public key of the caller. inputs_x ( :obj:`torch.Tensor`, `required`): torch inputs to be forward processed. request_type ( bittensor.proto.RequestType, `required`): the request type ('FORWARD' or 'BACKWARD'). """ uid = metagraph.hotkeys.index(pubkey) priority = metagraph.S[uid].item() / sys.getsizeof(inputs_x) return priority def blacklist(pubkey: str, request_type: bittensor.proto.RequestType) -> bool: r"""Axon security blacklisting, used to blacklist message from low stake members Args: pubkey ( str, `required`): The public key of the caller. request_type ( bittensor.proto.RequestType, `required`): the request type ('FORWARD' or 'BACKWARD'). """ # Check for stake def stake_check() -> bool: # If we allow non-registered requests return False = not blacklisted. is_registered = pubkey in metagraph.hotkeys if not is_registered: if config.neuron.blacklist_allow_non_registered: return False else: return True # Check stake. uid = metagraph.hotkeys.index(pubkey) if request_type == bittensor.proto.RequestType.FORWARD: if metagraph.S[uid].item( ) < config.neuron.blacklist.stake.forward: return True else: return False elif request_type == bittensor.proto.RequestType.BACKWARD: if metagraph.S[uid].item( ) < config.neuron.blacklist.stake.backward: return True else: return False # Check for time def time_check(): current_time = datetime.now() if pubkey in timecheck.keys(): prev_time = timecheck[pubkey] if current_time - prev_time >= timedelta( seconds=config.neuron.blacklist.time): timecheck[pubkey] = current_time return False else: timecheck[pubkey] = current_time return True else: timecheck[pubkey] = current_time return False # Black list or not if stake_check() or time_check(): return True else: return False # Create our axon server axon = bittensor.axon(wallet=wallet, forward_text=forward_text, backward_text=backward_text, blacklist=blacklist, priority=priority) # Training Data dataset = bittensor.dataset(config=config) # load our old model if config.neuron.no_restart != True: gp_server.load(config.neuron.full_path) if config.wandb.api_key != 'default': # --- Init Wandb. bittensor.wandb(config=config, cold_pubkey=wallet.coldkeypub.ss58_address, hot_pubkey=wallet.hotkey.ss58_address, root_dir=config.neuron.full_path) nn = subtensor.neuron_for_pubkey(wallet.hotkey.ss58_address) # --- last sync block last_sync_block = subtensor.get_current_block() last_set_block = last_sync_block # -- Main Training loop -- try: # -- download files from the mountain data = next(dataset) # --- creating our chain weights chain_weights = torch.zeros(metagraph.n) uid = nn.uid chain_weights[uid] = 1 # -- serve axon to the network. axon.start().serve(subtensor=subtensor) while True: # --- Run current_block = subtensor.get_current_block() end_block = current_block + config.neuron.blocks_per_epoch interation = 0 # --- Training step. while end_block >= current_block: if current_block != subtensor.get_current_block(): loss, _ = gp_server(next(dataset).to(gp_server.device)) if interation > 0: losses += loss else: losses = loss interation += 1 current_block = subtensor.get_current_block() #Custom learning rate if gp_server.backward_gradients > 0: optimizer.param_groups[0]['lr'] = 1 / ( gp_server.backward_gradients) else: optimizer.param_groups[0]['lr'] = 0.1 # --- Update parameters if interation != 0 or gp_server.backward_gradients != 0: with mutex: logger.info('Backpropagation Started') if interation != 0: losses.backward() clip_grad_norm_(gp_server.parameters(), 1.0) optimizer.step() optimizer.zero_grad() logger.info('Backpropagation Successful: Model updated') nn = subtensor.neuron_for_pubkey(wallet.hotkey.ss58_address) gp_server.backward_gradients = 0 # --- logging data wandb_data = { 'block': end_block, 'loss': losses.cpu().item() / interation, 'stake': nn.stake, 'rank': nn.rank, 'incentive': nn.incentive, 'trust': nn.trust, 'consensus': nn.consensus, 'incentive': nn.incentive, 'dividends': nn.dividends, 'emission': nn.emission, } bittensor.__console__.print('[green]Current Status:[/green]', wandb_data) # Add additional wandb data for axon, metagraph etc. if config.wandb.api_key != 'default': df = pandas.concat([ bittensor.utils.indexed_values_to_dataframe( prefix='w_i_{}'.format(nn.uid), index=metagraph.uids, values=metagraph.W[:, uid]), bittensor.utils.indexed_values_to_dataframe( prefix='s_i'.format(nn.uid), index=metagraph.uids, values=metagraph.S), axon.to_dataframe(metagraph=metagraph), ], axis=1) df['uid'] = df.index stats_data_table = wandb.Table(dataframe=df) wandb_info_axon = axon.to_wandb() wandb.log({ **wandb_data, **wandb_info_axon }, step=current_block) wandb.log({'stats': stats_data_table}, step=current_block) wandb.log({ 'axon_query_times': wandb.plot.scatter(stats_data_table, "uid", "axon_query_time", title="Axon Query time by UID") }) wandb.log({ 'in_weights': wandb.plot.scatter(stats_data_table, "uid", 'w_i_{}'.format(nn.uid), title="Inward weights by UID") }) wandb.log({ 'stake': wandb.plot.scatter(stats_data_table, "uid", 's_i', title="Stake by UID") }) # Save the model gp_server.save(config.neuron.full_path) if current_block - last_set_block > config.neuron.blocks_per_set_weights: # --- Setting weights try: last_set_block = current_block # Set self weights to maintain activity. chain_weights = torch.zeros(metagraph.n) chain_weights[uid] = 1 did_set = subtensor.set_weights( uids=metagraph.uids, weights=chain_weights, wait_for_inclusion=False, wallet=wallet, ) if did_set: logger.success('Successfully set weights on the chain') else: logger.error( 'Failed to set weights on chain. (Timeout)') except Exception as e: logger.error( 'Failure setting weights on chain with error: {}', e) if current_block - last_sync_block > config.neuron.metagraph_sync: metagraph.sync() last_sync_block = current_block except KeyboardInterrupt: # --- User ended session ---- axon.stop() except Exception as e: # --- Unknown error ---- logger.exception('Unknown exception: {} with traceback {}', e, traceback.format_exc())
def metagraph(self): r""" Prints an entire metagraph. """ console = bittensor.__console__ subtensor = bittensor.subtensor( config = self.config ) metagraph = bittensor.metagraph( subtensor = subtensor ) console.print(":satellite: Syncing with chain: [white]{}[/white] ...".format(self.config.subtensor.network)) metagraph.sync() metagraph.save() issuance = subtensor.total_issuance difficulty = subtensor.difficulty TABLE_DATA = [] total_stake = 0.0 total_rank = 0.0 total_trust = 0.0 total_consensus = 0.0 total_incentive = 0.0 total_dividends = 0.0 total_emission = 0 for uid in metagraph.uids: ep = metagraph.endpoint_objs[uid] row = [ str(ep.uid), '{:.5f}'.format( metagraph.stake[uid]), '{:.5f}'.format( metagraph.ranks[uid]), '{:.5f}'.format( metagraph.trust[uid]), '{:.5f}'.format( metagraph.consensus[uid]), '{:.5f}'.format( metagraph.incentive[uid]), '{:.5f}'.format( metagraph.dividends[uid]), '{}'.format( int(metagraph.emission[uid] * 1000000000)), str((metagraph.block.item() - metagraph.last_update[uid].item())), str( metagraph.active[uid].item() ), ep.ip + ':' + str(ep.port) if ep.is_serving else '[yellow]none[/yellow]', ep.hotkey[:10], ep.coldkey[:10] ] total_stake += metagraph.stake[uid] total_rank += metagraph.ranks[uid] total_trust += metagraph.trust[uid] total_consensus += metagraph.consensus[uid] total_incentive += metagraph.incentive[uid] total_dividends += metagraph.dividends[uid] total_emission += int(metagraph.emission[uid] * 1000000000) TABLE_DATA.append(row) total_neurons = len(metagraph.uids) table = Table(show_footer=False) table.title = ( "[white]Metagraph: name: {}, block: {}, N: {}/{}, tau: {}/block, stake: {}, issuance: {}, difficulty: {}".format(subtensor.network, metagraph.block.item(), sum(metagraph.active.tolist()), metagraph.n.item(), bittensor.Balance.from_tao(metagraph.tau.item()), bittensor.Balance.from_tao(total_stake), issuance, difficulty ) ) table.add_column("[overline white]UID", str(total_neurons), footer_style = "overline white", style='yellow') table.add_column("[overline white]STAKE(\u03C4)", '\u03C4{:.5f}'.format(total_stake), footer_style = "overline white", justify='right', style='green', no_wrap=True) table.add_column("[overline white]RANK", '{:.5f}'.format(total_rank), footer_style = "overline white", justify='right', style='green', no_wrap=True) table.add_column("[overline white]TRUST", '{:.5f}'.format(total_trust), footer_style = "overline white", justify='right', style='green', no_wrap=True) table.add_column("[overline white]CONSENSUS", '{:.5f}'.format(total_consensus), footer_style = "overline white", justify='right', style='green', no_wrap=True) table.add_column("[overline white]INCENTIVE", '{:.5f}'.format(total_incentive), footer_style = "overline white", justify='right', style='green', no_wrap=True) table.add_column("[overline white]DIVIDENDS", '{:.5f}'.format(total_dividends), footer_style = "overline white", justify='right', style='green', no_wrap=True) table.add_column("[overline white]EMISSION(\u03C1)", '\u03C1{}'.format(int(total_emission)), footer_style = "overline white", justify='right', style='green', no_wrap=True) table.add_column("[overline white]UPDATED", justify='right', no_wrap=True) table.add_column("[overline white]ACTIVE", justify='right', style='green', no_wrap=True) table.add_column("[overline white]AXON", justify='left', style='dim blue', no_wrap=True) table.add_column("[overline white]HOTKEY", style='dim blue', no_wrap=False) table.add_column("[overline white]COLDKEY", style='dim purple', no_wrap=False) table.show_footer = True for row in TABLE_DATA: table.add_row(*row) table.box = None table.pad_edge = False table.width = None console.print(table)