def check_config(config: Munch): assert config.miner.momentum > 0 and config.miner.momentum < 1, "momentum must be a value between 0 and 1" assert config.miner.batch_size_train > 0, "batch_size_train must a positive value" assert config.miner.learning_rate > 0, "learning_rate must be a positive value." full_path = '{}/{}/{}'.format(config.miner.root_dir, config.miner.name, config.miner.trial_uid) config.miner.full_path = os.path.expanduser(full_path) if not os.path.exists(config.miner.full_path): os.makedirs(config.miner.full_path) GPT2LMSynapse.check_config(config) bittensor.neuron.Neuron.check_config(config)
def __init__(self, config: Munch = None): if config == None: config = Miner.build_config(); logger.info(bittensor.config.Config.toString(config)) self.config = config # ---- Neuron ---- self.neuron = bittensor.neuron.Neuron(self.config) # ---- Model ---- self.model = GPT2LMSynapse( self.config ) # ---- Optimizer ---- self.optimizer = torch.optim.SGD(self.model.parameters(), lr = self.config.miner.learning_rate, momentum=self.config.miner.momentum) self.scheduler = WarmupCosineWithHardRestartsSchedule(self.optimizer, 50, 300) # ---- Model Load/Save tools ---- self.model_toolbox = ModelToolbox(GPT2LMSynapse, torch.optim.SGD) # ---- Dataset ---- # Dataset: 74 million sentences pulled from books. # self.dataset = load_dataset('ag_news')['train'] self.dataset = AdamCorpus(self.config.miner.custom_datasets) # ---- Logging ---- self.tensorboard = SummaryWriter(log_dir = self.config.miner.full_path) if self.config.miner.record_log: logger.add(self.config.miner.full_path + "/{}_{}.log".format(self.config.miner.name, self.config.miner.trial_uid),format="{time:YYYY-MM-DD at HH:mm:ss} | {level} | {message}")
def __init__(self, config: Munch = None, **kwargs): if config == None: config = Miner.default_config() bittensor.config.Config.update_with_kwargs(config.miner, kwargs) Miner.check_config(config) self.config = config # ---- Neuron ---- self.neuron = bittensor.neuron.Neuron(self.config) # ---- Model ---- self.model = GPT2LMSynapse(self.config) # ---- Optimizer ---- self.optimizer = torch.optim.SGD(self.model.parameters(), lr=self.config.miner.learning_rate, momentum=self.config.miner.momentum) self.scheduler = WarmupCosineWithHardRestartsSchedule( self.optimizer, 50, 300) # ---- Model Load/Save tools ---- self.model_toolbox = ModelToolbox(GPT2LMSynapse, torch.optim.SGD) # ---- Dataset ---- # The Genesis Dataset: # The dataset used to train Adam and his first 100 children. self.dataset = AdamCorpus(self.config.miner.custom_dataset) # ---- Logging ---- self.tensorboard = SummaryWriter(log_dir=self.config.miner.full_path) if self.config.miner.record_log: logger.add( self.config.miner.full_path + "/{}_{}.log".format( self.config.miner.name, self.config.miner.trial_uid), format="{time:YYYY-MM-DD at HH:mm:ss} | {level} | {message}")
def add_args(parser: argparse.ArgumentParser): parser.add_argument('--miner.learning_rate', default=0.01, type=float, help='Training initial learning rate.') parser.add_argument('--miner.momentum', default=0.98, type=float, help='Training initial momentum for SGD.') parser.add_argument('--miner.n_epochs', default=int(sys.maxsize), type=int, help='Number of training epochs.') parser.add_argument('--miner.epoch_length', default=500, type=int, help='Iterations of training per epoch') parser.add_argument('--miner.batch_size_train', default=1, type=int, help='Training batch size.') parser.add_argument('--miner.sync_interval', default=100, type=int, help='Batches before we sync with chain and emit new weights.') parser.add_argument('--miner.log_interval', default=10, type=int, help='Batches before we log miner info.') parser.add_argument('--miner.accumulation_interval', default=1, type=int, help='Batches before we apply acummulated gradients.') parser.add_argument('--miner.apply_remote_gradients', default=False, type=bool, help='If true, neuron applies gradients which accumulate from remotes calls.') parser.add_argument('--miner.root_dir', default='~/.bittensor/miners/', type=str, help='Root path to load and save data associated with each miner') parser.add_argument('--miner.name', default='gpt-wiki', type=str, help='Trials for this miner go in miner.root / miner.name') parser.add_argument('--miner.trial_uid', default=str(time.time()).split('.')[0], type=str, help='Saved models go in miner.root_dir / miner.name / miner.uid') parser.add_argument('--miner.record_log', default=True, help='Record all logs when running this miner') parser.add_argument('--miner.config_file', type=str, help='config file to run this neuron, if not using cmd line arguments.') GPT2LMSynapse.add_args(parser) bittensor.neuron.Neuron.add_args(parser)
def __init__(self, config: Munch = None, **kwargs): if config == None: config = Miner.default_config() bittensor.config.Config.update_with_kwargs(config.miner, kwargs) Miner.check_config(config) self.config = config # ---- Neuron ---- self.neuron = bittensor.neuron.Neuron(self.config) # ---- Model ---- self.model = GPT2LMSynapse(self.config) # ---- Optimizer ---- self.optimizer = torch.optim.SGD(self.model.parameters(), lr=self.config.miner.learning_rate, momentum=self.config.miner.momentum) self.scheduler = WarmupCosineWithHardRestartsSchedule( self.optimizer, 50, 300) # ---- Model Load/Save tools ---- self.model_toolbox = ModelToolbox(GPT2LMSynapse, torch.optim.SGD) # ---- Dataset ---- # Dataset: 74 million sentences pulled from books. self.dataset = load_dataset('ag_news')['train'] self.device = torch.device( "cuda" if torch.cuda.is_available() else "cpu") # ---- Logging ---- self.tensorboard = SummaryWriter(log_dir=self.config.miner.full_path) if self.config.miner.record_log: logger.add( self.config.miner.full_path + "/{}_{}.log".format( self.config.miner.name, self.config.miner.trial_uid), format="{time:YYYY-MM-DD at HH:mm:ss} | {level} | {message}")