def check_config(config: Munch):
assert config.miner.momentum > 0 and config.miner.momentum < 1, "momentum must be a value between 0 and 1"
assert config.miner.batch_size_train > 0, "batch_size_train must a positive value"
assert config.miner.learning_rate > 0, "learning_rate must be a positive value."
full_path = '{}/{}/{}'.format(config.miner.root_dir, config.miner.name, config.miner.trial_uid)
config.miner.full_path = os.path.expanduser(full_path)
if not os.path.exists(config.miner.full_path):
os.makedirs(config.miner.full_path)
GPT2LMSynapse.check_config(config)
bittensor.neuron.Neuron.check_config(config)
def __init__(self, config: Munch = None):
if config == None:
config = Miner.build_config(); logger.info(bittensor.config.Config.toString(config))
self.config = config
# ---- Neuron ----
self.neuron = bittensor.neuron.Neuron(self.config)
# ---- Model ----
self.model = GPT2LMSynapse( self.config )
# ---- Optimizer ----
self.optimizer = torch.optim.SGD(self.model.parameters(), lr = self.config.miner.learning_rate, momentum=self.config.miner.momentum)
self.scheduler = WarmupCosineWithHardRestartsSchedule(self.optimizer, 50, 300)
# ---- Model Load/Save tools ----
self.model_toolbox = ModelToolbox(GPT2LMSynapse, torch.optim.SGD)
# ---- Dataset ----
# Dataset: 74 million sentences pulled from books.
# self.dataset = load_dataset('ag_news')['train']
self.dataset = AdamCorpus(self.config.miner.custom_datasets)
# ---- Logging ----
self.tensorboard = SummaryWriter(log_dir = self.config.miner.full_path)
if self.config.miner.record_log:
logger.add(self.config.miner.full_path + "/{}_{}.log".format(self.config.miner.name, self.config.miner.trial_uid),format="{time:YYYY-MM-DD at HH:mm:ss} | {level} | {message}")
def __init__(self, config: Munch = None, **kwargs):
if config == None:
config = Miner.default_config()
bittensor.config.Config.update_with_kwargs(config.miner, kwargs)
Miner.check_config(config)
self.config = config
# ---- Neuron ----
self.neuron = bittensor.neuron.Neuron(self.config)
# ---- Model ----
self.model = GPT2LMSynapse(self.config)
# ---- Optimizer ----
self.optimizer = torch.optim.SGD(self.model.parameters(),
lr=self.config.miner.learning_rate,
momentum=self.config.miner.momentum)
self.scheduler = WarmupCosineWithHardRestartsSchedule(
self.optimizer, 50, 300)
# ---- Model Load/Save tools ----
self.model_toolbox = ModelToolbox(GPT2LMSynapse, torch.optim.SGD)
# ---- Dataset ----
# The Genesis Dataset:
# The dataset used to train Adam and his first 100 children.
self.dataset = AdamCorpus(self.config.miner.custom_dataset)
# ---- Logging ----
self.tensorboard = SummaryWriter(log_dir=self.config.miner.full_path)
if self.config.miner.record_log:
logger.add(
self.config.miner.full_path + "/{}_{}.log".format(
self.config.miner.name, self.config.miner.trial_uid),
format="{time:YYYY-MM-DD at HH:mm:ss} | {level} | {message}")
def add_args(parser: argparse.ArgumentParser):
parser.add_argument('--miner.learning_rate', default=0.01, type=float, help='Training initial learning rate.')
parser.add_argument('--miner.momentum', default=0.98, type=float, help='Training initial momentum for SGD.')
parser.add_argument('--miner.n_epochs', default=int(sys.maxsize), type=int, help='Number of training epochs.')
parser.add_argument('--miner.epoch_length', default=500, type=int, help='Iterations of training per epoch')
parser.add_argument('--miner.batch_size_train', default=1, type=int, help='Training batch size.')
parser.add_argument('--miner.sync_interval', default=100, type=int, help='Batches before we sync with chain and emit new weights.')
parser.add_argument('--miner.log_interval', default=10, type=int, help='Batches before we log miner info.')
parser.add_argument('--miner.accumulation_interval', default=1, type=int, help='Batches before we apply acummulated gradients.')
parser.add_argument('--miner.apply_remote_gradients', default=False, type=bool, help='If true, neuron applies gradients which accumulate from remotes calls.')
parser.add_argument('--miner.root_dir', default='~/.bittensor/miners/', type=str, help='Root path to load and save data associated with each miner')
parser.add_argument('--miner.name', default='gpt-wiki', type=str, help='Trials for this miner go in miner.root / miner.name')
parser.add_argument('--miner.trial_uid', default=str(time.time()).split('.')[0], type=str, help='Saved models go in miner.root_dir / miner.name / miner.uid')
parser.add_argument('--miner.record_log', default=True, help='Record all logs when running this miner')
parser.add_argument('--miner.config_file', type=str, help='config file to run this neuron, if not using cmd line arguments.')
GPT2LMSynapse.add_args(parser)
bittensor.neuron.Neuron.add_args(parser)
def __init__(self, config: Munch = None, **kwargs):
if config == None:
config = Miner.default_config()
bittensor.config.Config.update_with_kwargs(config.miner, kwargs)
Miner.check_config(config)
self.config = config
# ---- Neuron ----
self.neuron = bittensor.neuron.Neuron(self.config)
# ---- Model ----
self.model = GPT2LMSynapse(self.config)
# ---- Optimizer ----
self.optimizer = torch.optim.SGD(self.model.parameters(),
lr=self.config.miner.learning_rate,
momentum=self.config.miner.momentum)
self.scheduler = WarmupCosineWithHardRestartsSchedule(
self.optimizer, 50, 300)
# ---- Model Load/Save tools ----
self.model_toolbox = ModelToolbox(GPT2LMSynapse, torch.optim.SGD)
# ---- Dataset ----
# Dataset: 74 million sentences pulled from books.
self.dataset = load_dataset('ag_news')['train']
self.device = torch.device(
"cuda" if torch.cuda.is_available() else "cpu")
# ---- Logging ----
self.tensorboard = SummaryWriter(log_dir=self.config.miner.full_path)
if self.config.miner.record_log:
logger.add(
self.config.miner.full_path + "/{}_{}.log".format(
self.config.miner.name, self.config.miner.trial_uid),
format="{time:YYYY-MM-DD at HH:mm:ss} | {level} | {message}")