def solve(self): for t in monit.loop(self.epochs): if not self.is_online_update: for I in self.info_sets.values(): I.clear() for i in range(self.n_players): self.cfr(self.create_new_history(), cast(Player, i), [1 for _ in range(self.n_players)]) if not self.is_online_update: self.update() with monit.section("Track"): for I in self.info_sets.values(): for a in I.actions(): tracker.add({ f'strategy.{I.key}.{a}': I.strategy[a], f'average_strategy.{I.key}.{a}': I.average_strategy[a], f'regret.{I.key}.{a}': I.regret[a], f'current_regret.{I.key}.{a}': I.current_regret[a] }) if t % self.track_frequency == 0: tracker.save() logger.log() if (t + 1) % self.save_frequency == 0: experiment.save_checkpoint() logger.inspect(self.info_sets)
def _test(): from labml.logger import inspect projects = Projects() runs = projects.get_runs() for r in runs: inspect(r.to_dict())
def split_train_valid( files: List[EncodedFile], is_shuffle=True) -> (List[EncodedFile], List[EncodedFile]): """ Split training and validation sets """ if is_shuffle: np.random.shuffle(files) total_size = sum([len(f.codes) for f in files]) valid = [] valid_size = 0 while len(files) > 0: if valid_size > total_size * 0.15: break valid.append(files[0]) valid_size += len(files[0].codes) files.pop(0) train_size = sum(len(f.codes) for f in files) if train_size < total_size * 0.60: raise RuntimeError("Validation set too large") logger.inspect(train_size=train_size, valid_size=valid_size, vocab=tokenizer.VOCAB_SIZE) return files, valid
def print_env_vars(): inspect(world_size=os.environ['WORLD_SIZE'], run_uuid=os.environ['RUN_UUID'], local_rank=os.environ['LOCAL_RANK'], rank=os.environ['RANK'], master_addr=os.environ['MASTER_ADDR'], master_port=os.environ['MASTER_PORT'])
def _test(): """ ### Test the model with fake data """ chunk_len = 4 d_model = 8 d_ff = 32 n_heads = 2 d_k = 4 device = torch.device('cuda:0') m = RetroModel(5, d_model, 6, {2, 5}, chunk_len, n_heads, d_k, d_ff, encoder=NearestNeighborEncoder(chunk_len, 2, {1}, d_model, n_heads, d_k, d_ff)) m.to(device) x = [1, 2, 4, 4, 0, 1, 2, 3, 4, 3] ret = [ [[0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1]], [[0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1]], ] res = m( torch.tensor([x] * 10).to(device), torch.tensor([ret] * 10).to(device)) inspect(res)
def predict(): data = ["This year business is good", "Fortnite, Football And Soccer, And Their Surprising Similarities"] client = get_deploy_client('torchserve') for d in data: data_json = json.dumps({'data': [d], 'uuid': 'str'}) res = client.predict('news_classification_test', data_json) inspect(text=d, category=res)
def _test(): """ A simple test to verify the tensor sizes """ conv2d = Conv2d(10, 20, 5) from labml.logger import inspect inspect(conv2d.weight) import torch inspect(conv2d(torch.zeros(10, 10, 100, 100)))
def evaluate(predictor: Predictor, text: str): line_no = 1 logs = [(f"{line_no: 4d}: ", Text.meta), (text[0], Text.subtle)] correct = 0 i = 0 key_strokes = 0 while i + 1 < len(text): prefix = text[:i + 1] stripped, prompt = predictor.rstrip(prefix) rest = prefix[len(stripped):] prediction_complete = NextWordPredictionComplete(rest, 5) prompt = torch.tensor(prompt, dtype=torch.long).unsqueeze(-1) predictions = predictor.get_next_word(prompt, None, rest, [1.], prediction_complete, 5) predictions.sort(key=lambda x: -x[0]) if predictions: next_token = predictions[0].text[len(rest):] else: next_token = '' if next_token and next_token == text[i + 1:i + 1 + len(next_token)]: correct += len(next_token) right = True else: next_token = text[i + 1] right = False for j, c in enumerate(next_token): if c == '\n': logger.log(logs) line_no += 1 logs = [(f"{line_no: 4d}: ", Text.meta)] elif c == '\r': continue else: if right: if j == 0: logs.append((c, [Text.meta, Style.underline])) else: logs.append((c, [Text.success, Style.underline])) else: logs.append((c, [Text.warning])) i += len(next_token) key_strokes += 1 logger.log(logs) logger.inspect(accuracy=correct / (len(text) - 1), key_strokes=key_strokes, length=len(text))
def _test_rotary(): """ Testing RoPE with a simple example """ x = torch.tensor([[1, 2, 3, 4], [4, 5, 6, 7], [7, 8, 9, 10]], dtype=torch.float) x = x[:, None, None, :] inspect(x) rotary_pe = RotaryPositionalEmbeddings(3) inspect(rotary_pe(x))
def main(): source_files = _GetPythonFiles().files logger.inspect(source_files) with open(str(Path(os.getcwd()) / 'data' / 'all.py'), 'w') as f: for i, source in monit.enum("Parse", source_files): serialized = _read_file(source.path) # return serialized = [str(t) for t in serialized] f.write(f"{str(source.path)}\n") f.write(" ".join(serialized) + "\n")
def eval(self): keys_saved = 0 for line, content in enumerate(self.__content): # Keep reference to rest of the line rest_of_line = content # Build the line for logging with colors # The line number logs = [(f"{line: 4d}: ", Text.meta)] # Type the line character by character while rest_of_line != '': suggestion = self.__predictor.get_suggestion() # If suggestion matches if suggestion != '' and rest_of_line.startswith(suggestion): # Log logs.append((suggestion[0], [Style.underline, Text.danger])) logs.append((suggestion[1:], Style.underline)) keys_saved += len(suggestion) - 1 # Skip the prediction text rest_of_line = rest_of_line[len(suggestion):] # Add text to the predictor self.__predictor.add(suggestion) # If the suggestion doesn't match else: # Add the next character self.__predictor.add(rest_of_line[0]) logs.append((rest_of_line[0], Text.subtle)) rest_of_line = rest_of_line[1:] # Add a new line self.__predictor.add("\n") # Log the line logger.log(logs) # Log time taken for the file logger.inspect(add=self.__predictor.time_add, check=self.__predictor.time_check, predict=self.__predictor.time_predict) total_keys = sum([len(c) for c in self.__content]) logger.inspect(keys_saved=keys_saved, percentage_saved=100 * keys_saved / total_keys, total_keys=total_keys, total_lines=len(self.__content))
def main(): source_files = _GetPythonFiles().files np.random.shuffle(source_files) logger.inspect(source_files) train_valid_split = int(len(source_files) * 0.9) _load_code(lab.get_data_path() / 'train.py', source_files[:train_valid_split]) _load_code(lab.get_data_path() / 'valid.py', source_files[train_valid_split:])
def _test(): """ Simple test """ from labml.logger import inspect x = torch.zeros([2, 6, 2, 4]) inspect(x.shape) bn = InstanceNorm(6) x = bn(x) inspect(x.shape)
def _launcher(): import os world_size = int(os.environ['WORLD_SIZE']) run_uuid = os.environ['RUN_UUID'] local_rank = int(os.environ['LOCAL_RANK']) rank = int(os.environ['RANK']) inspect(world_size=os.environ['WORLD_SIZE'], run_uuid=os.environ['RUN_UUID'], local_rank=os.environ['LOCAL_RANK'], rank=os.environ['RANK'], master_addr=os.environ['MASTER_ADDR'], master_port=os.environ['MASTER_PORT']) main(local_rank, rank, world_size, run_uuid, 'env://')
def main(): try: batch() except KeyboardInterrupt: pass source_files = get_python_files() np.random.shuffle(source_files) logger.inspect(source_files) train_valid_split = int(len(source_files) * 0.9) concat_and_save(lab.get_data_path() / 'train.py', source_files[:train_valid_split]) concat_and_save(lab.get_data_path() / 'valid.py', source_files[train_valid_split:])
def main(): # Create experiment experiment.create(name='cifar10', comment='small model') # Create configurations conf = Configs() # Load configurations experiment.configs(conf, { 'optimizer.optimizer': 'Adam', 'optimizer.learning_rate': 2.5e-4, }) # Set model for saving/loading experiment.add_pytorch_models({'model': conf.model}) # Print number of parameters in the model logger.inspect(params=(sum(p.numel() for p in conf.model.parameters() if p.requires_grad))) # Start the experiment and run the training loop with experiment.start(): conf.run()
def to_numpy(df: pd.DataFrame): dates, packets = to_daily_packets(df) empty_mins = np.sum(packets[:, :, 4] == 0) filled_mins = np.sum(packets[:, :, 4] > 0) empty_mins_high_activity = np.sum(packets[:, :, 4] == 0) fill_empty_minutes_in_packets(packets) zero_price = np.sum(packets[:, :, 0:4] == 0) zero_volume = np.sum(packets[:, :, 4] == 0) packets[:, :, 4] = np.maximum(packets[:, :, 4], 1) logger.inspect(empty_mins=empty_mins, filled_mins=filled_mins, empty_mins_high_activity=empty_mins_high_activity, zero_price=zero_price, zero_volume=zero_volume) return dates, packets
def _test(): """ ### Code to test BERT embeddings """ from labml.logger import inspect # Initialize device = torch.device('cuda:0') bert = BERTChunkEmbeddings(device) # Sample text = ["Replace me by any text you'd like.", "Second sentence"] # Check BERT tokenizer encoded_input = bert.tokenizer(text, return_tensors='pt', add_special_tokens=False, padding=True) inspect(encoded_input, _expand=True) # Check BERT model outputs output = bert.model( input_ids=encoded_input['input_ids'].to(device), attention_mask=encoded_input['attention_mask'].to(device), token_type_ids=encoded_input['token_type_ids'].to(device)) inspect( { 'last_hidden_state': output['last_hidden_state'], 'pooler_output': output['pooler_output'] }, _expand=True) # Check recreating text from token ids inspect(bert.tokenizer.convert_ids_to_tokens( encoded_input['input_ids'][0]), _n=-1) inspect(bert.tokenizer.convert_ids_to_tokens( encoded_input['input_ids'][1]), _n=-1) # Get chunk embeddings inspect(bert(text))
def run(self): retries = 1 while not self.is_stopped: response = self.caller.send({'jobs': self.results}) if response is None: logger.log(f'Retrying again in 10 seconds ({retries})...', Text.highlight) time.sleep(10) retries += 1 continue retries = 1 self.results = [] jobs = response.get('jobs', []) logger.log(f'Jobs: {len(jobs)}') for j in jobs: inspect(j) res = self.do_job(j) self.results.append(res) inspect(res)
def evaluate(predictor: Predictor, text: str): line_no = 1 logs = [(f"{line_no: 4d}: ", Text.meta), (text[0], Text.subtle)] correct = 0 i = 0 right = False key_strokes = 0 while i + 1 < len(text): next_token = predictor.get_token(text[:i + 1]) if next_token == text[i + 1:i + 1 + len(next_token)]: correct += len(next_token) right = True else: next_token = text[i + 1] right = False for j, c in enumerate(next_token): if c == '\n': logger.log(logs) line_no += 1 logs = [(f"{line_no: 4d}: ", Text.meta)] elif c == '\r': continue else: if right: if j == 0: logs.append((c, [Text.meta, Style.underline])) else: logs.append((c, [Text.success, Style.underline])) else: logs.append((c, [Text.warning])) i += len(next_token) key_strokes += 1 logger.log(logs) logger.inspect(accuracy=correct / (len(text) - 1), key_strokes=key_strokes, length=len(text))
def main(): from labml_nn.transformers.knn.build_index import load_experiment # Load the experiment. Replace the run uuid with you run uuid from # [training the model](train_model.html). conf = load_experiment('4984b85c20bf11eb877a69c1a03717cd') # Set model to evaluation mode conf.model.eval() # Load index index, keys_store, vals_store = load_index(conf) # List of weights given to $k$-NN prediction. We will evaluate the validation loss for # each of the weights knn_weights = [i / 20 for i in range(10)] # Evaluate validation loss losses, n_samples = validation_loss(knn_weights, None, conf, index, keys_store, vals_store) # Output the losses for each of `knn_weights`. inspect({ c: np.sum(losses[i]) / np.sum(n_samples) for i, c in enumerate(knn_weights) })
def _test(): """ #### Code to test the FTA module """ from labml.logger import inspect # Initialize a = FTA(-10, 10, 2., 0.5) # Print $\mathbf{c}$ inspect(a.c) # Print number of bins $\frac{u - l}{\delta}$ inspect(a.expansion_factor) # Input $z$ z = torch.tensor([1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9., 10., 11.]) # Print $z$ inspect(z) # Print $\phi_\eta(z)$ inspect(a(z))
def test_nvidia_device(idx: int): from py3nvml import py3nvml as nvml handle = nvml.nvmlDeviceGetHandleByIndex(idx) pciInfo = nvml.nvmlDeviceGetPciInfo(handle) brands = { nvml.NVML_BRAND_UNKNOWN: "Unknown", nvml.NVML_BRAND_QUADRO: "Quadro", nvml.NVML_BRAND_TESLA: "Tesla", nvml.NVML_BRAND_NVS: "NVS", nvml.NVML_BRAND_GRID: "Grid", nvml.NVML_BRAND_GEFORCE: "GeForce" } inspect( idx=idx, # id=pciInfo.busId, # uuid=nvml.nvmlDeviceGetUUID(handle), name=nvml.nvmlDeviceGetName(handle), # brand=brands[nvml.nvmlDeviceGetBrand(handle)], # multi_gpu=nvml.nvmlDeviceGetMultiGpuBoard(handle), # pcie_link=nvml.nvmlDeviceGetCurrPcieLinkWidth(handle), fan=nvml.nvmlDeviceGetFanSpeed(handle), # power=nvml.nvmlDeviceGetPowerState(handle), mem_total=nvml.nvmlDeviceGetMemoryInfo(handle).total, mem_used=nvml.nvmlDeviceGetMemoryInfo(handle).used, util_gpu=nvml.nvmlDeviceGetUtilizationRates(handle).gpu, # util_mem=nvml.nvmlDeviceGetUtilizationRates(handle).memory, temp=nvml.nvmlDeviceGetTemperature(handle, nvml.NVML_TEMPERATURE_GPU), power=nvml.nvmlDeviceGetPowerUsage(handle), power_limit=nvml.nvmlDeviceGetPowerManagementLimit(handle), # display=nvml.nvmlDeviceGetDisplayMode(handle), display_active=nvml.nvmlDeviceGetDisplayActive(handle), ) logger.log() procs = nvml.nvmlDeviceGetGraphicsRunningProcesses(handle) for p in procs: inspect(name=nvml.nvmlSystemGetProcessName(p.pid), pid=p.pid, mem=p.usedGpuMemory) procs = nvml.nvmlDeviceGetComputeRunningProcesses(handle) for p in procs: inspect(name=nvml.nvmlSystemGetProcessName(p.pid), pid=p.pid, mem=p.usedGpuMemory) logger.log()
def _test(): """ Simple test """ from labml.logger import inspect x = torch.zeros([2, 3, 2, 4]) inspect(x.shape) ln = LayerNorm(x.shape[2:]) x = ln(x) inspect(x.shape) inspect(ln.gain.shape)
def test_sensors(): try: inspect(psutil.sensors_temperatures()) except AttributeError as e: pass try: inspect(psutil.sensors_fans()) except AttributeError as e: pass try: inspect(psutil.sensors_battery()._asdict()) except AttributeError as e: pass
def _test(): """ Simple test """ from labml.logger import inspect x = torch.zeros([2, 3, 2, 4]) inspect(x.shape) bn = BatchNorm(3) x = bn(x) inspect(x.shape) inspect(bn.exp_var.shape)
def test_nvidia(): # pip install py3nvml import py3nvml from py3nvml import py3nvml as nvml inspect(py3nvml.get_free_gpus()) nvml.nvmlInit() inspect(version=nvml.nvmlSystemGetDriverVersion()) inspect(count=nvml.nvmlDeviceGetCount()) for i in range(nvml.nvmlDeviceGetCount()): test_nvidia_device(i) nvml.nvmlShutdown()
def _test_shift_right(): x = torch.tensor([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) inspect(x) inspect(shift_right(x)) x = torch.arange(1, 6)[None, :, None, None].repeat(5, 1, 1, 1) inspect(x[:, :, 0, 0]) inspect(shift_right(x)[:, :, 0, 0]) x = torch.arange(1, 6)[None, :, None, None].repeat(3, 1, 1, 1) inspect(x[:, :, 0, 0]) inspect(shift_right(x)[:, :, 0, 0])
def _test_local_mask(): """ Test local mask """ from labml.logger import inspect inspect(AFTLocal.create_local_mask(10, 4))
def test_psutil(): # sudo apt-get install gcc python3-dev # xcode on mac # pip install psutil import psutil # https://psutil.readthedocs.io/en/latest/# inspect(mac=psutil.MACOS, linux=psutil.LINUX, windows=psutil.WINDOWS) inspect(psutil.net_io_counters()._asdict()) inspect(psutil.net_if_addrs()) inspect(psutil.net_if_stats()) inspect(psutil.virtual_memory()._asdict()) inspect(psutil.cpu_count()) inspect(psutil.cpu_times()._asdict()) inspect(psutil.cpu_stats()._asdict()) inspect(psutil.cpu_freq()._asdict()) inspect(psutil.cpu_percent(percpu=True)) inspect(psutil.disk_usage(lab.get_path())._asdict()) inspect(psutil.Process().as_dict()) inspect([p for p in psutil.process_iter()]) # inspect(psutil.Process().terminate()) # inspect('test') p = psutil.Process() with p.oneshot(): inspect(p.memory_info()._asdict()) inspect(p.memory_percent()) inspect(p.cpu_percent(1)) inspect(p.cpu_times()._asdict()) inspect(p.num_threads()) inspect(p.threads()) try: inspect(p.cpu_num()) except AttributeError as e: pass try: inspect(psutil.sensors_temperatures()) except AttributeError as e: pass try: inspect(psutil.sensors_fans()) except AttributeError as e: pass try: inspect(psutil.sensors_battery()._asdict()) except AttributeError as e: pass