def display_model_params(neon_args, neon_root_yaml): """ Display model parameters :param neon_args: contains command line arguments, :param neon_root_yaml: contains YAML elements """ output_string = '\n-- INFORMATION: HYPER PARAMETERS ------\n' try: output_string = add_param_to_output(output_string, 'backend', neon_args.backend) output_string = add_param_to_output(output_string, 'batch size', neon_args.batch_size) output_string = add_param_to_output(output_string, 'epochs', neon_args.epochs) output_string = add_param_to_output(output_string, 'optimizer type', neon_root_yaml['optimizer']['type']) output_string = add_param_to_output(output_string, 'learning rate', neon_root_yaml['optimizer']['config']['learning_rate']) output_string = add_param_to_output(output_string, 'momentum coef', neon_root_yaml['optimizer']['config']['momentum_coef']) except Exception: output_string += 'Some parameters cannot be displayed\n' output_string += '----------------------------------------' neon_logger.display(output_string)
def extract_images(self, overwrite=False): for setn in ('train', 'val'): img_dir = os.path.join(self.out_dir, setn) neon_logger.display("Extracting %s files" % (setn)) toptar = getattr(self, setn + '_tar') label_dict = getattr(self, setn + '_labels') name_slice = slice(None, 9) if setn == 'train' else slice(15, -5) with tarfile.open(toptar) as tf: for s in tf.getmembers(): label = label_dict[s.name[name_slice]] subpath = os.path.join(img_dir, str(label)) if not os.path.exists(subpath): os.makedirs(subpath) if setn == 'train': tarfp = tarfile.open(fileobj=tf.extractfile(s)) file_list = tarfp.getmembers() else: tarfp = tf file_list = [s] for fobj in file_list: fname = os.path.join(subpath, fobj.name) if not os.path.exists(fname) or overwrite: with open(fname, 'wb') as jf: jf.write(tarfp.extractfile(fobj).read())
def display_platform_information(): """ Display platform information. """ import platform output_string = '\n-- INFORMATION: PLATFORM & OS ---------\n' try: output_string = add_param_to_output(output_string, 'OS', platform.platform()) output_string = add_param_to_output(output_string, 'OS release version', platform.version()) output_string = add_param_to_output(output_string, 'machine', platform.machine()) output_string = add_param_to_output(output_string, 'node', platform.node()) output_string = add_param_to_output(output_string, 'python version', platform.python_version()) output_string = add_param_to_output(output_string, 'python build', platform.python_build()) output_string = add_param_to_output(output_string, 'python compiler', platform.python_compiler()) except Exception: output_string += 'Some platform information cannot be displayed\n' output_string += '----------------------------------------' neon_logger.display(output_string)
def write_csv_files(self, overwrite=False): self.extract_images() for setn in ('train', 'val'): img_dir = os.path.join(self.out_dir, setn) csvfile = getattr(self, setn + '_file') neon_logger.display("Getting %s file list" % (setn)) if os.path.exists(csvfile) and not overwrite: neon_logger.display("File %s exists, not overwriting" % (csvfile)) continue flines = [] subdirs = glob(os.path.join(img_dir, '*')) for subdir in subdirs: subdir_label = os.path.basename( subdir) # This is the int label files = glob(os.path.join(subdir, self.file_pattern)) flines += [(filename, subdir_label) for filename in files] if setn == 'train': np.random.seed(0) np.random.shuffle(flines) with gzip.open(csvfile, 'wb') as f: f.write('filename,l_id\n') for tup in flines: f.write('{},{}\n'.format(*tup))
def fetch_dataset(url, sourcefile, destfile, totalsz): """ Download the file specified by the given URL. Args: url (str): Base URL of the file to be downloaded. sourcefile (str): Name of the source file. destfile (str): Path to the destination. totalsz (int): Size of the file to be downloaded. """ req = Request(os.path.join(url, sourcefile), headers={'User-Agent': 'neon'}) # backport https limitation and workaround per http://python-future.org/imports.html cloudfile = urlopen(req) neon_logger.display("Downloading file: {}".format(destfile)) blockchar = u'\u2588' # character to display in progress bar with open(destfile, 'wb') as f: data_read = 0 chunksz = 1024**2 while 1: data = cloudfile.read(chunksz) if not data: break data_read = min(totalsz, data_read + chunksz) progress_string = u'Download Progress |{:<50}| '.format( blockchar * int(float(data_read) / totalsz * 50)) sys.stdout.write('\r') if PY3: sys.stdout.write(progress_string) else: sys.stdout.write(progress_string.encode("utf-8")) sys.stdout.flush() f.write(data) neon_logger.display("Download Complete")
def display_model_params(neon_args, neon_root_yaml): """ Display model parameters :param neon_args: contains command line arguments, :param neon_root_yaml: contains YAML elements """ output_string = '\n-- INFORMATION: HYPER PARAMETERS ------\n' try: output_string = add_param_to_output(output_string, 'backend', neon_args.backend) output_string = add_param_to_output(output_string, 'batch size', neon_args.batch_size) output_string = add_param_to_output(output_string, 'epochs', neon_args.epochs) output_string = add_param_to_output( output_string, 'optimizer type', neon_root_yaml['optimizer']['type']) output_string = add_param_to_output( output_string, 'learning rate', neon_root_yaml['optimizer']['config']['learning_rate']) output_string = add_param_to_output( output_string, 'momentum coef', neon_root_yaml['optimizer']['config']['momentum_coef']) except Exception: output_string += 'Some parameters cannot be displayed\n' output_string += '----------------------------------------' neon_logger.display(output_string)
def checkSequentialMatchesBatch(): """ check LSTM I/O forward/backward interactions """ n, b, d = (5, 3, 4) # sequence length, batch size, hidden size input_size = 10 WLSTM = LSTM.init(input_size, d) # input size, hidden size X = np.random.randn(n, b, input_size) h0 = np.random.randn(b, d) c0 = np.random.randn(b, d) # sequential forward cprev = c0 hprev = h0 caches = [{} for t in range(n)] Hcat = np.zeros((n, b, d)) for t in range(n): xt = X[t:t + 1] _, cprev, hprev, cache = LSTM.forward(xt, WLSTM, cprev, hprev) caches[t] = cache Hcat[t] = hprev # sanity check: perform batch forward to check that we get the same thing H, _, _, batch_cache = LSTM.forward(X, WLSTM, c0, h0) assert np.allclose(H, Hcat), 'Sequential and Batch forward don''t match!' # eval loss wrand = np.random.randn(*Hcat.shape) # loss = np.sum(Hcat * wrand) dH = wrand # get the batched version gradients BdX, BdWLSTM, Bdc0, Bdh0 = LSTM.backward(dH, batch_cache) # now perform sequential backward dX = np.zeros_like(X) dWLSTM = np.zeros_like(WLSTM) dc0 = np.zeros_like(c0) dh0 = np.zeros_like(h0) dcnext = None dhnext = None for t in reversed(range(n)): dht = dH[t].reshape(1, b, d) dx, dWLSTMt, dcprev, dhprev = LSTM.backward( dht, caches[t], dcnext, dhnext) dhnext = dhprev dcnext = dcprev dWLSTM += dWLSTMt # accumulate LSTM gradient dX[t] = dx[0] if t == 0: dc0 = dcprev dh0 = dhprev # and make sure the gradients match neon_logger.display('Making sure batched version agrees with sequential version: ' '(should all be True)') neon_logger.display(np.allclose(BdX, dX)) neon_logger.display(np.allclose(BdWLSTM, dWLSTM)) neon_logger.display(np.allclose(Bdc0, dc0)) neon_logger.display(np.allclose(Bdh0, dh0))
def train_model(lrmodel, opt, cost, X, Y, devX, devY, devscores): """ Train model, using pearsonr on dev for early stopping """ done = False best = -1.0 r = np.arange(1, 6) train_set = ArrayIterator(X=X, y=Y, make_onehot=False) valid_set = ArrayIterator(X=devX, y=devY, make_onehot=False) eval_epoch = 10 while not done: callbacks = Callbacks(lrmodel, eval_set=valid_set) lrmodel.fit(train_set, optimizer=opt, num_epochs=eval_epoch, cost=cost, callbacks=callbacks) # Every 10 epochs, check Pearson on development set yhat = np.dot(lrmodel.get_outputs(valid_set), r) score = pearsonr(yhat, devscores)[0] if score > best: neon_logger.display('Dev Pearson: {}'.format(score)) best = score bestlrmodel = copy.copy(lrmodel) else: done = True eval_epoch += 10 yhat = np.dot(bestlrmodel.get_outputs(valid_set), r) score = pearsonr(yhat, devscores)[0] neon_logger.display('Dev Pearson: {}'.format(score)) return bestlrmodel
def main(args): # load up the mnist data set dataset = MNIST(path=args.data_dir) # initialize model object mlp = Model(layers=[ Affine(nout=100, init=Gaussian(loc=0.0, scale=0.01), activation=Rectlin()), Affine(nout=10, init=Gaussian(loc=0.0, scale=0.01), activation=Logistic(shortcut=True)) ]) # setup optimizer optimizer = GradientDescentMomentum(0.1, momentum_coef=0.9, stochastic_round=args.rounding) # configure callbacks callbacks = Callbacks(mlp, eval_set=dataset.valid_iter, **args.callback_args) # run fit # setup cost function as CrossEntropy mlp.fit(dataset.train_iter, optimizer=optimizer, num_epochs=args.epochs, cost=GeneralizedCost(costfunc=CrossEntropyBinary()), callbacks=callbacks) error_rate = mlp.eval(dataset.valid_iter, metric=Misclassification()) neon_logger.display('Classification accuracy = %.4f' % (1 - error_rate))
def compute_vocab_expansion(orig_word_vectors, w2v_W, w2v_vocab, word_idict): neon_logger.display("Learning linear mapping from w2v -> rnn embedding...") clf = train_regressor(orig_word_vectors, w2v_W, w2v_vocab) neon_logger.display("Constructing map...") init_embed = apply_regressor(clf, w2v_W, w2v_vocab, orig_word_vectors, word_idict) return init_embed
def write_csv_files(self, overwrite=False): self.extract_images() for setn in ('train', 'val'): img_dir = os.path.join(self.out_dir, setn) csvfile = getattr(self, setn + '_file') neon_logger.display("Getting %s file list" % (setn)) if os.path.exists(csvfile) and not overwrite: neon_logger.display("File %s exists, not overwriting" % (csvfile)) continue flines = [] subdirs = glob(os.path.join(img_dir, '*')) for subdir in subdirs: subdir_label = os.path.basename(subdir) # This is the int label files = glob(os.path.join(subdir, self.file_pattern)) flines += [(filename, subdir_label) for filename in files] if setn == 'train': np.random.seed(0) np.random.shuffle(flines) with gzip.open(csvfile, 'wb') as f: f.write('filename,l_id\n') for tup in flines: f.write('{},{}\n'.format(*tup))
def write_batches(self, offset, labels, imfiles): npts = -(-len(imfiles) // self.macro_size) starts = [i * self.macro_size for i in range(npts)] imfiles = [imfiles[s:s + self.macro_size] for s in starts] labels = [{k: v[s:s + self.macro_size] for k, v in labels.items()} for s in starts] for i, jpeg_file_batch in enumerate(imfiles): bfile = os.path.join(self.out_dir, '%s%d.cpio' % (self.batch_prefix, offset + i)) label_batch = labels[i]['l_id'] if os.path.exists(bfile): neon_logger.display("File %s exists, skipping..." % (bfile)) else: self.write_individual_batch(bfile, label_batch, jpeg_file_batch) neon_logger.display("Wrote batch %d" % (i)) # Check the batchfile for the max item value batch_max_item = self.writerlib.read_max_item(ct.c_char_p(bfile)) if batch_max_item == 0: raise ValueError("Batch file %s probably empty or corrupt" % (bfile)) self.item_max_size = max(batch_max_item, self.item_max_size)
def fetch_dataset(url, sourcefile, destfile, totalsz): """ Download the file specified by the given URL. Args: url (str): Base URL of the file to be downloaded. sourcefile (str): Name of the source file. destfile (str): Path to the destination. totalsz (int): Size of the file to be downloaded. """ req = Request('/'.join([url, sourcefile]), headers={'User-Agent': 'neon'}) # backport https limitation and workaround per http://python-future.org/imports.html cloudfile = urlopen(req) neon_logger.display("Downloading file: {}".format(destfile)) blockchar = u'\u2588' # character to display in progress bar with open(destfile, 'wb') as f: data_read = 0 chunksz = 1024**2 while 1: data = cloudfile.read(chunksz) if not data: break data_read = min(totalsz, data_read + chunksz) progress_string = u'Download Progress |{:<50}| '.format( blockchar * int(float(data_read) / totalsz * 50)) sys.stdout.write('\r') if PY3: sys.stdout.write(progress_string) else: sys.stdout.write(progress_string.encode("utf-8")) sys.stdout.flush() f.write(data) neon_logger.display("Download Complete")
def train_or_val_pairs(self, setn): """ untar imagenet tar files into directories that indicate their label. returns [(filename, label), ...] for train or val set partitions """ img_dir = os.path.join(self.out_dir, setn) neon_logger.display("Extracting %s files" % (setn)) root_tf_path = self.tars[setn] if not os.path.exists(root_tf_path): raise IOError(("tar file {} not found. Ensure you have ImageNet downloaded" ).format(root_tf_path)) try: root_tf = tarfile.open(root_tf_path) except tarfile.ReadError as e: raise ValueError('ReadError opening {}: {}'.format(root_tf_path, e)) label_dict = self.extract_labels(setn) subpaths = root_tf.getmembers() arg_iterator = zip(repeat(self.target_size), repeat(root_tf_path), repeat(img_dir), repeat(setn), repeat(label_dict), subpaths) pool = multiprocessing.Pool() pairs = [] for pair_list in tqdm.tqdm(pool.imap_unordered(process_i1k_tar_subpath, arg_iterator), total=len(subpaths)): pairs.extend(pair_list) pool.close() pool.join() root_tf.close() return pairs
def display_cpu_information(): """ Display CPU information. Assumes all CPUs are the same. """ import cpuinfo output_string = '\n-- INFORMATION: CPU -------------------\n' cpu_info = cpuinfo.get_cpu_info() try: output_string = add_param_to_output(output_string, 'brand', cpu_info['brand']) output_string = add_param_to_output(output_string, 'vendor id', cpu_info['vendor_id']) output_string = add_param_to_output(output_string, 'model', cpu_info['model']) output_string = add_param_to_output(output_string, 'family', cpu_info['family']) output_string = add_param_to_output(output_string, 'bits', cpu_info['bits']) output_string = add_param_to_output(output_string, 'architecture', cpu_info['arch']) output_string = add_param_to_output(output_string, 'cores', cpu_info['count']) output_string = add_param_to_output(output_string, 'advertised Hz', cpu_info['hz_advertised']) output_string = add_param_to_output(output_string, 'actual Hz', cpu_info['hz_actual']) output_string = add_param_to_output(output_string, 'l2 cache size', cpu_info['l2_cache_size']) except Exception: output_string += 'Some CPU information cannot be displayed\n' output_string += '----------------------------------------' neon_logger.display(output_string)
def evaluate(model, val_iter, Metric): running_error = np.zeros((len(Metric.metric_names)), dtype=np.float32) nprocessed = 0 dataset = val_iter dataset.reset() if hasattr(dataset, 'seq_length'): ndata = dataset.ndata * dataset.seq_length else: ndata = dataset.ndata Metric = Misclassification() N = 0 for x, t in dataset: x = model.fprop(x, inference=True) # This logic is for handling partial batch sizes at the end of the dataset nsteps = x.shape[1] // model.be.bsz if not isinstance(x, list) else \ x[0].shape[1] // model.be.bsz bsz = min(ndata - nprocessed, model.be.bsz) tmp = float(running_error) if not math.isnan(float(running_error)): running_error += Metric(x, t, calcrange=slice( 0, nsteps * bsz)) * nsteps * bsz nprocessed += bsz * nsteps if not math.isnan(float(running_error)): running_error /= nprocessed if math.isnan(float(running_error)): running_error = tmp break neon_logger.display('Misclassification error = %.1f%%' % (running_error * 100))
def run(self): if not os.path.exists(self.out_dir): os.makedirs(self.out_dir) neon_logger.display("Writing train macrobatches") self.write_batches(self.train_start, self.labels['train'], self.imgs['train']) neon_logger.display("Writing validation macrobatches") self.write_batches(self.val_start, self.labels['val'], self.imgs['val']) self.save_meta()
def on_epoch_end(self, callback_data, model, epoch): preds = model.get_outputs(self.eval_set)[:, 1] idx_file = os.path.join( self.data_dir, 'eval-' + str(self.subj) + '-' + str(0) + '-index.csv') labels = np.loadtxt(idx_file, delimiter=',', skiprows=1, usecols=[1]) logger.display('Eval AUC for subject %d epoch %d: %.4f\n' % (self.subj, epoch, score(labels, preds)))
def load_vocab(self): """ Load vocab and initialize buffers Input sentence batch is of dimension (vocab_size, max_sentence_length * batch_size) where each column is the 1-hot representation of a word and the first batch_size columns are the first words of each sentence. """ sentences = [sent['tokens'] for sent in self.iterSentences()] # Flatten list of list of words to one list of words words = [word for sentence in sentences for word in sentence] # Count words and keep words greater than threshold word_counts = Counter(words) vocab = [self.end_token] + \ [word for word in list(word_counts.keys()) if word_counts[word] >= 5] self.vocab_size = len(vocab) self.vocab_to_index = dict((c, i) for i, c in enumerate(vocab)) self.index_to_vocab = dict((i, c) for i, c in enumerate(vocab)) # Compute optional bias vector for initializing final linear layer bias word_counts[self.end_token] = len(sentences) self.bias_init = np.array([ 1.0 * word_counts[self.index_to_vocab[i]] for i in self.index_to_vocab ]).reshape((self.vocab_size, 1)) self.bias_init /= np.sum(self.bias_init) self.bias_init = np.log(self.bias_init) self.bias_init -= np.max(self.bias_init) self.max_sentence_length = max(len(sent) for sent in sentences) + 1 self.dev_image = self.be.iobuf(self.image_size) self.dev_imageT = self.be.empty(self.dev_image.shape[::-1]) self.dev_X = self.be.iobuf((self.vocab_size, self.max_sentence_length)) self.dev_y = self.be.iobuf( (self.vocab_size, self.max_sentence_length + 1)) # Create mask to deal with variable length sentences self.dev_y_mask = self.be.iobuf( (self.vocab_size, self.max_sentence_length + 1)) self.y_mask = np.zeros(self.dev_y_mask.shape, dtype=np.uint8).reshape( self.vocab_size, self.max_sentence_length + 1, -1) self.y_mask_reshape = self.y_mask.reshape(self.dev_y_mask.shape) self.dev_lbl = self.be.iobuf(self.max_sentence_length, dtype=np.int32) self.dev_lblT = self.be.empty(self.dev_lbl.shape[::-1]) self.dev_lblflat = self.dev_lbl.reshape((1, self.dev_lbl.size)) self.dev_y_lbl = self.be.iobuf(self.max_sentence_length + 1, dtype=np.int32) self.dev_y_lblT = self.be.empty(self.dev_y_lbl.shape[::-1]) self.dev_y_lblflat = self.dev_y_lbl.reshape((1, self.dev_y_lbl.size)) self.shape = [ self.image_size, (self.vocab_size, self.max_sentence_length) ] neon_logger.display("Vocab size: %d, Max sentence length: %d" % (self.vocab_size, self.max_sentence_length))
def __init__(self, path='.', task='qa1_single-supporting-fact', subset='en'): """ Load bAbI dataset and extract text and read the stories For a particular task, the class will read both train and test files and combine the vocabulary. Arguments: path (str): Directory to store the dataset task (str): a particular task to solve (all bAbI tasks are train and tested separately) subset (str): subset of the dataset to use: {en, en-10k, shuffled, hn, hn-10k, shuffled-10k} """ url = 'http://www.thespermwhale.com/jaseweston/babi' size = 11745123 filename = 'tasks_1-20_v1-2.tar.gz' super(BABI, self).__init__(filename, url, size, path=path) self.task = task self.subset = subset neon_logger.display('Preparing bAbI dataset or extracting from %s' % path) neon_logger.display('Task is %s/%s' % (subset, task)) self.tasks = [ 'qa1_single-supporting-fact', 'qa2_two-supporting-facts', 'qa3_three-supporting-facts', 'qa4_two-arg-relations', 'qa5_three-arg-relations', 'qa6_yes-no-questions', 'qa7_counting', 'qa8_lists-sets', 'qa9_simple-negation', 'qa10_indefinite-knowledge', 'qa11_basic-coreference', 'qa12_conjunction', 'qa13_compound-coreference', 'qa14_time-reasoning', 'qa15_basic-deduction', 'qa16_basic-induction', 'qa17_positional-reasoning', 'qa18_size-reasoning', 'qa19_path-finding', 'qa20_agents-motivations' ] assert task in self.tasks, "given task is not in the bAbI dataset" self.train_file, self.test_file = self.load_data(path, task) self.train_parsed = BABI.parse_babi(self.train_file) self.test_parsed = BABI.parse_babi(self.test_file) self.compute_statistics() self.train = self.vectorize_stories(self.train_parsed) self.test = self.vectorize_stories(self.test_parsed)
def test_dataset(backend_default, data): dataset = MNIST(path=data) dataset.gen_iterators() train_set = dataset.data_dict['train'] train_set.be = NervanaObject.be for i in range(2): for X_batch, y_batch in train_set: neon_logger.display("Xshape: {}, yshape: {}".format(X_batch.shape, y_batch.shape)) train_set.index = 0
def test_dataset(backend_default, data): (X_train, y_train), (X_test, y_test), nclass = load_mnist(path=data) train_set = ArrayIterator(X_train, y_train, nclass=nclass) train_set.be = NervanaObject.be for i in range(2): for X_batch, y_batch in train_set: neon_logger.display("Xshape: {}, yshape: {}".format(X_batch.shape, y_batch.shape)) train_set.index = 0
def test_pool_layer(poolargs, backend_pair_bench): op = poolargs[0] dtype = np.float32 ng, nc = backend_pair_bench N, C = 32, 32 D, H, W = 1, 32, 32 J, T, R, S = 2, 1, 3, 3 padding_j, padding_d, padding_h, padding_w = 0, 0, 0, 0 strides_j, strides_d, strides_h, strides_w = 2, 1, 2, 2 pool_ng = ng.pool_layer(dtype, op, N, C, D, H, W, J, T, R, S, padding_j, padding_d, padding_h, padding_w, strides_j, strides_d, strides_h, strides_w) pool_nc = nc.pool_layer(dtype, op, N, C, D, H, W, J, T, R, S, padding_j, padding_d, padding_h, padding_w, strides_j, strides_d, strides_h, strides_w) assert pool_ng.dimI == pool_nc.dimI assert pool_ng.dimO == pool_nc.dimO dimI = pool_ng.dimI dimO = pool_ng.dimO # generating input arrays for inputs and errors cpuI = np.random.uniform(0.0, 1.0, sliceable(dimI, 1)).astype(np.float16).astype(dtype) cpuE = np.random.uniform(-0.2, 0.2, dimO).astype(dtype) # zero pad the last row of cpu input for the sake of numpy if op == "max": cpuI[-1, :] = np.finfo(dtype).min else: cpuI[-1, :] = 0 # ========= GPU, CPU and numpy ========== beI = cpuI[:-1, :].reshape(dimI) beE = cpuE ngO, ngB = run_backend_pool(ng, pool_ng, beI, beE, dtype) ncO, ncB = run_backend_pool(nc, pool_nc, beI, beE, dtype) cpuO, cpuB = run_numpy_pool(op, cpuI, cpuE, dtype, pool_ng) for opA, ngA, ncA, cpuA in (("fprop", ngO, ncO, cpuO), ("bprop", ngB, ncB.reshape(dimI), cpuB[:-1, :].reshape(dimI))): neon_logger.display(opA) assert allclose_with_out(ngA.get(), ncA.get(), rtol=0, atol=1e-4) assert allclose_with_out(ncA.get(), cpuA, rtol=0, atol=1e-5)
def test_dataset(backend_default, data): dataset = MNIST(path=data) dataset.gen_iterators() train_set = dataset.data_dict['train'] train_set.be = NervanaObject.be for i in range(2): for X_batch, y_batch in train_set: neon_logger.display("Xshape: {}, yshape: {}".format( X_batch.shape, y_batch.shape)) train_set.index = 0
def load_vocab(self): """ Load vocab and initialize buffers Input sentence batch is of dimension (vocab_size, max_sentence_length * batch_size) where each column is the 1-hot representation of a word and the first batch_size columns are the first words of each sentence. """ sentences = [sent['tokens'] for sent in self.iterSentences()] # Flatten list of list of words to one list of words words = [word for sentence in sentences for word in sentence] # Count words and keep words greater than threshold word_counts = Counter(words) vocab = [self.end_token] + \ [word for word in list(word_counts.keys()) if word_counts[word] >= 5] self.vocab_size = len(vocab) self.vocab_to_index = dict((c, i) for i, c in enumerate(vocab)) self.index_to_vocab = dict((i, c) for i, c in enumerate(vocab)) # Compute optional bias vector for initializing final linear layer bias word_counts[self.end_token] = len(sentences) self.bias_init = np.array([1.0 * word_counts[self.index_to_vocab[i]] for i in self.index_to_vocab]).reshape((self.vocab_size, 1)) self.bias_init /= np.sum(self.bias_init) self.bias_init = np.log(self.bias_init) self.bias_init -= np.max(self.bias_init) self.max_sentence_length = max(len(sent) for sent in sentences) + 1 self.dev_image = self.be.iobuf(self.image_size) self.dev_imageT = self.be.empty(self.dev_image.shape[::-1]) self.dev_X = self.be.iobuf((self.vocab_size, self.max_sentence_length)) self.dev_y = self.be.iobuf((self.vocab_size, self.max_sentence_length + 1)) # Create mask to deal with variable length sentences self.dev_y_mask = self.be.iobuf((self.vocab_size, self.max_sentence_length + 1)) self.y_mask = np.zeros(self.dev_y_mask.shape, dtype=np.uint8).reshape(self.vocab_size, self.max_sentence_length + 1, -1) self.y_mask_reshape = self.y_mask.reshape(self.dev_y_mask.shape) self.dev_lbl = self.be.iobuf(self.max_sentence_length, dtype=np.int32) self.dev_lblT = self.be.empty(self.dev_lbl.shape[::-1]) self.dev_lblflat = self.dev_lbl.reshape((1, self.dev_lbl.size)) self.dev_y_lbl = self.be.iobuf(self.max_sentence_length + 1, dtype=np.int32) self.dev_y_lblT = self.be.empty(self.dev_y_lbl.shape[::-1]) self.dev_y_lblflat = self.dev_y_lbl.reshape((1, self.dev_y_lbl.size)) self.shape = [self.image_size, (self.vocab_size, self.max_sentence_length)] neon_logger.display("Vocab size: %d, Max sentence length: %d" % (self.vocab_size, self.max_sentence_length))
def test_roipooling_bprop_ref(backend_default, rois=None, inputs=None, outputs_fprop_ref=None, input_errors=None): if rois is None and inputs is None and outputs_fprop_ref is None and input_errors is None: return (bsz, img_fm_c, img_fm_h, img_fm_w) = inputs.shape (rois_per_batch, _, roi_size, _) = input_errors.shape outputs_fprop_ref_in = outputs_fprop_ref.reshape(rois_per_batch, -1).T feature_maps = inputs.reshape(bsz, -1).T.astype(np.float, order='C') input_errors_in = input_errors.reshape(rois_per_batch, -1).T.astype(np.float, order='C') # compare with GPU kernel, need to call fprop first, then bprop NervanaObject.be.bsz = bsz be = NervanaObject.be input_dev = be.array(feature_maps) rois_dev = be.array(rois) output_shape = (img_fm_c, roi_size, roi_size, rois_per_batch) outputs_dev = be.zeros(output_shape, dtype=np.float32) # make sure the type being int argmax_dev = be.zeros(output_shape, dtype=np.int32) input_error_dev = be.array(input_errors_in) output_error_dev = be.zeros(outputs_fprop_ref_in.shape) be.roipooling_fprop(input_dev, rois_dev, outputs_dev, argmax_dev, rois_per_batch, img_fm_c, img_fm_h, img_fm_w, roi_size, roi_size, spatial_scale) outputs_fprop_be = outputs_dev.get().reshape(-1, rois_per_batch) assert np.allclose(outputs_fprop_ref_in, outputs_fprop_be, atol=1e-6, rtol=0) start_time = timeit() be.roipooling_bprop(input_error_dev, rois_dev, output_error_dev, argmax_dev, rois_per_batch, img_fm_c, img_fm_h, img_fm_w, roi_size, roi_size, spatial_scale) neon_logger.display( "NervanaGPU roipooling bprop (sec): {}".format(timeit() - start_time)) outputs_backend = output_error_dev.get() assert np.allclose(outputs_fprop_ref_in, outputs_backend, atol=1e-6, rtol=0)
def _print_tree(node, level=0): """ print tree with indentation """ if type(node) is list: neon_logger.display((" " * level) + ", ".join(native_str(s) for s in node[0:3])) if len(node) > 3: _print_tree(node[3], level + 1) if len(node) > 4: _print_tree(node[4], level + 1) else: neon_logger.display((" " * level) + native_str(node))
def train_mlp_classifier(dataset, model_file_path, num_epochs, callback_args): """ Train the np_semantic_segmentation mlp classifier Args: model_file_path (str): model path num_epochs (int): number of epochs callback_args (dict): callback_arg dataset: NpSemanticSegData object containing the dataset Returns: print error_rate, test_accuracy_rate and precision_recall_rate evaluation from the model """ model = NpSemanticSegClassifier(num_epochs, callback_args) model.build() # run fit model.fit(dataset.test_set, dataset.train_set) # save model params model.save(model_file_path) # set evaluation error rates error_rate, test_accuracy_rate, precision_recall_rate = model.eval( dataset.test_set) neon_logger.display('Misclassification error = %.1f%%' % (error_rate * 100)) neon_logger.display('Test accuracy rate = %.1f%%' % (test_accuracy_rate * 100)) neon_logger.display('precision rate = %s!!' % (str(precision_recall_rate[0]))) neon_logger.display('recall rate = %s!!' % (str(precision_recall_rate[1])))
def display_text(index_to_token, gt, pr): """ Print out some example strings of input - output pairs. """ index_to_token[0] = '|' # remove actual line breaks display_len = 3 * time_steps # sample 3 sentences and their start and end time steps (s1_s, s1_e) = (0, time_steps) (s2_s, s2_e) = (time_steps, 2*time_steps) (s3_s, s3_e) = (2*time_steps, 3*time_steps) gt_string = "".join([index_to_token[gt[k]] for k in range(display_len)]) pr_string = "".join([index_to_token[pr[k]] for k in range(display_len)]) match = np.where([gt_string[k] == pr_string[k] for k in range(display_len)]) di_string = "".join([gt_string[k] if k in match[0] else '.' for k in range(display_len)]) neon_logger.display('GT: [' + gt_string[s1_s:s1_e] + '] ' '[' + gt_string[s2_s:s2_e] + '] ' '[' + gt_string[s3_s:s3_e] + '] ') neon_logger.display('Pred: [' + pr_string[s1_s:s1_e] + '] ' '[' + pr_string[s2_s:s2_e] + '] ' '[' + pr_string[s3_s:s3_e] + '] ') neon_logger.display('Difference indicated by .') neon_logger.display('Diff: [' + di_string[s1_s:s1_e] + '] ' '[' + di_string[s2_s:s2_e] + '] ' '[' + di_string[s3_s:s3_e] + '] ')
def display_text(index_to_token, gt, pr): index_to_token[0] = '|' # remove actual line breaks display_len = 3 * time_steps # sample 3 sentences and its start and end time steps (s1_s, s1_e) = (0, time_steps) (s2_s, s2_e) = (time_steps, 2 * time_steps) (s3_s, s3_e) = (2 * time_steps, 3 * time_steps) gt_string = "".join([index_to_token[gt[k]] for k in range(display_len)]) pr_string = "".join([index_to_token[pr[k]] for k in range(display_len)]) match = np.where( [gt_string[k] == pr_string[k] for k in range(display_len)]) di_string = "".join( [gt_string[k] if k in match[0] else '.' for k in range(display_len)]) neon_logger.display('GT: [' + gt_string[s1_s:s1_e] + '] ' '[' + gt_string[s2_s:s2_e] + '] ' '[' + gt_string[s3_s:s3_e] + '] ') neon_logger.display('Pred: [' + pr_string[s1_s:s1_e] + '] ' '[' + pr_string[s2_s:s2_e] + '] ' '[' + pr_string[s3_s:s3_e] + '] ') neon_logger.display('Difference indicated by .') neon_logger.display('Diff: [' + di_string[s1_s:s1_e] + '] ' '[' + di_string[s2_s:s2_e] + '] ' '[' + di_string[s3_s:s3_e] + '] ')
def train_mlp_classifier(dataset, model_file_path, num_epochs, callback_args): """ Train the np_semantic_segmentation mlp classifier Args: model_file_path (str): model path num_epochs (int): number of epochs callback_args (dict): callback_arg dataset: NpSemanticSegData object containing the dataset Returns: print error_rate, test_accuracy_rate and precision_recall_rate evaluation from the model """ model = NpSemanticSegClassifier(num_epochs, callback_args) model.build() # run fit model.fit(dataset.test_set, dataset.train_set) # save model params model.save(model_file_path) # set evaluation error rates error_rate, test_accuracy_rate, precision_recall_rate = model.eval(dataset.test_set) neon_logger.display('Misclassification error = %.1f%%' % (error_rate * 100)) neon_logger.display('Test accuracy rate = %.1f%%' % (test_accuracy_rate * 100)) neon_logger.display('precision rate = %s!!' % (str(precision_recall_rate[0]))) neon_logger.display('recall rate = %s!!' % (str(precision_recall_rate[1])))
def main(): # parse the command line arguments parser = NeonArgparser(__doc__) parser.add_argument('--output_path', required=True, help='Output path used when training model') parser.add_argument('--w2v_path', required=False, default=None, help='Path to GoogleNews w2v file for voab expansion.') parser.add_argument('--eval_data_path', required=False, default='./SICK_data', help='Path to the SICK dataset for evaluating semantic relateness') parser.add_argument('--max_vocab_size', required=False, default=1000000, help='Limit the vocabulary expansion to fit in GPU memory') parser.add_argument('--subset_pct', required=False, default=100, help='subset of training dataset to use (use to retreive \ preprocessed data from training)') args = parser.parse_args(gen_be=True) # load vocab file from training _, vocab_file = load_data(args.data_dir, output_path=args.output_path, subset_pct=float(args.subset_pct)) vocab, _, _ = load_obj(vocab_file) vocab_size = len(vocab) neon_logger.display("\nVocab size from the dataset is: {}".format(vocab_size)) index_from = 2 # 0: padding 1: oov vocab_size_layer = vocab_size + index_from max_len = 30 # load trained model model_dict = load_obj(args.model_file) # Vocabulary expansion trick needs to pass the correct vocab set to evaluate (for tokenization) if args.w2v_path: neon_logger.display("Performing Vocabulary Expansion... Loading W2V...") w2v_vocab, w2v_vocab_size = get_w2v_vocab(args.w2v_path, int(args.max_vocab_size), cache=True) vocab_size_layer = w2v_vocab_size + index_from model = load_sent_encoder(model_dict, expand_vocab=True, orig_vocab=vocab, w2v_vocab=w2v_vocab, w2v_path=args.w2v_path, use_recur_last=True) vocab = w2v_vocab else: # otherwise stick with original vocab size used to train the model model = load_sent_encoder(model_dict, use_recur_last=True) model.initialize(dataset=(max_len, 1)) evaluate(model, vocab=vocab, data_path=args.eval_data_path, evaltest=True, vocab_size_layer=vocab_size_layer)
def test_roipooling_fprop_random(backend_default, fargs): rois_per_image, img_fm_c, img_fm_h, img_fm_w, roi_size, bsz = fargs # generate a random feature map and some random ROIs feature_maps = np.random.random( (img_fm_c, img_fm_h, img_fm_w, bsz)).reshape(-1, bsz) rois_per_batch = rois_per_image * bsz rois_idx = np.vstack( [i * np.ones((rois_per_image, 1)) for i in range(bsz)]) rois = np.random.random((rois_per_batch, 4)) * min(img_fm_h, img_fm_w) rois = np.zeros((rois_per_batch, 4)) rois[:, 0] = np.random.random((rois_per_batch, )) * 10 / spatial_scale rois[:, 1] = np.random.random((rois_per_batch, )) * 25 / spatial_scale rois[:, 2] = (np.random.random( (rois_per_batch, )) * 27 + (img_fm_w - 27)) / spatial_scale rois[:, 3] = (np.random.random( (rois_per_batch, )) * 12 + (img_fm_h - 12)) / spatial_scale rois = np.hstack((rois_idx, rois)) # run the numpy roi fprop (function inside this test script) outputs_np = fprop_roipooling_ref(feature_maps, rois, img_fm_c, img_fm_h, img_fm_w, bsz, rois_per_image, roi_size, roi_size) # call backend roipooling kernel NervanaObject.be.bsz = bsz be = NervanaObject.be input_dev = be.array(feature_maps) rois_dev = be.array(rois) output_shape = (img_fm_c, roi_size, roi_size, rois_per_batch) outputs_dev = be.zeros(output_shape) # make sure the type being int argmax_dev = be.zeros(output_shape, np.int32) start_time = timeit() be.roipooling_fprop(input_dev, rois_dev, outputs_dev, argmax_dev, rois_per_batch, img_fm_c, img_fm_h, img_fm_w, roi_size, roi_size, spatial_scale) neon_logger.display( "Nervana backend roipooling fprop (sec): {}".format(timeit() - start_time)) outputs_be = outputs_dev.get().reshape(-1, rois_per_batch) assert np.allclose(outputs_np, outputs_be, atol=1e-6, rtol=0)
def test_roipooling_fprop_ref(backend_default, rois=None, inputs=None, outputs_ref=None): if rois is None and inputs is None and outputs_ref is None: return (bsz, img_fm_c, img_fm_h, img_fm_w) = inputs.shape (rois_per_batch, _, roi_size, _) = outputs_ref.shape outputs_ref_in = outputs_ref.reshape(rois_per_batch, -1).T rois_per_image = rois_per_batch // bsz feature_maps = inputs.reshape(bsz, -1).T.astype(np.float, order='C') # run the numpy roi fprop (function inside this test script) outputs_np = fprop_roipooling_ref(feature_maps, rois, img_fm_c, img_fm_h, img_fm_w, bsz, rois_per_image, roi_size, roi_size) assert allclose_with_out(outputs_ref_in, outputs_np, atol=1e-6, rtol=0) # call NervanaGPU roipooling kernel NervanaObject.be.bsz = bsz be = NervanaObject.be input_dev = be.array(feature_maps) rois_dev = be.array(rois) output_shape = (img_fm_c, roi_size, roi_size, rois_per_batch) outputs_dev = be.zeros(output_shape, dtype=np.float32) # make sure the type being int argmax_dev = be.zeros(output_shape, dtype=np.int32) start_time = timeit() be.roipooling_fprop(input_dev, rois_dev, outputs_dev, argmax_dev, rois_per_batch, img_fm_c, img_fm_h, img_fm_w, roi_size, roi_size, spatial_scale) outputs_backend = outputs_dev.get().reshape(-1, rois_per_batch) neon_logger.display( "Nervana backend roipooling fprop (sec): {}".format(timeit() - start_time)) assert allclose_with_out(outputs_ref_in, outputs_backend, atol=1e-6, rtol=0)
def load_vgg_weights(model, path): # load a pre-trained VGG16 from Neon model zoo to the local url = 'https://s3-us-west-1.amazonaws.com/nervana-modelzoo/VGG/' filename = 'VGG_D_Conv.p' size = 169645138 workdir, filepath = Dataset._valid_path_append(path, '', filename) if not os.path.exists(filepath): Dataset.fetch_dataset(url, filename, filepath, size) neon_logger.display('De-serializing the pre-trained VGG16 model...') pdict = load_obj(filepath) param_layers = [l for l in model.layers.layers[0].layers[0].layers] param_dict_list = pdict['model']['config']['layers'] for layer, ps in zip(param_layers, param_dict_list): neon_logger.display("{}".format(layer.name, ps['config']['name'])) layer.load_weights(ps, load_states=True)
def bleu_score(self, sents, targets): """ Compute the BLEU score from a list of predicted sentences and reference sentences Args: sents (list): list of predicted sentences targets (list): list of reference sentences where each element is a list of multiple references. """ num_ref = len(targets[0]) output_file = self.path + '/output' reference_files = [ self.path + '/reference%d' % i for i in range(num_ref) ] bleu_script_url = 'https://raw.githubusercontent.com/karpathy/neuraltalk/master/eval/' bleu_script = 'multi-bleu.perl' neon_logger.display("Writing output and reference sents to dir %s" % self.path) output_f = open(output_file, 'w+') for sent in sents: sent = sent.strip(self.end_token).split() output_f.write(" ".join(sent) + '\n') reference_f = [open(f, 'w') for f in reference_files] for i in range(num_ref): for target_sents in targets: reference_f[i].write(target_sents[i] + '\n') output_f.close() [x.close() for x in reference_f] owd = os.getcwd() os.chdir(self.path) if not os.path.exists(bleu_script): Dataset.fetch_dataset(bleu_script_url, bleu_script, bleu_script, 6e6) bleu_command = 'perl multi-bleu.perl reference < output' neon_logger.display( "Executing bleu eval script: {}".format(bleu_command)) os.system(bleu_command) os.chdir(owd)
def test_roipooling_fprop_random(backend_default, fargs): rois_per_image, img_fm_c, img_fm_h, img_fm_w, roi_size, bsz = fargs # generate a random feature map and some random ROIs feature_maps = np.random.random( (img_fm_c, img_fm_h, img_fm_w, bsz)).reshape(-1, bsz) rois_per_batch = rois_per_image * bsz rois_idx = np.vstack([i * np.ones((rois_per_image, 1)) for i in range(bsz)]) rois = np.random.random((rois_per_batch, 4)) * min(img_fm_h, img_fm_w) rois = np.zeros((rois_per_batch, 4)) rois[:, 0] = np.random.random((rois_per_batch,)) * 10 / spatial_scale rois[:, 1] = np.random.random((rois_per_batch,)) * 25 / spatial_scale rois[:, 2] = ( np.random.random((rois_per_batch,)) * 27 + (img_fm_w - 27)) / spatial_scale rois[:, 3] = ( np.random.random((rois_per_batch,)) * 12 + (img_fm_h - 12)) / spatial_scale rois = np.hstack((rois_idx, rois)) # run the numpy roi fprop (function inside this test script) outputs_np = fprop_roipooling_ref(feature_maps, rois, img_fm_c, img_fm_h, img_fm_w, bsz, rois_per_image, roi_size, roi_size) # call backend roipooling kernel NervanaObject.be.bsz = bsz be = NervanaObject.be input_dev = be.array(feature_maps) rois_dev = be.array(rois) output_shape = (img_fm_c, roi_size, roi_size, rois_per_batch) outputs_dev = be.zeros(output_shape) # make sure the type being int argmax_dev = be.zeros(output_shape, np.int32) start_time = timeit() be.roipooling_fprop(input_dev, rois_dev, outputs_dev, argmax_dev, rois_per_batch, img_fm_c, img_fm_h, img_fm_w, roi_size, roi_size, spatial_scale) neon_logger.display("Nervana backend roipooling fprop (sec): {}".format(timeit() - start_time)) outputs_be = outputs_dev.get().reshape(-1, rois_per_batch) assert allclose_with_out(outputs_np, outputs_be, atol=1e-6, rtol=0)
def test_sr(backend_gpu): """ Performs stochastic rounding with 1 bit mantissa for an addition operation and checks that the resulting array is rounded correctly """ be = NervanaObject.be n = 10 A = be.ones((n, n), dtype=np.float16) B = be.ones((n, n), dtype=np.float16) be.multiply(B, 0.1, out=B) C = be.ones((n, n), dtype=np.float16) C.rounding = 1 C[:] = A + B C_host = C.get() # make sure everything is either 1. (rounded down 1 bit) or 1.5 (rounded up 1 bit) logger.display("Rounded Buf: {}".format(C_host)) assert sum([C_host.flatten()[i] in [1.0, 1.5] for i in range(n ** 2)]) == n ** 2 assert sum([C_host.flatten()[i] in [1.5] for i in range(n ** 2)]) > 0.1 * n ** 2 assert sum([C_host.flatten()[i] in [1.0] for i in range(n ** 2)]) > 0.7 * n ** 2
def bprop_stats(self): if self.bprop_out is not None: neon_logger.display("bprop:%10.5f mean %11.5f max %s" % (self.get_delta_mean(), self.get_delta_max(), self)) if self.weights is not None: up_mean, up_max = (self.get_update_mean(), self.get_update_max()) wt_mean, wt_max = (self.get_weight_mean(), self.get_weight_max()) rt_mean, rt_max = (0.0001 * up_mean / wt_mean, 0.0001 * up_max / wt_max) neon_logger.display("updat:%10.5f mean %11.5f max %s" % (up_mean, up_max, self)) neon_logger.display("weigh:%10.5f mean %11.5f max" % (wt_mean, wt_max)) neon_logger.display("ratio:%10.5f mean %11.5f max" % (rt_mean, rt_max))
def test_sr_cpu(backend_cpu): """ Performs stochastic rounding with 1 bit mantissa for an addition operation and checks that the resulting array is rounded correctly """ be = NervanaObject.be n = 10 A = be.ones((n, n), dtype=np.float16) B = be.ones((n, n), dtype=np.float16) be.multiply(B, 0.1, out=B) C = be.ones((n, n), dtype=np.float16) C.rounding = 1 C[:] = A + B C_host = C.get() # make sure everything is either 1. (rounded down 1 bit) or 1.5 (rounded up 1 bit) logger.display("Rounded Buf: {}".format(C_host)) assert sum([C_host.flatten()[i] in [1., 1.5] for i in range(n**2)]) == n**2 assert sum([C_host.flatten()[i] in [1.5] for i in range(n**2)]) > .1 * n**2 assert sum([C_host.flatten()[i] in [1.] for i in range(n**2)]) > .7 * n**2
def pytest_generate_tests(metafunc): if metafunc.config.option.all: bsz_rng = [16, 32, 64] else: bsz_rng = [128] if 'basic_linargs' in metafunc.fixturenames: fargs = [] if metafunc.config.option.all: nin_rng = [1, 2, 64, 128] nout_rng = [1, 4, 128, 64] vocab_size = [1, 4, 1000, 2000] else: nin_rng = [4, 32] nout_rng = [3, 33] vocab_size = [10, 34] fargs = itt.product(nin_rng, nout_rng, vocab_size, bsz_rng) neon_logger.display('{}'.format(fargs)) metafunc.parametrize('basic_linargs', fargs)
def test_roipooling_bprop_ref(backend_default, rois=None, inputs=None, outputs_fprop_ref=None, input_errors=None): if rois is None and inputs is None and outputs_fprop_ref is None and input_errors is None: return (bsz, img_fm_c, img_fm_h, img_fm_w) = inputs.shape (rois_per_batch, _, roi_size, _) = input_errors.shape outputs_fprop_ref_in = outputs_fprop_ref.reshape(rois_per_batch, -1).T feature_maps = inputs.reshape(bsz, -1).T.astype(np.float, order='C') input_errors_in = input_errors.reshape( rois_per_batch, -1).T.astype(np.float, order='C') # compare with GPU kernel, need to call fprop first, then bprop NervanaObject.be.bsz = bsz be = NervanaObject.be input_dev = be.array(feature_maps) rois_dev = be.array(rois) output_shape = (img_fm_c, roi_size, roi_size, rois_per_batch) outputs_dev = be.zeros(output_shape, dtype=np.float32) # make sure the type being int argmax_dev = be.zeros(output_shape, dtype=np.int32) input_error_dev = be.array(input_errors_in) output_error_dev = be.zeros(outputs_fprop_ref_in.shape) be.roipooling_fprop(input_dev, rois_dev, outputs_dev, argmax_dev, rois_per_batch, img_fm_c, img_fm_h, img_fm_w, roi_size, roi_size, spatial_scale) outputs_fprop_be = outputs_dev.get().reshape(-1, rois_per_batch) assert allclose_with_out( outputs_fprop_ref_in, outputs_fprop_be, atol=1e-6, rtol=0) start_time = timeit() be.roipooling_bprop(input_error_dev, rois_dev, output_error_dev, argmax_dev, rois_per_batch, img_fm_c, img_fm_h, img_fm_w, roi_size, roi_size, spatial_scale) neon_logger.display("NervanaGPU roipooling bprop (sec): {}".format(timeit() - start_time)) outputs_backend = output_error_dev.get() assert allclose_with_out(outputs_fprop_ref_in, outputs_backend, atol=1e-6, rtol=0)
def __init__(self, path): self.path = path neon_logger.display('Reading test images and sentences from %s' % self.path) # Load vocab using training set and then load test set self.read_images('train') self.load_vocab() self.read_images('test') trainIter = self.iterImageSentenceGroup() trainSents, trainImgs = [], [] for i, img_sent in enumerate(trainIter): trainImgs.append(img_sent['image']) trainSents.append([' '.join(sent['tokens']) for sent in img_sent['sentences']]) self.nbatches = len(trainImgs) // self.be.bsz self.ndata = self.nbatches * self.be.bsz self.images = np.vstack(trainImgs) self.ref_sents = trainSents
def get_device_count(verbose=False): """ Query device count through PyCuda. Arguments: verbose (bool): prints verbose logging if True, default False. Returns: int: Number of GPUs available. """ try: import pycuda import pycuda.driver as drv except ImportError: if verbose: neon_logger.display("PyCUDA module not found") return 0 try: drv.init() except pycuda._driver.RuntimeError as e: neon_logger.display("PyCUDA Runtime error: {0}".format(str(e))) return 0 count = drv.Device.count() if verbose: neon_logger.display("Found {} GPU(s)".format(count)) return count
def bleu_score(self, sents, targets): """ Compute the BLEU score from a list of predicted sentences and reference sentences Args: sents (list): list of predicted sentences targets (list): list of reference sentences where each element is a list of multiple references. """ num_ref = len(targets[0]) output_file = self.path + '/output' reference_files = [self.path + '/reference%d' % i for i in range(num_ref)] bleu_script_url = 'https://raw.githubusercontent.com/karpathy/neuraltalk/master/eval/' bleu_script = 'multi-bleu.perl' neon_logger.display("Writing output and reference sents to dir %s" % self.path) output_f = open(output_file, 'w+') for sent in sents: sent = sent.strip(self.end_token).split() output_f.write(" ".join(sent) + '\n') reference_f = [open(f, 'w') for f in reference_files] for i in range(num_ref): for target_sents in targets: reference_f[i].write(target_sents[i] + '\n') output_f.close() [x.close() for x in reference_f] owd = os.getcwd() os.chdir(self.path) if not os.path.exists(bleu_script): Dataset.fetch_dataset(bleu_script_url, bleu_script, bleu_script, 6e6) bleu_command = 'perl multi-bleu.perl reference < output' neon_logger.display("Executing bleu eval script: {}".format(bleu_command)) os.system(bleu_command) os.chdir(owd)
def run(self): self.write_csv_files() if self.validation_pct == 0: namelist = ['train'] filelist = [self.train_file] startlist = [self.train_start] elif self.validation_pct == 1: namelist = ['validation'] filelist = [self.val_file] startlist = [self.val_start] else: namelist = ['train', 'validation'] filelist = [self.train_file, self.val_file] startlist = [self.train_start, self.val_start] for sname, fname, start in zip(namelist, filelist, startlist): neon_logger.display("Writing %s %s %s" % (sname, fname, start)) if fname is not None and os.path.exists(fname): imgs, labels = self.parse_file_list(fname) self.write_batches(start, labels, imgs) else: neon_logger.display("Skipping %s, file missing" % (sname)) # Get the max item size and store it for meta file self.save_meta()
def convert_file(iopair, keylist): """ Function for converting from an imageset batch cpickle file into a flat binary with a choice of keys. Input file is cpickled dict with the following fields: dict['data']: list of jpeg strings dict['labels']: dict of integer lists, default is 'l_id' for the category label of the corresponding jpeg. The following condition should be true (a label for each jpeg) len(dict['data']) == len(dict['labels']['l_id']) Arguments: iopair(tuple): Names of input and output files. keylist(list): A list of keys to be used in the flat binary file. """ ifname, ofname = iopair with open(ifname, 'rb') as ifp: neon_logger.display("Converting {}".format(ifname)) tdata = pickle.load(ifp) jpegs = tdata['data'] labels = tdata['labels'] num_imgs = len(jpegs) with open(ofname, 'wb') as f: f.write(struct.pack('I', num_imgs)) f.write(struct.pack('I', len(keylist))) for key in keylist: ksz = len(key) f.write(struct.pack('L' + 'B' * ksz, ksz, *bytearray(key))) f.write(struct.pack('I' * num_imgs, *labels[key])) for i in range(num_imgs): jsz = len(jpegs[i]) bin = struct.pack('I' + 'B' * jsz, jsz, *bytearray(jpegs[i])) f.write(bin)