def main(): # parse the command line arguments parser = NeonArgparser(__doc__) parser.add_argument('--output_path', required=True, help='Output path used when training model') parser.add_argument('--w2v_path', required=False, default=None, help='Path to GoogleNews w2v file for voab expansion.') parser.add_argument('--eval_data_path', required=False, default='./SICK_data', help='Path to the SICK dataset for evaluating semantic relateness') parser.add_argument('--max_vocab_size', required=False, default=1000000, help='Limit the vocabulary expansion to fit in GPU memory') parser.add_argument('--subset_pct', required=False, default=100, help='subset of training dataset to use (use to retreive \ preprocessed data from training)') args = parser.parse_args(gen_be=True) # load vocab file from training _, vocab_file = load_data(args.data_dir, output_path=args.output_path, subset_pct=float(args.subset_pct)) vocab, _, _ = load_obj(vocab_file) vocab_size = len(vocab) neon_logger.display("\nVocab size from the dataset is: {}".format(vocab_size)) index_from = 2 # 0: padding 1: oov vocab_size_layer = vocab_size + index_from max_len = 30 # load trained model model_dict = load_obj(args.model_file) # Vocabulary expansion trick needs to pass the correct vocab set to evaluate (for tokenization) if args.w2v_path: neon_logger.display("Performing Vocabulary Expansion... Loading W2V...") w2v_vocab, w2v_vocab_size = get_w2v_vocab(args.w2v_path, int(args.max_vocab_size), cache=True) vocab_size_layer = w2v_vocab_size + index_from model = load_sent_encoder(model_dict, expand_vocab=True, orig_vocab=vocab, w2v_vocab=w2v_vocab, w2v_path=args.w2v_path, use_recur_last=True) vocab = w2v_vocab else: # otherwise stick with original vocab size used to train the model model = load_sent_encoder(model_dict, use_recur_last=True) model.initialize(dataset=(max_len, 1)) evaluate(model, vocab=vocab, data_path=args.eval_data_path, evaltest=True, vocab_size_layer=vocab_size_layer)
def get_p1b1_parser(): # Construct neon arg parser. It generates a large set of options by default parser = NeonArgparser(__doc__) # Specify the default config_file parser.add_argument("--config_file", dest='config_file', type=str, default=os.path.join(p1b1.file_path, 'p1b1_default_model.txt'), help="specify model configuration file") # Parse other options that are not included on neon arg parser parser = p1_common.get_p1_common_parser(parser) return parser
from neon.callbacks.callbacks import Callbacks, TrainMulticostCallback from neon.transforms.cost import Misclassification from zmlcore.neonfixes.metrics import MultiMetric from zmlcore.classifier.classifier import TextClassifier, Config from zmlcore.classifier.traincallbacks import TrainingProgress, MisclassificationTest from zmlcore.data.dataiterator import TrainingIterator, BatchIterator from zmlcore.data.sentiment_loader import SentimentLoader from zmlcore.neonfixes.transforms import fix_logistic import os, email, mailbox # random percentage to holdout for validation when training holdout_pct = 0.1 if __name__ == '__main__': p = NeonArgparser(__doc__) p.add_argument('--word_vectors', type=str, required=False, default='./data/vocabularies/glove.6B.300d.txt', help='Path to word vector file, including word, followed by vector per line, space separated') p.add_argument('--lookup_size', type=int, default=0, help='If non-zero, a lookup table and an auto-vocabulary will be used instead of word vectors.') p.add_argument('--lookup_dim', type=int, default=100, help='Word embedding dimensions when lookup_size specified.') p.add_argument('--exclusive_classes', type=str, required=False, default='\"finance promos social forums updates\"', help='The labels of the exclusive classes to either train on or classify. ' + 'Should be a quoted, space separated list.') p.add_argument('--overlapping_classes', type=str, required=False, default='\"important\"', help='The labels of the classes, which can overlap with themselves or an exclusive class. ' + 'Should be a quoted, space separated list.') p.add_argument('--sentiment_path', type=str, required=False, default=None, help='This overrides the email classification function altogether to benchmark the content ' + 'classification network only. The parameter is a path to the root directory of ' +
import util from objectlocalization import PASCALVOC from neon.backends import gen_backend from neon.util.persist import get_data_cache_dir, save_obj from neon.util.argparser import NeonArgparser, extract_valid_args from neon import logger as neon_logger from voc_eval import voc_eval import numpy as np import faster_rcnn from tqdm import tqdm # parse the command line arguments parser = NeonArgparser(__doc__, default_overrides={'batch_size': 1}) parser.add_argument('--normalize', action='store_true', help='Normalize the final bounding box regression layers.') parser.add_argument('--output', default=None, help='File to save inference results (optional)') parser.add_argument('--width', type=int, default=1000, help='Width of input image') parser.add_argument('--height', type=int, default=1000, help='Height of input image') args = parser.parse_args()
# ---------------------------------------------------------------------------- import os from neon.optimizers import GradientDescentMomentum, Schedule from neon.transforms import Misclassification from neon.callbacks.callbacks import Callbacks, BatchNormTuneCallback from neon.util.argparser import NeonArgparser from network import create_network from data import make_train_loader, make_validation_loader, make_tuning_loader # parse the command line arguments (generates the backend) train_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'train.cfg') config_files = [train_config] if os.path.exists(train_config) else [] parser = NeonArgparser(__doc__, default_config_files=config_files) parser.add_argument('--depth', type=int, default=2, help='depth of each stage (network depth will be 9n+2)') parser.add_argument('--subset_pct', type=float, default=100, help='subset of training dataset to use (percentage)') args = parser.parse_args() random_seed = args.rng_seed if args.rng_seed else 0 # Check that the proper manifest sets have been supplied assert 'train' in args.manifest, "Missing train manifest" assert 'val' in args.manifest, "Missing validation manifest" model, cost = create_network(args.depth) # setup data provider train = make_train_loader(args.manifest['train'], args.manifest_root, model.be, args.subset_pct, random_seed) test = make_validation_loader(args.manifest['val'], args.manifest_root, model.be, args.subset_pct)
from network_gan import create_model from lsun_data import make_loader # parse the command line arguments train_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'train.cfg') config_files = [train_config] if os.path.exists(train_config) else [] parser = NeonArgparser(__doc__, default_config_files=config_files, default_overrides={ 'rng_seed': 0, 'batch_size': 64 }) parser.add_argument('-D', '--dmodel', type=str, default='dc', help='discriminator model type: dc or mlp, default dc') parser.add_argument('-G', '--gmodel', type=str, default='dc', help='generator model type: dc or mlp, default dc') parser.add_argument( '--subset_pct', type=float, default=100, help='subset of training dataset to use (percentage), default 100') args = parser.parse_args() random_seed = args.rng_seed if args.rng_seed else 0
Usage: use -t to specify which bAbI task to run python examples/babi/demo.py -t 1 --rlayer_type gru --model_weights babi.p """ import numpy as np from util import create_model, babi_handler from neon.backends import gen_backend from neon.data import BABI, QA from neon.data.text import Text from neon.util.argparser import NeonArgparser, extract_valid_args # parse the command line arguments parser = NeonArgparser(__doc__) parser.add_argument('-t', '--task', type=int, default='1', choices=xrange(1, 21), help='the task ID to train/test on from bAbI dataset (1-20)') parser.add_argument('--rlayer_type', default='gru', choices=['gru', 'lstm'], help='type of recurrent layer to use (gru or lstm)') parser.add_argument('--model_weights', help='pickle file of trained weights') args = parser.parse_args(gen_be=False) # setup backend be = gen_backend(**extract_valid_args(args, gen_backend)) be.bsz = 1 # load the bAbI dataset babi = babi_handler(args.data_dir, args.task) valid_set = QA(*babi.test) # create model
neon_logger.display("Running this example requires scipy packages.") neon_logger.display("try activating your virtualenv then: pip install scipy") sys.exit(1) from neon.models import Model from neon.layers import Activation from neon.data.datasets import Dataset from neon.layers import GeneralizedCost from neon.transforms.cost import Cost from neon.util.argparser import NeonArgparser # force use of CPU backend since we require a batch size of 1 # (GPU needs a multiple of 32) default_overrides = dict(backend='cpu', batch_size=1) parser = NeonArgparser(__doc__, default_overrides=default_overrides) parser.add_argument("image", help="Base image to create dream on.") parser.add_argument("--dream_file", default='dream_out.png', help="Save dream to named file.") args = parser.parse_args() # redirect the dream file to the path of output_file if args.output_file is None: output_dir = parser.work_dir elif osp.isdir(args.output_file): output_dir = args.output_file else: output_dir = osp.dirname(args.output_file) args.dream_file = osp.expanduser( osp.join(output_dir, osp.basename(args.dream_file)))
neon_logger.display( "try activating your virtualenv then: pip install scipy") sys.exit(1) from neon.models import Model from neon.layers import Activation from neon.data.datasets import Dataset from neon.layers import GeneralizedCost from neon.transforms.cost import Cost from neon.util.argparser import NeonArgparser # force use of CPU backend since we require a batch size of 1 # (GPU needs a multiple of 32) default_overrides = dict(backend='cpu', batch_size=1) parser = NeonArgparser(__doc__, default_overrides=default_overrides) parser.add_argument("image", help="Base image to create dream on.") parser.add_argument("--dream_file", default='dream_out.png', help="Save dream to named file.") args = parser.parse_args() # redirect the dream file to the path of output_file if args.output_file is None: output_dir = parser.work_dir elif osp.isdir(args.output_file): output_dir = args.output_file else: output_dir = osp.dirname(args.output_file) args.dream_file = osp.expanduser( osp.join(output_dir, osp.basename(args.dream_file)))
import json import numpy as np from neon.util.argparser import NeonArgparser from neon.initializers import Gaussian from neon.layers import Conv, Deconv, GeneralizedCost from neon.optimizers import Adadelta from neon.transforms import Rectlin, Logistic, SumSquared from neon.models import Model from neon.callbacks.callbacks import Callbacks from localizer_loader import LocalizerLoader from evaluator import Evaluator parser = NeonArgparser(__doc__) parser.add_argument('-tw', '--test_data_dir', default='', help='directory in which to find test images') parser.add_argument('-pn', '--point_num', default=None, help='1 or 2') parser.add_argument('-iw', '--image_width', default=384, help='image width') args = parser.parse_args() point_num = int(args.point_num) imwidth = int(args.image_width) train = LocalizerLoader(repo_dir=args.data_dir, inner_size=imwidth, set_name='train', nlabels=4, do_transforms=False, point_num=point_num) test = LocalizerLoader(repo_dir=args.test_data_dir, inner_size=imwidth, set_name='validation', nlabels=4, do_transforms=False, point_num=point_num) train.init_batch_provider() test.init_batch_provider()
from neon.backends import gen_backend # noqa from neon.data import ArrayIterator # noqa from neon.initializers import Uniform, GlorotUniform, Array # noqa from neon.layers import GeneralizedCost, Affine, Dropout, LookupTable, LSTM, RecurrentSum # noqa from neon.models import Model # noqa from neon.optimizers import Adagrad # noqa from neon.transforms import Logistic, Tanh, Softmax, CrossEntropyMulti, Accuracy # noqa from neon.util.argparser import NeonArgparser, extract_valid_args # noqa from neon.util.compat import pickle # noqa from neon.callbacks.callbacks import Callbacks # noqa from neon.data.text_preprocessing import get_paddedXY, get_google_word2vec_W # noqa import h5py # noqa # parse the command line arguments parser = NeonArgparser(__doc__) parser.add_argument("-f", "--review_file", default="labeledTrainData.tsv", help="input movie review file") parser.add_argument( "--vocab_file", default="labeledTrainData.tsv.vocab", help="output file to save the processed vocabulary" ) parser.add_argument("--use_w2v", action="store_true", help="use downloaded Google Word2Vec") parser.add_argument("--w2v", default="GoogleNews-vectors-negative300.bin", help="the pre-built Word2Vec") args = parser.parse_args() # hyperparameters hidden_size = 128 embedding_dim = 128 vocab_size = 20000 sentence_length = 128 batch_size = 32 gradient_limit = 5
and no partial minibatches, dropout is turned off for reproducibility on gpu and the learning rate is scaled to handle the reduced dropout percentage. """ from neon.util.argparser import NeonArgparser from neon.initializers import Constant, Gaussian from neon.layers import Conv, Dropout, Pooling, GeneralizedCost, Affine from neon.optimizers import GradientDescentMomentum, MultiOptimizer, Schedule from neon.transforms import Rectlin, Softmax, CrossEntropyMulti, TopKMisclassification from neon.models import Model from neon.data import ImageLoader from neon.callbacks.callbacks import Callbacks # parse the command line arguments (generates the backend) parser = NeonArgparser(__doc__) parser.add_argument('--direct', action='store_true', help='do not initialize layers, deserialize directly') args = parser.parse_args() # setup data provider img_set_options = dict(repo_dir=args.data_dir, inner_size=224, subset_pct=0.09990891117239205) train = ImageLoader(set_name='train', scale_range=(256, 256), shuffle=False, do_transforms=False, **img_set_options) test = ImageLoader(set_name='validation', scale_range=(256, 384), shuffle=False, do_transforms=False, **img_set_options) layers = [Conv((11, 11, 64), init=Gaussian(scale=0.01), bias=Constant(0), activation=Rectlin(), padding=3, strides=4), Pooling(3, strides=2), Conv((5, 5, 192), init=Gaussian(scale=0.01), bias=Constant(1),
for idx in range(data.shape[0]): im = np.pad(data[idx].reshape((3, 32, 32)), self.pad_width, mode='mean') im = np.uint8(np.transpose(im, axes=[1, 2, 0]).copy()) im = Image.fromarray(im) path = os.path.join(img_dir, str(labels[idx][0]), str(idx) + '.png') im.save(path, format='PNG') if setn == 'train': self.pixel_mean = list(data.mean(axis=0).reshape(3, -1).mean(axis=1)) self.pixel_mean.reverse() # We will see this in BGR order b/c of opencv if __name__ == "__main__": from neon.util.argparser import NeonArgparser parser = NeonArgparser(__doc__) parser.add_argument('--set_type', help='(i1k|cifar10|directory|csv)', required=True, choices=['i1k', 'cifar10', 'directory', 'csv']) parser.add_argument('--image_dir', help='Directory to find images', default=None) parser.add_argument('--target_size', type=int, default=0, help='Size in pixels to scale shortest side DOWN to (0 means no scaling)') parser.add_argument('--macro_size', type=int, default=5000, help='Images per processed batch') parser.add_argument('--file_pattern', default='*.jpg', help='Image extension to include in' 'directory crawl') args = parser.parse_args() logger = logging.getLogger(__name__) if args.set_type == 'i1k': args.target_size = 256 # (maybe 512 for Simonyan's methodology?) bw = BatchWriterI1K(out_dir=args.data_dir, image_dir=args.image_dir, target_size=args.target_size, macro_size=args.macro_size, file_pattern="*.JPEG")
predictions: the model's predictions """ results_list = predictions.tolist() with open(output, 'w', encoding='utf-8') as out_file: writer = csv.writer(out_file, delimiter=',', quotechar='"') for result in results_list: writer.writerow([result]) print("Results of inference saved in {0}".format(output)) if __name__ == "__main__": # parse the command line arguments parser = NeonArgparser() parser.set_defaults(epochs=200) parser.add_argument('--data', help='prepared data CSV file path', type=validate_existing_filepath) parser.add_argument('--model', help='path to the trained model file', type=validate_existing_filepath) parser.add_argument('--print_stats', action='store_true', default=False, help='print evaluation stats for the model predictions - if ' 'your data has tagging') parser.add_argument('--output', help='path to location for inference output file', type=validate_parent_exists) args = parser.parse_args() data_path = absolute_path(args.data) model_path = absolute_path(args.model) print_stats = args.print_stats output_path = absolute_path(args.output) # generate backend be = gen_backend(batch_size=10) data_set = NpSemanticSegData(data_path, train_to_test_ratio=1)
Striving for Simplicity: the All Convolutional Net `[Springenberg2015]`_ .. _[Springenberg2015]: http://arxiv.org/pdf/1412.6806.pdf """ from neon.initializers import Gaussian from neon.optimizers import GradientDescentMomentum, Schedule from neon.layers import Conv, Dropout, Activation, Pooling, GeneralizedCost from neon.transforms import Rectlin, Softmax, CrossEntropyMulti, Misclassification from neon.models import Model from neon.data import ArrayIterator, load_cifar10 from neon.callbacks.callbacks import Callbacks from neon.util.argparser import NeonArgparser # parse the command line arguments parser = NeonArgparser(__doc__) parser.add_argument("--learning_rate", default=0.05, help="initial learning rate") parser.add_argument("--weight_decay", default=0.001, help="weight decay") parser.add_argument("--deconv", action="store_true", help="save visualization data from deconvolution") args = parser.parse_args() # hyperparameters num_epochs = args.epochs (X_train, y_train), (X_test, y_test), nclass = load_cifar10( path=args.data_dir, normalize=False, contrast_normalize=True, whiten=True ) # really 10 classes, pad to nearest power of 2 to match conv output train_set = ArrayIterator(X_train, y_train, nclass=16, lshape=(3, 32, 32)) valid_set = ArrayIterator(X_test, y_test, nclass=16, lshape=(3, 32, 32))
standard_library.install_aliases() # triggers E402, hence noqa below from builtins import input # noqa import numpy as np # noqa from neon.backends import gen_backend # noqa from neon.initializers import Uniform, GlorotUniform # noqa from neon.layers import LSTM, Affine, Dropout, LookupTable, RecurrentSum # noqa from neon.models import Model # noqa from neon.transforms import Logistic, Tanh, Softmax # noqa from neon.util.argparser import NeonArgparser, extract_valid_args # noqa from neon.util.compat import pickle # noqa from neon.data.text_preprocessing import clean_string # noqa # parse the command line arguments parser = NeonArgparser(__doc__) parser.add_argument('--model_weights', required=True, help='pickle file of trained weights') parser.add_argument('--vocab_file', required=True, help='vocabulary file') args = parser.parse_args() # hyperparameters from the reference batch_size = 1 clip_gradients = True gradient_limit = 5 vocab_size = 20000 sentence_length = 128 embedding_dim = 128 hidden_size = 128 reset_cells = True num_epochs = args.epochs
# for example, your data_dir is in an NFS mounted location) from neon.util.argparser import NeonArgparser from neon.initializers import Kaiming from neon.layers import Conv, Pooling, GeneralizedCost, Activation, Affine from neon.layers import MergeSum, SkipNode, BatchNorm from neon.optimizers import GradientDescentMomentum, Schedule from neon.transforms import Rectlin, Softmax, CrossEntropyMulti, Misclassification from neon.models import Model from neon.data import ImageLoader from neon.callbacks.callbacks import Callbacks, BatchNormTuneCallback # parse the command line arguments (generates the backend) parser = NeonArgparser(__doc__) parser.add_argument('--depth', type=int, default=9, help='depth of each stage (network depth will be 9n+2)') parser.add_argument('--subset_pct', type=float, default=100, help='subset of training dataset to use (percentage)') args = parser.parse_args() # setup data provider imgset_options = dict(inner_size=32, scale_range=40, aspect_ratio=110, repo_dir=args.data_dir, subset_pct=args.subset_pct) train = ImageLoader(set_name='train', shuffle=True,
# [SimpleItk](http://www.simpleitk.org/) is an open-source library for reading and processing 3D models. It was particularly designed for medical imaging and was built on top of the Insight and Segmentation and Registration ([ITK](https://itk.org/)) toolkit sponsored by the National Library of Medicine. # # What's nice about SimpleITK is that it has pre-built methods to read all of the DICOM slices into a 3D object and perform segmentation and morphological operations on that 3D object. I believe it automatically arranges the slices in the correct order. It also can handle compressed DICOM files. import SimpleITK as sitk ####################### # Parse the command line from neon.util.argparser import NeonArgparser parser = NeonArgparser(__doc__) # We can pass the input directory and output file name from the command line parser.add_argument('-out', '--outFilename', default='dicom_out.h5', help='Name of the output HDF5 file') parser.set_defaults(data_dir='/Volumes/data/tonyr/dicom/Lung CT/stage1') parser.set_defaults(save_path='.') args = parser.parse_args() data_dir = args.data_dir outFilename = args.save_path + '/' + args.outFilename ############################## def verbosePrint(txt): print(txt)
python examples/babi/demo.py -t 1 --rlayer_type gru --model_weights babi.p """ import numpy as np from util import create_model, babi_handler from neon.backends import gen_backend from neon.data import BABI, QA from neon.data.text import Text from neon.util.argparser import NeonArgparser, extract_valid_args # parse the command line arguments parser = NeonArgparser(__doc__) parser.add_argument( '-t', '--task', type=int, default='1', choices=xrange(1, 21), help='the task ID to train/test on from bAbI dataset (1-20)') parser.add_argument('--rlayer_type', default='gru', choices=['gru', 'lstm'], help='type of recurrent layer to use (gru or lstm)') parser.add_argument('--model_weights', help='pickle file of trained weights') args = parser.parse_args(gen_be=False) # setup backend be = gen_backend(**extract_valid_args(args, gen_backend)) be.bsz = 1 # load the bAbI dataset
-b gpu -i 0 -e 40 --dataset_dir ~/NABirds --model_file alexnet.p -vvvv """ from neon.util.argparser import NeonArgparser from neon.initializers import Constant, Gaussian from neon.layers import Conv, Dropout, Pooling, GeneralizedCost, Affine from neon.transforms import Rectlin, Softmax, CrossEntropyMulti, TopKMisclassification from neon.models import Model from neon.data import ImageLoader from neon.callbacks.callbacks import Callbacks from model_descriptions import create_model # parse the command line arguments (generates the backend) parser = NeonArgparser(__doc__) parser.add_argument('--model_type', help='Name of model', required=True, choices=['alexnet', 'vgg']) parser.add_argument('--model_tree', help='Whether or not to train tree of classifiers', default=False, type=bool) parser.add_argument('--freeze', type=int, help='Layers to freeze starting from end', default=0) parser.add_argument('--dataset_dir', help='Directory containing images folder and label text files') args = parser.parse_args() # setup data provider train_set_options = dict(repo_dir=args.data_dir, inner_size=224, dtype=args.datatype, subset_pct=100) test_set_options = dict(repo_dir=args.data_dir, inner_size=224, dtype=args.datatype, subset_pct=20)
from neon.data.dataloaders import load_imdb from neon.data.dataiterator import ArrayIterator from neon.data.text_preprocessing import pad_data from neon.initializers import Uniform, GlorotUniform from neon.layers import (GeneralizedCost, LSTM, Affine, Dropout, LookupTable, RecurrentSum, Recurrent, DeepBiLSTM, DeepBiRNN) from neon.models import Model from neon.optimizers import Adagrad from neon.transforms import Logistic, Tanh, Softmax, CrossEntropyMulti, Accuracy from neon.callbacks.callbacks import Callbacks from neon.util.argparser import NeonArgparser, extract_valid_args # parse the command line arguments parser = NeonArgparser(__doc__) parser.add_argument('--rlayer_type', default='lstm', choices=['bilstm', 'lstm', 'birnn', 'bibnrnn', 'rnn'], help='type of recurrent layer to use (lstm, bilstm, rnn, birnn, bibnrnn)') args = parser.parse_args(gen_be=False) # hyperparameters from the reference args.batch_size = 128 gradient_clip_value = 15 vocab_size = 20000 sentence_length = 128 embedding_dim = 128 hidden_size = 128 reset_cells = True # setup backend be = gen_backend(**extract_valid_args(args, gen_backend))
yield self.X_dev, self.y_dev # replicate neon's mse error metric def err(y, t): feature_axis = 1 return (0.5 * np.square(y - t).mean(axis=feature_axis).mean()) if __name__ == '__main__': # parse the command line arguments parser = NeonArgparser(__doc__) parser.add_argument( '--curvetype', default='Lissajous1', choices=['Lissajous1', 'Lissajous2'], help='type of input curve data to use (Lissajous1 or Lissajous2)') args = parser.parse_args(gen_be=False) # network hyperparameters hidden = 32 args.batch_size = 1 # The following flag will switch between 2 training strategies: # 1. return_sequence True: # Inputs are sequences, and target outputs will be sequences. # The RNN layer's output at EVERY step will be used for errors and optimized. # The RNN model contains a RNN layer and an Affine layer # The data iterator will format the data accordingly, and will stride along the # whole series with no overlap
Train WGAN to generate MNIST images. """ import os from datetime import datetime from neon.callbacks.callbacks import Callbacks, GANCostCallback from neon.callbacks.plotting_callbacks import GANPlotCallback from neon.optimizers import RMSProp from neon.util.argparser import NeonArgparser from neon.util.persist import ensure_dirs_exist from network_gan import create_model from neon.data.image import MNIST # parse the command line arguments parser = NeonArgparser(__doc__, default_overrides={'epochs': 32, 'rng_seed': 0, 'batch_size': 64}) parser.add_argument('-D', '--dmodel', type=str, default='dc', help='discriminator model type: dc or mlp, default dc') parser.add_argument('-G', '--gmodel', type=str, default='dc', help='generator model type: dc or mlp, default dc') parser.add_argument('--n_dis_ftr', type=int, default=64, help='base discriminator feature number, default 64') parser.add_argument('--n_gen_ftr', type=int, default=64, help='base generator feature number, default 64') args = parser.parse_args() random_seed = args.rng_seed if args.rng_seed else 0 # load up the mnist data set, padding images to size 32 dataset = MNIST(path=args.data_dir, sym_range=True, size=32, shuffle=True) train = dataset.train_iter # create a GAN model, cost = create_model(dis_model=args.dmodel, gen_model=args.gmodel,
from glob import glob import functools import gzip from multiprocessing import Pool import numpy as np import os import tarfile import struct from PIL import Image as PILImage from neon.util.compat import range, StringIO from neon.util.persist import load_obj, save_obj from neon.data import load_i1kmeta from neon.util.argparser import NeonArgparser parser = NeonArgparser(__doc__) parser.add_argument('--set_type', help='(i1k|directory)', required=True, choices=['i1k', 'directory']) parser.add_argument('--image_dir', help='Directory to find images', required=True) parser.add_argument('--target_size', type=int, default=256, help='Size in pixels to scale images (Must be 256 for i1k dataset)') parser.add_argument('--macro_size', type=int, default=5000, help='Images per processed batch') parser.add_argument('--file_pattern', default='*.jpg', help='Image extension to include in' 'directory crawl') args = parser.parse_args() logger = logging.getLogger(__name__) # NOTE: We have to leave this helper function out of the class to use multiprocess pool.map def proc_img(target_size, squarecrop, is_string=False, imgfile=None): imgfile = StringIO(imgfile) if is_string else imgfile im = PILImage.open(imgfile)
from neon.data import ArrayIterator from neon.initializers import Uniform, GlorotUniform, Array from neon.layers import GeneralizedCost, Affine, Dropout, LookupTable, LSTM, RecurrentSum from neon.models import Model from neon.optimizers import Adagrad from neon.transforms import Logistic, Tanh, Softmax, CrossEntropyMulti, Accuracy from neon.util.argparser import NeonArgparser, extract_valid_args from neon.callbacks.callbacks import Callbacks from neon.data.text_preprocessing import get_paddedXY, get_google_word2vec_W import h5py import cPickle # parse the command line arguments parser = NeonArgparser(__doc__) parser.add_argument('-f', '--review_file', default='labeledTrainData.tsv', help='input movie review file') parser.add_argument('--vocab_file', default='labeledTrainData.tsv.vocab', help='output file to save the processed vocabulary') parser.add_argument('--use_w2v', action='store_true', help='use downloaded Google Word2Vec') parser.add_argument('--w2v', default='GoogleNews-vectors-negative300.bin', help='the pre-built Word2Vec') args = parser.parse_args() # hyperparameters hidden_size = 128 embedding_dim = 128
from neon.util.argparser import NeonArgparser from util import create_frcn_model do_plots = True try: import matplotlib.pyplot as plt plt.switch_backend('agg') except ImportError: neon_logger.display('matplotlib needs to be installed manually to generate plots needed ' 'for this example. Skipping plot generation') do_plots = False # parse the command line arguments parser = NeonArgparser(__doc__) parser.add_argument('--img_prefix', type=str, help='prefix for the saved image file names. If None, use ' 'the model file name') args = parser.parse_args(gen_be=True) assert args.model_file is not None, "need a model file to do Fast R-CNN testing" if args.img_prefix is None: args.img_prefix = os.path.splitext(os.path.basename(args.model_file))[0] output_dir = os.path.join(args.data_dir, 'frcn_output') if not os.path.isdir(output_dir): os.mkdir(output_dir) # hyperparameters args.batch_size = 1 n_mb = 40 img_per_batch = args.batch_size
from neon.layers import Conv, Pooling, MergeBroadcast, BranchNode, Affine, Tree, Dropout from neon.layers import GeneralizedCost, Multicost from neon.initializers import Constant, Xavier from neon.optimizers import GradientDescentMomentum, PolySchedule, MultiOptimizer from neon.transforms import Rectlin, Softmax, CrossEntropyMulti from neon.models import Model from neon.data import ImageLoader from neon.callbacks.callbacks import Callbacks from neon.util.argparser import NeonArgparser from neon.util.persist import load_obj, save_obj from neon import logger as neon_logger # parse the command line arguments parser = NeonArgparser(__doc__) parser.add_argument( '--resume', action='store_true', help='first run without this and then run with so resume from saved file') args = parser.parse_args() if args.backend == 'mgpu': batch_size = 32 * 8 # for some reason bs 128 does not work with bias layers else: batch_size = 32 # subset pct is set to make sure that every epoch has the same mb count img_set_options = dict(repo_dir=args.data_dir, inner_size=224, dtype=np.float32, subset_pct=0.09990891117239205) train = ImageLoader(set_name='train', scale_range=(256, 256),
from ssd_dataloader import build_dataloader from callbacks import MAP_Callback, ssd_image_callback from mboxloss import MBoxLoss from collections import OrderedDict import json """ Trains the SSD model on the provided dataset. The ingest scripts for datasets are stored in the /datasets directory. We include ingest scripts for PASCALVOC, KITTI, and SPACENET datasets. """ # Parse the command line arguments arg_defaults = {'batch_size': 0} parser = NeonArgparser(__doc__, default_overrides=arg_defaults) parser.add_argument('--height', type=int, help='image height') parser.add_argument('--width', type=int, help='image width') parser.add_argument('--subset_pct', type=float, default=100.0, help='fraction of full training data set to use') parser.add_argument('--ssd_config', action='append', required=True, help='path to ssd json file') parser.add_argument('--lr_scale', type=float, default=1.0, help='scale lr by this amount') parser.add_argument('--image_sample_dir', type=str, help='path to save image samples') parser.add_argument('--num_images', type=int, help='number of images to save') parser.add_argument('--lr_step', type=int, action='append', help='epochs to step lr') args = parser.parse_args(gen_be=False) if args.ssd_config: args.ssd_config = {k: v for k, v in [ss.split(':') for ss in args.ssd_config]} # directory to store VGG weights cache_dir = get_data_cache_dir(args.data_dir, subdir='ssd_cache')
import os from neon.util.argparser import NeonArgparser from neon.optimizers import GradientDescentMomentum, Schedule from neon.transforms import TopKMisclassification from neon.callbacks.callbacks import Callbacks, BatchNormTuneCallback from data import make_msra_train_loader, make_validation_loader, make_tuning_loader from network_msra import create_network # parse the command line arguments (generates the backend) train_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'train.cfg') config_files = [train_config] if os.path.exists(train_config) else [] parser = NeonArgparser(__doc__, default_config_files=config_files) parser.add_argument('--depth', type=int, default=0, help='network configuration') parser.add_argument( '--bottleneck', action="store_true", help="use bottleneck modules compared to double 3x3 modules") parser.add_argument('--subset_pct', type=float, default=100, help='subset of training dataset to use (percentage)') args = parser.parse_args() model, cost = create_network(args.depth, args.bottleneck) rseed = 0 if args.rng_seed is None else args.rng_seed # setup data provider
from neon.backends import gen_backend from neon.data.dataloader_transformers import TypeCast import numpy as np from neon.initializers import Kaiming, IdentityInit, Constant from neon.layers import Conv, Pooling, GeneralizedCost, Affine, Activation, Dropout from neon.layers import MergeSum, SkipNode, BatchNorm from neon.data.datasets import Dataset from neon.util.persist import load_obj import os # parse the command line arguments parser = NeonArgparser(__doc__) parser.add_argument('--depth', type=int, default=18, help='choices 18, 34, 50, 101, 152') args = parser.parse_args() # hyperparameters num_epochs = args.epochs # Next line gets rid of the deterministic warning args.deterministic = None if (args.rng_seed is None): args.rng_seed = 16 print('Batch size = {}'.format(args.batch_size))
from neon.transforms import TopKMisclassification from neon.callbacks.callbacks import Callbacks from data import make_alexnet_train_loader, make_validation_loader from network_allcnn import create_network # parse the command line arguments (generates the backend) train_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'train.cfg') config_files = [train_config] if os.path.exists(train_config) else [] parser = NeonArgparser(__doc__, default_config_files=config_files, default_overrides=dict(batch_size=64)) parser.add_argument('--deconv', action='store_true', help='save visualization data from deconvolution') parser.add_argument('--subset_pct', type=float, default=100, help='subset of training dataset to use (percentage)') args = parser.parse_args() model, cost = create_network() rseed = 0 if args.rng_seed is None else args.rng_seed # setup data provider assert 'train' in args.manifest, "Missing train manifest" assert 'val' in args.manifest, "Missing validation manifest" train = make_alexnet_train_loader(args.manifest['train'], args.manifest_root, model.be, args.subset_pct, rseed)
from neon import logger as neon_logger from neon.util.argparser import NeonArgparser from neon.initializers import Kaiming from neon.layers import Conv, Pooling, GeneralizedCost, Activation from neon.layers import MergeSum, SkipNode from neon.optimizers import GradientDescentMomentum, Schedule from neon.transforms import Rectlin, Softmax, CrossEntropyMulti, TopKMisclassification from neon.models import Model from neon.data import ImageLoader from neon.callbacks.callbacks import Callbacks, BatchNormTuneCallback import itertools as itt import sys # parse the command line arguments (generates the backend) parser = NeonArgparser(__doc__) parser.add_argument('--depth', type=int, default=0, help='network configuration') parser.add_argument('--bottleneck', action="store_true", help="use bottleneck modules compared to double 3x3 modules") parser.add_argument('--subset_pct', type=float, default=100, help='subset of training dataset to use (percentage)') args = parser.parse_args() if args.depth in (0, 18): stages = (2, 2, 2, 2) elif args.depth in (1, 34, 50): stages = (3, 4, 6, 3) elif args.depth in (2, 68, 101): stages = (3, 4, 23, 3) elif args.depth in (3, 102, 152): stages = (3, 8, 36, 3) elif args.depth in (4, 98, 138):
from neon import logger as neon_logger from neon.models import Model from aeon import DataLoader from neon.util.argparser import NeonArgparser, extract_valid_args from neon.transforms import Misclassification, PrecisionRecall from neon.backends import gen_backend from neon.data.dataloader_transformers import TypeCast, OneHot import numpy as np import pandas as pd from neon.data import HDF5IteratorOneHot, HDF5Iterator import h5py # parse the command line arguments parser = NeonArgparser(__doc__) parser.add_argument('--subset', type=int, default=9) args = parser.parse_args() subset = args.subset testFileName = '/mnt/data/medical/luna16/luna16_roi_subset{}_ALL.h5'.format(subset) print('Using test file: {}'.format(testFileName)) # Next line gets rid of the deterministic warning args.deterministic = None if (args.rng_seed is None): args.rng_seed = 16 print('Batch size = {}'.format(args.batch_size))
from neon import logger as neon_logger from neon.backends import gen_backend from neon.data import PASCALVOCTrain from neon.transforms import CrossEntropyMulti, SmoothL1Loss, ObjectDetection from neon.util.argparser import NeonArgparser, extract_valid_args from neon.optimizers import GradientDescentMomentum, MultiOptimizer from neon.callbacks.callbacks import Callbacks from neon.layers import Multicost, GeneralizedCostMask from neon.util.persist import save_obj from util import load_vgg_weights, create_frcn_model, scale_bbreg_weights # main script # parse the command line arguments parser = NeonArgparser(__doc__, default_overrides=dict(batch_size=4)) parser.add_argument('--subset_pct', type=float, default=100, help='subset of training dataset to use (percentage)') args = parser.parse_args(gen_be=False) # Override save path if None if args.save_path is None: args.save_path = 'frcn_vgg.pkl' if args.callback_args['save_path'] is None: args.callback_args['save_path'] = args.save_path if args.callback_args['serialize'] is None: args.callback_args['serialize'] = min(args.epochs, 10) # hyperparameters args.batch_size = 4
from __future__ import division from neon.backends import gen_backend from neon.util.argparser import NeonArgparser, extract_valid_args from neon.optimizers import GradientDescentMomentum, MultiOptimizer, StepSchedule from neon.callbacks.callbacks import Callbacks, TrainMulticostCallback from neon.util.persist import save_obj from objectlocalization import PASCAL from neon.transforms import CrossEntropyMulti, SmoothL1Loss from neon.layers import Multicost, GeneralizedCostMask import util # parse the command line arguments parser = NeonArgparser(__doc__, default_overrides={'batch_size': 1}) parser.add_argument('--subset_pct', type=float, default=100, help='subset of training dataset to use (percentage)') args = parser.parse_args(gen_be=False) # hyperparameters assert args.batch_size is 1, "Faster-RCNN only supports batch size 1" n_mb = None rpn_rois_per_img = 256 # number of rois to sample to train rpn frcn_rois_per_img = 128 # number of rois to sample to train frcn # setup backend be = gen_backend(**extract_valid_args(args, gen_backend)) be.enable_winograd = 4 # default to winograd 4 for fast autotune year = '2007'
from neon.optimizers import GradientDescentMomentum, MultiOptimizer, StepSchedule from neon.callbacks.callbacks import Callbacks from neon.util.persist import save_obj, get_data_cache_dir from objectlocalization import PASCALVOC from neon.transforms import CrossEntropyMulti, SmoothL1Loss from neon.layers import Multicost, GeneralizedCostMask import util import faster_rcnn import os train_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'pascalvoc.cfg') config_files = [train_config] if os.path.exists(train_config) else [] # parse the command line arguments parser = NeonArgparser(__doc__, default_config_files=config_files) parser.add_argument('--width', type=int, default=1000, help='Width of input image') parser.add_argument('--height', type=int, default=1000, help='Height of input image') parser.add_argument('--subset_pct', type=float, default=100, help='subset of training dataset to use (percentage)') args = parser.parse_args(gen_be=False) # hyperparameters assert args.batch_size is 1, "Faster-RCNN only supports batch size 1" assert 'train' in args.manifest rpn_rois_per_img = 256 # number of rois to sample to train rpn frcn_rois_per_img = 128 # number of rois to sample to train frcn # setup backend be = gen_backend(**extract_valid_args(args, gen_backend)) be.enable_winograd = 4 # default to winograd 4 for fast autotune
self.batch_index += 1 yield self.X_dev, self.y_dev # replicate neon's mse error metric def err(y, t): feature_axis = 1 return (0.5 * np.square(y - t).mean(axis=feature_axis).mean()) if __name__ == '__main__': # parse the command line arguments parser = NeonArgparser(__doc__) parser.add_argument('--curvetype', default='Lissajous1', choices=['Lissajous1', 'Lissajous2'], help='type of input curve data to use (Lissajous1 or Lissajous2)') args = parser.parse_args(gen_be=False) # network hyperparameters hidden = 32 args.batch_size = 1 # The following flag will switch between 2 training strategies: # 1. return_sequence True: # Inputs are sequences, and target outputs will be sequences. # The RNN layer's output at EVERY step will be used for errors and optimized. # The RNN model contains a RNN layer and an Affine layer # The data iterator will format the data accordingly, and will stride along the # whole series with no overlap # 2. return_sequence False: # Inputs are sequences, and target output will be a single step.
""" import os from neon.util.argparser import NeonArgparser from neon.layers import Conv, Pooling, MergeBroadcast, BranchNode, Affine, Tree, Dropout from neon.layers import GeneralizedCost, Multicost from neon.initializers import Constant, Xavier from neon.backends import gen_backend from neon.optimizers import GradientDescentMomentum, MultiOptimizer from neon.transforms import Rectlin, Softmax, CrossEntropyMulti, TopKMisclassification from neon.models import Model from neon.data import ImageLoader parser = NeonArgparser(__doc__) parser.add_argument('--subset_pct', type=float, default=100, help='subset of training dataset to use (percentage)') parser.add_argument('--test_only', action='store_true', help='skip fitting - evaluate metrics on trained model weights') args = parser.parse_args() # setup data provider img_set_options = dict(repo_dir=args.data_dir, inner_size=224, dtype=args.datatype, subset_pct=args.subset_pct) test = ImageLoader(set_name='validation', scale_range=(256, 256), do_transforms=False, **img_set_options) init1 = Xavier(local=False) initx = Xavier(local=True) bias = Constant(val=0.20) relu = Rectlin()
from neon.data.dataiterator import ArrayIterator from neon.data.text_preprocessing import pad_data from neon.initializers import Uniform, GlorotUniform from neon.layers import (GeneralizedCost, LSTM, Affine, Dropout, LookupTable, RecurrentSum, Recurrent, DeepBiLSTM, DeepBiRNN) from neon.models import Model from neon.optimizers import Adagrad from neon.transforms import Logistic, Tanh, Softmax, CrossEntropyMulti, Accuracy from neon.callbacks.callbacks import Callbacks from neon.util.argparser import NeonArgparser, extract_valid_args # parse the command line arguments parser = NeonArgparser(__doc__) parser.add_argument( '--rlayer_type', default='lstm', choices=['bilstm', 'lstm', 'birnn', 'bibnrnn', 'rnn'], help='type of recurrent layer to use (lstm, bilstm, rnn, birnn, bibnrnn)') args = parser.parse_args(gen_be=False) # hyperparameters from the reference args.batch_size = 128 gradient_clip_value = 15 vocab_size = 20000 sentence_length = 128 embedding_dim = 128 hidden_size = 128 reset_cells = True # setup backend
from neon.util.persist import load_obj from neon.util.argparser import NeonArgparser, extract_valid_args from neon.initializers import Uniform, Orthonormal, Constant from neon.layers import GeneralizedCostMask, Multicost, GRU, SkipThought from neon.models import Model from neon.transforms import CrossEntropyMulti, Logistic, Tanh from neon.callbacks.callbacks import Callbacks, MetricCallback from neon.optimizers import Adam from neon import logger as neon_logger from data_loader import load_data from data_iterator import SentenceHomogenous # parse the command line arguments parser = NeonArgparser(__doc__) parser.add_argument('--output_dir', default='output/', help='choose the directory to save the files') parser.add_argument('--max_vocab_size', default=20000, help='number of (most frequent) words to use from vocabulary') parser.add_argument('--max_len_w', default=30, help='maximum sentence length for training') parser.add_argument('--subset_pct', type=float, default=100, help='subset of training dataset to use (percentage)') args = parser.parse_args(gen_be=False) # hyperparameters from the reference args.batch_size = 64 embed_dim = 620 valid_split = None # setup backend be = gen_backend(**extract_valid_args(args, gen_backend))
def caption_video(infile, caption, outfile): cmd = '''ffmpeg -i {0} -an \ -vf drawtext="textfile={1}: fontcolor=white: fontsize=16: box=1: [email protected]" \ -y {2}''' proc = subprocess.Popen(cmd.format(infile, caption, outfile), shell=True) proc.communicate() # parse the command line arguments demo_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'test.cfg') config_files = [demo_config] if os.path.exists(demo_config) else [] parser = NeonArgparser(__doc__, default_config_files=config_files) parser.add_argument('--input_video', help='video file') parser.add_argument('--output_video', help='Video file with overlayed inference hypotheses') args = parser.parse_args() assert args.model_file is not None, "need a model file for testing" model = Model(args.model_file) assert 'categories' in args.manifest, "Missing categories file" category_map = {t[0]: t[1] for t in np.genfromtxt(args.manifest['categories'], dtype=None, delimiter=',')} # Make a temporary directory and clean up afterwards outdir = mkdtemp() atexit.register(shutil.rmtree, outdir) caption_file = os.path.join(outdir, 'caption.txt')
import spacy from neon.backends import gen_backend from neon.models import Model from neon.util.argparser import NeonArgparser, extract_valid_args from nlp_architect.contrib.neon.text_iterators import TaggedTextSequence, MultiSequenceDataIterator from nlp_architect.utils.generic import get_paddedXY_sequence from nlp_architect.utils.embedding import load_word_embeddings from nlp_architect.utils.io import validate_existing_filepath from utils import extract_nps if __name__ == '__main__': parser = NeonArgparser() parser.add_argument('--model', type=validate_existing_filepath, required=True, help='Path to model file') parser.add_argument('--settings', type=validate_existing_filepath, required=True, help='Path to model settings file') parser.add_argument( '--input', type=validate_existing_filepath, required=True, help='Input texts file path (samples to pass for inference)') parser.add_argument('--embedding_model', type=validate_existing_filepath, help='Pre-trained word embedding model file path') parser.add_argument('--print_only_nps', default=True,
from future import standard_library standard_library.install_aliases() # triggers E402, hence noqa below from builtins import input # noqa import numpy as np # noqa from neon.backends import gen_backend # noqa from neon.initializers import Uniform, GlorotUniform # noqa from neon.layers import LSTM, Affine, Dropout, LookupTable, RecurrentSum # noqa from neon.models import Model # noqa from neon.transforms import Logistic, Tanh, Softmax # noqa from neon.util.argparser import NeonArgparser, extract_valid_args # noqa from neon.util.compat import pickle # noqa from neon.data.text_preprocessing import clean_string # noqa # parse the command line arguments parser = NeonArgparser(__doc__) parser.add_argument( '--model_weights', required=True, help='pickle file of trained weights') parser.add_argument('--vocab_file', required=True, help='vocabulary file') args = parser.parse_args() # hyperparameters from the reference batch_size = 1 clip_gradients = True gradient_limit = 5 vocab_size = 20000 sentence_length = 128 embedding_dim = 128 hidden_size = 128 reset_cells = True num_epochs = args.epochs # setup backend
""" logging information and command parsing Can specify these argument on command line or here """ parser = NeonArgparser(__doc__) argss=[ "--file_name","--train_num_p","--valid_num_p","--test_num_p", "--train_num_n","--valid_num_n","--test_num_n","--nclass", #data dependent arguments "--data_dict","--norm_type", #output dependent arguments "--out_dir"] for a in argss: parser.add_argument(a) parser.set_defaults( #constant arguments rng_seed=2, backend= "cpu", progress_bar=True, verbose=4, evaluation_freq=2, #data epochs= 10, batch_size=128, data_dir="/global/project/projectdirs/nervana/yunjie/climate_neon1.0run/conv/DATA/", file_name="hurricanes.h5",
""" from neon.backends import gen_backend from neon.data import Text, load_text from neon.initializers import Uniform from neon.layers import GeneralizedCost, LSTM, Affine, GRU, LookupTable from neon.models import Model from neon.optimizers import GradientDescentMomentum, Schedule from neon.transforms import Logistic, Tanh, Softmax, CrossEntropyMulti from neon.callbacks.callbacks import Callbacks from neon.util.argparser import NeonArgparser, extract_valid_args # parse the command line arguments parser = NeonArgparser(__doc__) parser.add_argument('--rlayer_type', default='lstm', choices=['gru', 'lstm'], help='type of recurrent layer to use (gru or lstm)') args = parser.parse_args(gen_be=False) # hyperparameters from the reference args.batch_size = 20 time_steps = 20 hidden_size = 200 gradient_clip_norm = 5 # setup backend be = gen_backend(**extract_valid_args(args, gen_backend)) # download penn treebank train_path = load_text('ptb-train', path=args.data_dir) valid_path = load_text('ptb-valid', path=args.data_dir)
import numpy as np from neon.util.argparser import NeonArgparser from neon.initializers import Gaussian, GlorotUniform from neon.layers import Conv, Pooling, GeneralizedCost, Affine from neon.layers import DeepBiRNN, RecurrentMean from neon.optimizers import Adagrad from neon.transforms import Tanh, Rectlin, Softmax, CrossEntropyBinary from neon.models import Model from neon.data import DataLoader, AudioParams from neon.callbacks.callbacks import Callbacks from sklearn import metrics from indexer import Indexer parser = NeonArgparser(__doc__) parser.add_argument('-elec', '--electrode', default=0, help='electrode index') parser.add_argument('-out', '--out_dir', default='preds', help='directory to write output files') parser.add_argument('-test', '--test_mode', action="store_true", help="testing mode") args = parser.parse_args() pattern = '*.' + str(args.electrode) + '.wav' data_dir = args.data_dir out_dir = args.out_dir if not os.path.exists(out_dir): os.makedirs(out_dir) if data_dir[-1] != '/': data_dir += '/' subj = int(data_dir[-2]) assert subj in [1, 2, 3] indexer = Indexer() tain_idx, test_idx = indexer.run(data_dir, pattern, testing=args.test_mode)
self.subj = subj self.data_dir = data_dir self.eval_set = eval_set def on_epoch_end(self, callback_data, model, epoch): preds = model.get_outputs(self.eval_set)[:, 1] idx_file = os.path.join( self.data_dir, 'eval-' + str(self.subj) + '-' + str(0) + '-index.csv') labels = np.loadtxt(idx_file, delimiter=',', skiprows=1, usecols=[1]) logger.display('Eval AUC for subject %d epoch %d: %.4f\n' % (self.subj, epoch, score(labels, preds))) parser = NeonArgparser(__doc__) parser.add_argument('-elec', '--electrode', default=0, help='electrode index') parser.add_argument('-out', '--out_dir', default='preds', help='directory to write output files') parser.add_argument('-validate', '--validate_mode', action="store_true", help="validate on training data") args = parser.parse_args() data_dir = os.path.normpath(args.data_dir) out_dir = args.out_dir if not os.path.exists(out_dir): os.makedirs(out_dir) subj = int(data_dir[-1])
--model_weights <trained_pickle_file> """ from builtins import map, zip import cv2 import numpy as np import os from neon.backends import gen_backend from neon.data import DataLoader, VideoParams, ImageParams from neon.util.argparser import NeonArgparser, extract_valid_args from network import create_network # parse the command line arguments parser = NeonArgparser(__doc__) parser.add_argument('--class_ind_file', help='Path of two column file mapping integer' 'class labels to their canonical names.') parser.add_argument('--model_weights', help='Pickle file of trained model weights.') args = parser.parse_args(gen_be=False) # setup backend be = gen_backend(**extract_valid_args(args, gen_backend)) be.bsz = 32 # setup data provider shape = dict(channel_count=3, height=112, width=112, scale_min=128, scale_max=128) testParams = VideoParams(frame_params=ImageParams(center=True, flip=False, **shape), frames_per_clip=16) common = dict(target_size=1, nclasses=101, datum_dtype=np.uint8)
from neon.data import ArrayIterator from neon.initializers import Uniform, GlorotUniform from neon.layers import GeneralizedCost, Affine, Dropout, LookupTable, LSTM, RecurrentSum from neon.models import Model from neon.optimizers import Adagrad from neon.transforms import Logistic, Tanh, Softmax, CrossEntropyMulti, Accuracy from neon.util.argparser import NeonArgparser, extract_valid_args from neon.callbacks.callbacks import Callbacks from neon.data.text_preprocessing import get_paddedXY, get_google_word2vec_W import h5py import cPickle # parse the command line arguments parser = NeonArgparser(__doc__) parser.add_argument('-f', '--review_file', default='labeledTrainData.tsv', help='input movie review file') parser.add_argument('--vocab_file', default='labeledTrainData.tsv.vocab', help='output file to save the processed vocabulary') parser.add_argument('--use_w2v', action='store_true', help='use downloaded Google Word2Vec') parser.add_argument('--w2v', default='GoogleNews-vectors-negative300.bin', help='the pre-built Word2Vec') args = parser.parse_args() # hyperparameters hidden_size = 128 embedding_dim = 128
def caption_video(infile, caption, outfile): cmd = '''ffmpeg -i {0} -an \ -vf drawtext="textfile={1}: fontcolor=white: fontsize=16: box=1: [email protected]" \ -y {2}''' proc = subprocess.Popen(cmd.format(infile, caption, outfile), shell=True) proc.communicate() # parse the command line arguments demo_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'test.cfg') config_files = [demo_config] if os.path.exists(demo_config) else [] parser = NeonArgparser(__doc__, default_config_files=config_files) parser.add_argument('--input_video', help='video file') parser.add_argument('--output_video', help='Video file with overlayed inference hypotheses') args = parser.parse_args() assert args.model_file is not None, "need a model file for testing" model = Model(args.model_file) assert 'categories' in args.manifest, "Missing categories file" category_map = { t[0]: t[1] for t in np.genfromtxt( args.manifest['categories'], dtype=None, delimiter=',') } # Make a temporary directory and clean up afterwards
import sys from neon.util.argparser import NeonArgparser from neon.backends import gen_backend from neon.initializers import Constant, Gaussian from neon.layers import Conv, Dropout, Pooling, GeneralizedCost, Affine from neon.optimizers import GradientDescentMomentum, MultiOptimizer, Schedule from neon.transforms import Rectlin, Softmax, CrossEntropyMulti, TopKMisclassification from neon.models import Model from neon.data import ImgMaster from neon.callbacks.callbacks import Callbacks, Callback # For running complete alexnet # alexnet.py -e 90 -val 1 -s <save-path> -w <path-to-saved-batches> # parse the command line arguments parser = NeonArgparser(__doc__) parser.add_argument('--model_file', help='load model from pkl file') args = parser.parse_args() # hyperparameters batch_size = 128 # setup backend be = gen_backend(backend=args.backend, rng_seed=args.rng_seed, device_id=args.device_id, batch_size=batch_size, default_dtype=args.datatype) try: train = ImgMaster(repo_dir=args.data_dir, inner_size=224, set_name='train') test = ImgMaster(repo_dir=args.data_dir, inner_size=224, set_name='validation', do_transforms=False) except (OSError, IOError, ValueError) as err: print err
from neon.callbacks.callbacks import Callbacks, GANCostCallback from neon.callbacks.plotting_callbacks import GANPlotCallback from neon.initializers import Gaussian from neon.layers import GeneralizedGANCost, Sequential, Conv, Deconv, Dropout, Pooling from neon.layers.layer import Linear, Reshape from neon.layers.container import GenerativeAdversarial from neon.models.model import GAN from neon.transforms import Rectlin, Logistic, GANCost from neon.util.argparser import NeonArgparser from neon.util.persist import ensure_dirs_exist from neon.layers.layer import Dropout # parse the command line arguments parser = NeonArgparser(__doc__) parser.add_argument('--kbatch', type=int, default=1, help='number of data batches per noise batch in training') args = parser.parse_args() # load up the data set # setup weight initialization function init = Gaussian() # discriminiator using convolution layers lrelu = Rectlin(slope=0.1) # leaky relu for discriminator # sigmoid = Logistic() # sigmoid activation function conv1 = dict( init=init, batch_norm=False, activation=lrelu) # what's about BatchNorm Layer and batch_norm parameter? conv2 = dict(init=init, batch_norm=True, activation=lrelu, padding=2)
from neon.backends import gen_backend from neon.data.text import Text from neon.data.dataloaders import load_ptb_train, load_ptb_test from neon.initializers import Uniform from neon.layers import GeneralizedCost, LSTM, Affine, GRU, LookupTable from neon.models import Model from neon.optimizers import GradientDescentMomentum, Schedule from neon.transforms import Logistic, Tanh, Softmax, CrossEntropyMulti from neon.callbacks.callbacks import Callbacks from neon.util.argparser import NeonArgparser, extract_valid_args # parse the command line arguments parser = NeonArgparser(__doc__) parser.add_argument('--rlayer_type', default='lstm', choices=['gru', 'lstm'], help='type of recurrent layer to use (gru or lstm)') args = parser.parse_args(gen_be=False) # hyperparameters from the reference args.batch_size = 20 time_steps = 20 hidden_size = 200 gradient_clip_norm = 5 # setup backend be = gen_backend(**extract_valid_args(args, gen_backend)) # download penn treebank train_path = load_ptb_train(path=args.data_dir) valid_path = load_ptb_test(path=args.data_dir)
.. _[Springenberg2014]: http://arxiv.org/pdf/1412.6806.pdf """ from neon.util.argparser import NeonArgparser from neon.backends import gen_backend from neon.initializers import GlorotUniform from neon.optimizers import GradientDescentMomentum, Schedule from neon.layers import Conv, Dropout, Activation, Pooling, GeneralizedCost, DataTransform from neon.transforms import Rectlin, Softmax, CrossEntropyMulti, Normalizer from neon.models import Model from neon.callbacks.callbacks import Callbacks from neon.data import ImgMaster, ImageLoader # parse the command line arguments parser = NeonArgparser(__doc__) parser.add_argument('--deconv', action='store_true', help='save visualization data from deconvolution') parser.add_argument('--loader_version', default='old', choices=['old', 'new'], help='whether to use old dataloader (ImgMaster) or new (ImageLoader)') args = parser.parse_args() # hyperparameters batch_size = 64 # setup backend be = gen_backend(backend=args.backend, batch_size=batch_size, rng_seed=args.rng_seed, device_id=args.device_id, datatype=args.datatype)
--model_file <saved weights file> """ from neon.util.argparser import NeonArgparser from neon.initializers import Constant, Gaussian from neon.layers import Conv, Dropout, Pooling, GeneralizedCost, Affine, LRN from neon.optimizers import GradientDescentMomentum, MultiOptimizer, Schedule from neon.transforms import Rectlin, Softmax, CrossEntropyMulti, TopKMisclassification from neon.models import Model from neon.data import ImageLoader from neon.callbacks.callbacks import Callbacks # parse the command line arguments (generates the backend) parser = NeonArgparser(__doc__) parser.add_argument('--subset_pct', type=float, default=100, help='subset of training dataset to use (percentage)') parser.add_argument( '--test_only', action='store_true', help='skip fitting - evaluate metrics on trained model weights') args = parser.parse_args() if args.test_only: if args.model_file is None: raise ValueError('To test model, trained weights need to be provided') # setup data provider img_set_options = dict(repo_dir=args.data_dir, inner_size=224, subset_pct=args.subset_pct)