def main():
# parse the command line arguments
parser = NeonArgparser(__doc__)
parser.add_argument('--output_path', required=True,
help='Output path used when training model')
parser.add_argument('--w2v_path', required=False, default=None,
help='Path to GoogleNews w2v file for voab expansion.')
parser.add_argument('--eval_data_path', required=False, default='./SICK_data',
help='Path to the SICK dataset for evaluating semantic relateness')
parser.add_argument('--max_vocab_size', required=False, default=1000000,
help='Limit the vocabulary expansion to fit in GPU memory')
parser.add_argument('--subset_pct', required=False, default=100,
help='subset of training dataset to use (use to retreive \
preprocessed data from training)')
args = parser.parse_args(gen_be=True)
# load vocab file from training
_, vocab_file = load_data(args.data_dir, output_path=args.output_path,
subset_pct=float(args.subset_pct))
vocab, _, _ = load_obj(vocab_file)
vocab_size = len(vocab)
neon_logger.display("\nVocab size from the dataset is: {}".format(vocab_size))
index_from = 2 # 0: padding 1: oov
vocab_size_layer = vocab_size + index_from
max_len = 30
# load trained model
model_dict = load_obj(args.model_file)
# Vocabulary expansion trick needs to pass the correct vocab set to evaluate (for tokenization)
if args.w2v_path:
neon_logger.display("Performing Vocabulary Expansion... Loading W2V...")
w2v_vocab, w2v_vocab_size = get_w2v_vocab(args.w2v_path,
int(args.max_vocab_size), cache=True)
vocab_size_layer = w2v_vocab_size + index_from
model = load_sent_encoder(model_dict, expand_vocab=True, orig_vocab=vocab,
w2v_vocab=w2v_vocab, w2v_path=args.w2v_path, use_recur_last=True)
vocab = w2v_vocab
else:
# otherwise stick with original vocab size used to train the model
model = load_sent_encoder(model_dict, use_recur_last=True)
model.initialize(dataset=(max_len, 1))
evaluate(model, vocab=vocab, data_path=args.eval_data_path, evaltest=True,
vocab_size_layer=vocab_size_layer)
def main():
# parse the command line arguments
parser = NeonArgparser(__doc__)
parser.add_argument('--output_path', required=True,
help='Output path used when training model')
parser.add_argument('--w2v_path', required=False, default=None,
help='Path to GoogleNews w2v file for voab expansion.')
parser.add_argument('--eval_data_path', required=False, default='./SICK_data',
help='Path to the SICK dataset for evaluating semantic relateness')
parser.add_argument('--max_vocab_size', required=False, default=1000000,
help='Limit the vocabulary expansion to fit in GPU memory')
parser.add_argument('--subset_pct', required=False, default=100,
help='subset of training dataset to use (use to retreive \
preprocessed data from training)')
args = parser.parse_args(gen_be=True)
# load vocab file from training
_, vocab_file = load_data(args.data_dir, output_path=args.output_path,
subset_pct=float(args.subset_pct))
vocab, _, _ = load_obj(vocab_file)
vocab_size = len(vocab)
neon_logger.display("\nVocab size from the dataset is: {}".format(vocab_size))
index_from = 2 # 0: padding 1: oov
vocab_size_layer = vocab_size + index_from
max_len = 30
# load trained model
model_dict = load_obj(args.model_file)
# Vocabulary expansion trick needs to pass the correct vocab set to evaluate (for tokenization)
if args.w2v_path:
neon_logger.display("Performing Vocabulary Expansion... Loading W2V...")
w2v_vocab, w2v_vocab_size = get_w2v_vocab(args.w2v_path,
int(args.max_vocab_size), cache=True)
vocab_size_layer = w2v_vocab_size + index_from
model = load_sent_encoder(model_dict, expand_vocab=True, orig_vocab=vocab,
w2v_vocab=w2v_vocab, w2v_path=args.w2v_path, use_recur_last=True)
vocab = w2v_vocab
else:
# otherwise stick with original vocab size used to train the model
model = load_sent_encoder(model_dict, use_recur_last=True)
model.initialize(dataset=(max_len, 1))
evaluate(model, vocab=vocab, data_path=args.eval_data_path, evaltest=True,
vocab_size_layer=vocab_size_layer)
def get_p1b1_parser():
# Construct neon arg parser. It generates a large set of options by default
parser = NeonArgparser(__doc__)
# Specify the default config_file
parser.add_argument("--config_file",
dest='config_file',
type=str,
default=os.path.join(p1b1.file_path,
'p1b1_default_model.txt'),
help="specify model configuration file")
# Parse other options that are not included on neon arg parser
parser = p1_common.get_p1_common_parser(parser)
return parser
from neon.callbacks.callbacks import Callbacks, TrainMulticostCallback
from neon.transforms.cost import Misclassification
from zmlcore.neonfixes.metrics import MultiMetric
from zmlcore.classifier.classifier import TextClassifier, Config
from zmlcore.classifier.traincallbacks import TrainingProgress, MisclassificationTest
from zmlcore.data.dataiterator import TrainingIterator, BatchIterator
from zmlcore.data.sentiment_loader import SentimentLoader
from zmlcore.neonfixes.transforms import fix_logistic
import os, email, mailbox
# random percentage to holdout for validation when training
holdout_pct = 0.1
if __name__ == '__main__':
p = NeonArgparser(__doc__)
p.add_argument('--word_vectors', type=str, required=False, default='./data/vocabularies/glove.6B.300d.txt',
help='Path to word vector file, including word, followed by vector per line, space separated')
p.add_argument('--lookup_size', type=int, default=0,
help='If non-zero, a lookup table and an auto-vocabulary will be used instead of word vectors.')
p.add_argument('--lookup_dim', type=int, default=100,
help='Word embedding dimensions when lookup_size specified.')
p.add_argument('--exclusive_classes', type=str, required=False,
default='\"finance promos social forums updates\"',
help='The labels of the exclusive classes to either train on or classify. ' +
'Should be a quoted, space separated list.')
p.add_argument('--overlapping_classes', type=str, required=False,
default='\"important\"',
help='The labels of the classes, which can overlap with themselves or an exclusive class. ' +
'Should be a quoted, space separated list.')
p.add_argument('--sentiment_path', type=str, required=False, default=None,
help='This overrides the email classification function altogether to benchmark the content ' +
'classification network only. The parameter is a path to the root directory of ' +
import util
from objectlocalization import PASCALVOC
from neon.backends import gen_backend
from neon.util.persist import get_data_cache_dir, save_obj
from neon.util.argparser import NeonArgparser, extract_valid_args
from neon import logger as neon_logger
from voc_eval import voc_eval
import numpy as np
import faster_rcnn
from tqdm import tqdm
# parse the command line arguments
parser = NeonArgparser(__doc__, default_overrides={'batch_size': 1})
parser.add_argument('--normalize',
action='store_true',
help='Normalize the final bounding box regression layers.')
parser.add_argument('--output',
default=None,
help='File to save inference results (optional)')
parser.add_argument('--width',
type=int,
default=1000,
help='Width of input image')
parser.add_argument('--height',
type=int,
default=1000,
help='Height of input image')
args = parser.parse_args()
# ----------------------------------------------------------------------------
import os
from neon.optimizers import GradientDescentMomentum, Schedule
from neon.transforms import Misclassification
from neon.callbacks.callbacks import Callbacks, BatchNormTuneCallback
from neon.util.argparser import NeonArgparser
from network import create_network
from data import make_train_loader, make_validation_loader, make_tuning_loader
# parse the command line arguments (generates the backend)
train_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'train.cfg')
config_files = [train_config] if os.path.exists(train_config) else []
parser = NeonArgparser(__doc__, default_config_files=config_files)
parser.add_argument('--depth', type=int, default=2,
help='depth of each stage (network depth will be 9n+2)')
parser.add_argument('--subset_pct', type=float, default=100,
help='subset of training dataset to use (percentage)')
args = parser.parse_args()
random_seed = args.rng_seed if args.rng_seed else 0
# Check that the proper manifest sets have been supplied
assert 'train' in args.manifest, "Missing train manifest"
assert 'val' in args.manifest, "Missing validation manifest"
model, cost = create_network(args.depth)
# setup data provider
train = make_train_loader(args.manifest['train'], args.manifest_root, model.be, args.subset_pct,
random_seed)
test = make_validation_loader(args.manifest['val'], args.manifest_root, model.be, args.subset_pct)
from network_gan import create_model
from lsun_data import make_loader
# parse the command line arguments
train_config = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'train.cfg')
config_files = [train_config] if os.path.exists(train_config) else []
parser = NeonArgparser(__doc__,
default_config_files=config_files,
default_overrides={
'rng_seed': 0,
'batch_size': 64
})
parser.add_argument('-D',
'--dmodel',
type=str,
default='dc',
help='discriminator model type: dc or mlp, default dc')
parser.add_argument('-G',
'--gmodel',
type=str,
default='dc',
help='generator model type: dc or mlp, default dc')
parser.add_argument(
'--subset_pct',
type=float,
default=100,
help='subset of training dataset to use (percentage), default 100')
args = parser.parse_args()
random_seed = args.rng_seed if args.rng_seed else 0
Usage:
use -t to specify which bAbI task to run
python examples/babi/demo.py -t 1 --rlayer_type gru --model_weights babi.p
"""
import numpy as np
from util import create_model, babi_handler
from neon.backends import gen_backend
from neon.data import BABI, QA
from neon.data.text import Text
from neon.util.argparser import NeonArgparser, extract_valid_args
# parse the command line arguments
parser = NeonArgparser(__doc__)
parser.add_argument('-t', '--task', type=int, default='1', choices=xrange(1, 21),
help='the task ID to train/test on from bAbI dataset (1-20)')
parser.add_argument('--rlayer_type', default='gru', choices=['gru', 'lstm'],
help='type of recurrent layer to use (gru or lstm)')
parser.add_argument('--model_weights',
help='pickle file of trained weights')
args = parser.parse_args(gen_be=False)
# setup backend
be = gen_backend(**extract_valid_args(args, gen_backend))
be.bsz = 1
# load the bAbI dataset
babi = babi_handler(args.data_dir, args.task)
valid_set = QA(*babi.test)
# create model
neon_logger.display("Running this example requires scipy packages.")
neon_logger.display("try activating your virtualenv then: pip install scipy")
sys.exit(1)
from neon.models import Model
from neon.layers import Activation
from neon.data.datasets import Dataset
from neon.layers import GeneralizedCost
from neon.transforms.cost import Cost
from neon.util.argparser import NeonArgparser
# force use of CPU backend since we require a batch size of 1
# (GPU needs a multiple of 32)
default_overrides = dict(backend='cpu', batch_size=1)
parser = NeonArgparser(__doc__, default_overrides=default_overrides)
parser.add_argument("image", help="Base image to create dream on.")
parser.add_argument("--dream_file", default='dream_out.png',
help="Save dream to named file.")
args = parser.parse_args()
# redirect the dream file to the path of output_file
if args.output_file is None:
output_dir = parser.work_dir
elif osp.isdir(args.output_file):
output_dir = args.output_file
else:
output_dir = osp.dirname(args.output_file)
args.dream_file = osp.expanduser(
osp.join(output_dir, osp.basename(args.dream_file)))
neon_logger.display(
"try activating your virtualenv then: pip install scipy")
sys.exit(1)
from neon.models import Model
from neon.layers import Activation
from neon.data.datasets import Dataset
from neon.layers import GeneralizedCost
from neon.transforms.cost import Cost
from neon.util.argparser import NeonArgparser
# force use of CPU backend since we require a batch size of 1
# (GPU needs a multiple of 32)
default_overrides = dict(backend='cpu', batch_size=1)
parser = NeonArgparser(__doc__, default_overrides=default_overrides)
parser.add_argument("image", help="Base image to create dream on.")
parser.add_argument("--dream_file",
default='dream_out.png',
help="Save dream to named file.")
args = parser.parse_args()
# redirect the dream file to the path of output_file
if args.output_file is None:
output_dir = parser.work_dir
elif osp.isdir(args.output_file):
output_dir = args.output_file
else:
output_dir = osp.dirname(args.output_file)
args.dream_file = osp.expanduser(
osp.join(output_dir, osp.basename(args.dream_file)))
import json
import numpy as np
from neon.util.argparser import NeonArgparser
from neon.initializers import Gaussian
from neon.layers import Conv, Deconv, GeneralizedCost
from neon.optimizers import Adadelta
from neon.transforms import Rectlin, Logistic, SumSquared
from neon.models import Model
from neon.callbacks.callbacks import Callbacks
from localizer_loader import LocalizerLoader
from evaluator import Evaluator
parser = NeonArgparser(__doc__)
parser.add_argument('-tw', '--test_data_dir',
default='',
help='directory in which to find test images')
parser.add_argument('-pn', '--point_num', default=None, help='1 or 2')
parser.add_argument('-iw', '--image_width', default=384, help='image width')
args = parser.parse_args()
point_num = int(args.point_num)
imwidth = int(args.image_width)
train = LocalizerLoader(repo_dir=args.data_dir, inner_size=imwidth,
set_name='train', nlabels=4, do_transforms=False,
point_num=point_num)
test = LocalizerLoader(repo_dir=args.test_data_dir, inner_size=imwidth,
set_name='validation', nlabels=4, do_transforms=False,
point_num=point_num)
train.init_batch_provider()
test.init_batch_provider()
from neon.backends import gen_backend # noqa
from neon.data import ArrayIterator # noqa
from neon.initializers import Uniform, GlorotUniform, Array # noqa
from neon.layers import GeneralizedCost, Affine, Dropout, LookupTable, LSTM, RecurrentSum # noqa
from neon.models import Model # noqa
from neon.optimizers import Adagrad # noqa
from neon.transforms import Logistic, Tanh, Softmax, CrossEntropyMulti, Accuracy # noqa
from neon.util.argparser import NeonArgparser, extract_valid_args # noqa
from neon.util.compat import pickle # noqa
from neon.callbacks.callbacks import Callbacks # noqa
from neon.data.text_preprocessing import get_paddedXY, get_google_word2vec_W # noqa
import h5py # noqa
# parse the command line arguments
parser = NeonArgparser(__doc__)
parser.add_argument("-f", "--review_file", default="labeledTrainData.tsv", help="input movie review file")
parser.add_argument(
"--vocab_file", default="labeledTrainData.tsv.vocab", help="output file to save the processed vocabulary"
)
parser.add_argument("--use_w2v", action="store_true", help="use downloaded Google Word2Vec")
parser.add_argument("--w2v", default="GoogleNews-vectors-negative300.bin", help="the pre-built Word2Vec")
args = parser.parse_args()
# hyperparameters
hidden_size = 128
embedding_dim = 128
vocab_size = 20000
sentence_length = 128
batch_size = 32
gradient_limit = 5
and no partial minibatches, dropout is turned off for reproducibility on gpu
and the learning rate is scaled to handle the reduced dropout percentage.
"""
from neon.util.argparser import NeonArgparser
from neon.initializers import Constant, Gaussian
from neon.layers import Conv, Dropout, Pooling, GeneralizedCost, Affine
from neon.optimizers import GradientDescentMomentum, MultiOptimizer, Schedule
from neon.transforms import Rectlin, Softmax, CrossEntropyMulti, TopKMisclassification
from neon.models import Model
from neon.data import ImageLoader
from neon.callbacks.callbacks import Callbacks
# parse the command line arguments (generates the backend)
parser = NeonArgparser(__doc__)
parser.add_argument('--direct', action='store_true',
help='do not initialize layers, deserialize directly')
args = parser.parse_args()
# setup data provider
img_set_options = dict(repo_dir=args.data_dir,
inner_size=224,
subset_pct=0.09990891117239205)
train = ImageLoader(set_name='train', scale_range=(256, 256), shuffle=False,
do_transforms=False, **img_set_options)
test = ImageLoader(set_name='validation', scale_range=(256, 384), shuffle=False,
do_transforms=False, **img_set_options)
layers = [Conv((11, 11, 64), init=Gaussian(scale=0.01), bias=Constant(0),
activation=Rectlin(), padding=3, strides=4),
Pooling(3, strides=2),
Conv((5, 5, 192), init=Gaussian(scale=0.01), bias=Constant(1),
for idx in range(data.shape[0]):
im = np.pad(data[idx].reshape((3, 32, 32)), self.pad_width, mode='mean')
im = np.uint8(np.transpose(im, axes=[1, 2, 0]).copy())
im = Image.fromarray(im)
path = os.path.join(img_dir, str(labels[idx][0]), str(idx) + '.png')
im.save(path, format='PNG')
if setn == 'train':
self.pixel_mean = list(data.mean(axis=0).reshape(3, -1).mean(axis=1))
self.pixel_mean.reverse() # We will see this in BGR order b/c of opencv
if __name__ == "__main__":
from neon.util.argparser import NeonArgparser
parser = NeonArgparser(__doc__)
parser.add_argument('--set_type', help='(i1k|cifar10|directory|csv)', required=True,
choices=['i1k', 'cifar10', 'directory', 'csv'])
parser.add_argument('--image_dir', help='Directory to find images', default=None)
parser.add_argument('--target_size', type=int, default=0,
help='Size in pixels to scale shortest side DOWN to (0 means no scaling)')
parser.add_argument('--macro_size', type=int, default=5000, help='Images per processed batch')
parser.add_argument('--file_pattern', default='*.jpg', help='Image extension to include in'
'directory crawl')
args = parser.parse_args()
logger = logging.getLogger(__name__)
if args.set_type == 'i1k':
args.target_size = 256 # (maybe 512 for Simonyan's methodology?)
bw = BatchWriterI1K(out_dir=args.data_dir, image_dir=args.image_dir,
target_size=args.target_size, macro_size=args.macro_size,
file_pattern="*.JPEG")
predictions:
the model's predictions
"""
results_list = predictions.tolist()
with open(output, 'w', encoding='utf-8') as out_file:
writer = csv.writer(out_file, delimiter=',', quotechar='"')
for result in results_list:
writer.writerow([result])
print("Results of inference saved in {0}".format(output))
if __name__ == "__main__":
# parse the command line arguments
parser = NeonArgparser()
parser.set_defaults(epochs=200)
parser.add_argument('--data', help='prepared data CSV file path',
type=validate_existing_filepath)
parser.add_argument('--model', help='path to the trained model file',
type=validate_existing_filepath)
parser.add_argument('--print_stats', action='store_true', default=False,
help='print evaluation stats for the model predictions - if '
'your data has tagging')
parser.add_argument('--output', help='path to location for inference output file',
type=validate_parent_exists)
args = parser.parse_args()
data_path = absolute_path(args.data)
model_path = absolute_path(args.model)
print_stats = args.print_stats
output_path = absolute_path(args.output)
# generate backend
be = gen_backend(batch_size=10)
data_set = NpSemanticSegData(data_path, train_to_test_ratio=1)
Striving for Simplicity: the All Convolutional Net `[Springenberg2015]`_
.. _[Springenberg2015]: http://arxiv.org/pdf/1412.6806.pdf
"""
from neon.initializers import Gaussian
from neon.optimizers import GradientDescentMomentum, Schedule
from neon.layers import Conv, Dropout, Activation, Pooling, GeneralizedCost
from neon.transforms import Rectlin, Softmax, CrossEntropyMulti, Misclassification
from neon.models import Model
from neon.data import ArrayIterator, load_cifar10
from neon.callbacks.callbacks import Callbacks
from neon.util.argparser import NeonArgparser
# parse the command line arguments
parser = NeonArgparser(__doc__)
parser.add_argument("--learning_rate", default=0.05, help="initial learning rate")
parser.add_argument("--weight_decay", default=0.001, help="weight decay")
parser.add_argument("--deconv", action="store_true", help="save visualization data from deconvolution")
args = parser.parse_args()
# hyperparameters
num_epochs = args.epochs
(X_train, y_train), (X_test, y_test), nclass = load_cifar10(
path=args.data_dir, normalize=False, contrast_normalize=True, whiten=True
)
# really 10 classes, pad to nearest power of 2 to match conv output
train_set = ArrayIterator(X_train, y_train, nclass=16, lshape=(3, 32, 32))
valid_set = ArrayIterator(X_test, y_test, nclass=16, lshape=(3, 32, 32))
standard_library.install_aliases() # triggers E402, hence noqa below
from builtins import input # noqa
import numpy as np # noqa
from neon.backends import gen_backend # noqa
from neon.initializers import Uniform, GlorotUniform # noqa
from neon.layers import LSTM, Affine, Dropout, LookupTable, RecurrentSum # noqa
from neon.models import Model # noqa
from neon.transforms import Logistic, Tanh, Softmax # noqa
from neon.util.argparser import NeonArgparser, extract_valid_args # noqa
from neon.util.compat import pickle # noqa
from neon.data.text_preprocessing import clean_string # noqa
# parse the command line arguments
parser = NeonArgparser(__doc__)
parser.add_argument('--model_weights',
required=True,
help='pickle file of trained weights')
parser.add_argument('--vocab_file', required=True, help='vocabulary file')
args = parser.parse_args()
# hyperparameters from the reference
batch_size = 1
clip_gradients = True
gradient_limit = 5
vocab_size = 20000
sentence_length = 128
embedding_dim = 128
hidden_size = 128
reset_cells = True
num_epochs = args.epochs
# for example, your data_dir is in an NFS mounted location)
from neon.util.argparser import NeonArgparser
from neon.initializers import Kaiming
from neon.layers import Conv, Pooling, GeneralizedCost, Activation, Affine
from neon.layers import MergeSum, SkipNode, BatchNorm
from neon.optimizers import GradientDescentMomentum, Schedule
from neon.transforms import Rectlin, Softmax, CrossEntropyMulti, Misclassification
from neon.models import Model
from neon.data import ImageLoader
from neon.callbacks.callbacks import Callbacks, BatchNormTuneCallback
# parse the command line arguments (generates the backend)
parser = NeonArgparser(__doc__)
parser.add_argument('--depth',
type=int,
default=9,
help='depth of each stage (network depth will be 9n+2)')
parser.add_argument('--subset_pct',
type=float,
default=100,
help='subset of training dataset to use (percentage)')
args = parser.parse_args()
# setup data provider
imgset_options = dict(inner_size=32,
scale_range=40,
aspect_ratio=110,
repo_dir=args.data_dir,
subset_pct=args.subset_pct)
train = ImageLoader(set_name='train',
shuffle=True,
# [SimpleItk](http://www.simpleitk.org/) is an open-source library for reading and processing 3D models. It was particularly designed for medical imaging and was built on top of the Insight and Segmentation and Registration ([ITK](https://itk.org/)) toolkit sponsored by the National Library of Medicine.
#
# What's nice about SimpleITK is that it has pre-built methods to read all of the DICOM slices into a 3D object and perform segmentation and morphological operations on that 3D object. I believe it automatically arranges the slices in the correct order. It also can handle compressed DICOM files.
import SimpleITK as sitk
#######################
# Parse the command line
from neon.util.argparser import NeonArgparser
parser = NeonArgparser(__doc__)
# We can pass the input directory and output file name from the command line
parser.add_argument('-out',
'--outFilename',
default='dicom_out.h5',
help='Name of the output HDF5 file')
parser.set_defaults(data_dir='/Volumes/data/tonyr/dicom/Lung CT/stage1')
parser.set_defaults(save_path='.')
args = parser.parse_args()
data_dir = args.data_dir
outFilename = args.save_path + '/' + args.outFilename
##############################
def verbosePrint(txt):
print(txt)
python examples/babi/demo.py -t 1 --rlayer_type gru --model_weights babi.p
"""
import numpy as np
from util import create_model, babi_handler
from neon.backends import gen_backend
from neon.data import BABI, QA
from neon.data.text import Text
from neon.util.argparser import NeonArgparser, extract_valid_args
# parse the command line arguments
parser = NeonArgparser(__doc__)
parser.add_argument(
'-t',
'--task',
type=int,
default='1',
choices=xrange(1, 21),
help='the task ID to train/test on from bAbI dataset (1-20)')
parser.add_argument('--rlayer_type',
default='gru',
choices=['gru', 'lstm'],
help='type of recurrent layer to use (gru or lstm)')
parser.add_argument('--model_weights', help='pickle file of trained weights')
args = parser.parse_args(gen_be=False)
# setup backend
be = gen_backend(**extract_valid_args(args, gen_backend))
be.bsz = 1
# load the bAbI dataset
-b gpu -i 0 -e 40 --dataset_dir ~/NABirds --model_file alexnet.p -vvvv
"""
from neon.util.argparser import NeonArgparser
from neon.initializers import Constant, Gaussian
from neon.layers import Conv, Dropout, Pooling, GeneralizedCost, Affine
from neon.transforms import Rectlin, Softmax, CrossEntropyMulti, TopKMisclassification
from neon.models import Model
from neon.data import ImageLoader
from neon.callbacks.callbacks import Callbacks
from model_descriptions import create_model
# parse the command line arguments (generates the backend)
parser = NeonArgparser(__doc__)
parser.add_argument('--model_type', help='Name of model', required=True, choices=['alexnet', 'vgg'])
parser.add_argument('--model_tree', help='Whether or not to train tree of classifiers',
default=False, type=bool)
parser.add_argument('--freeze', type=int, help='Layers to freeze starting from end', default=0)
parser.add_argument('--dataset_dir', help='Directory containing images folder and label text files')
args = parser.parse_args()
# setup data provider
train_set_options = dict(repo_dir=args.data_dir,
inner_size=224,
dtype=args.datatype,
subset_pct=100)
test_set_options = dict(repo_dir=args.data_dir,
inner_size=224,
dtype=args.datatype,
subset_pct=20)
from neon.data.dataloaders import load_imdb
from neon.data.dataiterator import ArrayIterator
from neon.data.text_preprocessing import pad_data
from neon.initializers import Uniform, GlorotUniform
from neon.layers import (GeneralizedCost, LSTM, Affine, Dropout, LookupTable,
RecurrentSum, Recurrent, DeepBiLSTM, DeepBiRNN)
from neon.models import Model
from neon.optimizers import Adagrad
from neon.transforms import Logistic, Tanh, Softmax, CrossEntropyMulti, Accuracy
from neon.callbacks.callbacks import Callbacks
from neon.util.argparser import NeonArgparser, extract_valid_args
# parse the command line arguments
parser = NeonArgparser(__doc__)
parser.add_argument('--rlayer_type', default='lstm',
choices=['bilstm', 'lstm', 'birnn', 'bibnrnn', 'rnn'],
help='type of recurrent layer to use (lstm, bilstm, rnn, birnn, bibnrnn)')
args = parser.parse_args(gen_be=False)
# hyperparameters from the reference
args.batch_size = 128
gradient_clip_value = 15
vocab_size = 20000
sentence_length = 128
embedding_dim = 128
hidden_size = 128
reset_cells = True
# setup backend
be = gen_backend(**extract_valid_args(args, gen_backend))
yield self.X_dev, self.y_dev
# replicate neon's mse error metric
def err(y, t):
feature_axis = 1
return (0.5 * np.square(y - t).mean(axis=feature_axis).mean())
if __name__ == '__main__':
# parse the command line arguments
parser = NeonArgparser(__doc__)
parser.add_argument(
'--curvetype',
default='Lissajous1',
choices=['Lissajous1', 'Lissajous2'],
help='type of input curve data to use (Lissajous1 or Lissajous2)')
args = parser.parse_args(gen_be=False)
# network hyperparameters
hidden = 32
args.batch_size = 1
# The following flag will switch between 2 training strategies:
# 1. return_sequence True:
# Inputs are sequences, and target outputs will be sequences.
# The RNN layer's output at EVERY step will be used for errors and optimized.
# The RNN model contains a RNN layer and an Affine layer
# The data iterator will format the data accordingly, and will stride along the
# whole series with no overlap
Train WGAN to generate MNIST images.
"""
import os
from datetime import datetime
from neon.callbacks.callbacks import Callbacks, GANCostCallback
from neon.callbacks.plotting_callbacks import GANPlotCallback
from neon.optimizers import RMSProp
from neon.util.argparser import NeonArgparser
from neon.util.persist import ensure_dirs_exist
from network_gan import create_model
from neon.data.image import MNIST
# parse the command line arguments
parser = NeonArgparser(__doc__, default_overrides={'epochs': 32, 'rng_seed': 0, 'batch_size': 64})
parser.add_argument('-D', '--dmodel', type=str, default='dc',
help='discriminator model type: dc or mlp, default dc')
parser.add_argument('-G', '--gmodel', type=str, default='dc',
help='generator model type: dc or mlp, default dc')
parser.add_argument('--n_dis_ftr', type=int, default=64,
help='base discriminator feature number, default 64')
parser.add_argument('--n_gen_ftr', type=int, default=64,
help='base generator feature number, default 64')
args = parser.parse_args()
random_seed = args.rng_seed if args.rng_seed else 0
# load up the mnist data set, padding images to size 32
dataset = MNIST(path=args.data_dir, sym_range=True, size=32, shuffle=True)
train = dataset.train_iter
# create a GAN
model, cost = create_model(dis_model=args.dmodel, gen_model=args.gmodel,
from glob import glob
import functools
import gzip
from multiprocessing import Pool
import numpy as np
import os
import tarfile
import struct
from PIL import Image as PILImage
from neon.util.compat import range, StringIO
from neon.util.persist import load_obj, save_obj
from neon.data import load_i1kmeta
from neon.util.argparser import NeonArgparser
parser = NeonArgparser(__doc__)
parser.add_argument('--set_type', help='(i1k|directory)', required=True,
choices=['i1k', 'directory'])
parser.add_argument('--image_dir', help='Directory to find images', required=True)
parser.add_argument('--target_size', type=int, default=256,
help='Size in pixels to scale images (Must be 256 for i1k dataset)')
parser.add_argument('--macro_size', type=int, default=5000, help='Images per processed batch')
parser.add_argument('--file_pattern', default='*.jpg', help='Image extension to include in'
'directory crawl')
args = parser.parse_args()
logger = logging.getLogger(__name__)
# NOTE: We have to leave this helper function out of the class to use multiprocess pool.map
def proc_img(target_size, squarecrop, is_string=False, imgfile=None):
imgfile = StringIO(imgfile) if is_string else imgfile
im = PILImage.open(imgfile)
from neon.data import ArrayIterator
from neon.initializers import Uniform, GlorotUniform, Array
from neon.layers import GeneralizedCost, Affine, Dropout, LookupTable, LSTM, RecurrentSum
from neon.models import Model
from neon.optimizers import Adagrad
from neon.transforms import Logistic, Tanh, Softmax, CrossEntropyMulti, Accuracy
from neon.util.argparser import NeonArgparser, extract_valid_args
from neon.callbacks.callbacks import Callbacks
from neon.data.text_preprocessing import get_paddedXY, get_google_word2vec_W
import h5py
import cPickle
# parse the command line arguments
parser = NeonArgparser(__doc__)
parser.add_argument('-f', '--review_file',
default='labeledTrainData.tsv',
help='input movie review file')
parser.add_argument('--vocab_file',
default='labeledTrainData.tsv.vocab',
help='output file to save the processed vocabulary')
parser.add_argument('--use_w2v', action='store_true',
help='use downloaded Google Word2Vec')
parser.add_argument('--w2v',
default='GoogleNews-vectors-negative300.bin',
help='the pre-built Word2Vec')
args = parser.parse_args()
# hyperparameters
hidden_size = 128
embedding_dim = 128
from neon.util.argparser import NeonArgparser
from util import create_frcn_model
do_plots = True
try:
import matplotlib.pyplot as plt
plt.switch_backend('agg')
except ImportError:
neon_logger.display('matplotlib needs to be installed manually to generate plots needed '
'for this example. Skipping plot generation')
do_plots = False
# parse the command line arguments
parser = NeonArgparser(__doc__)
parser.add_argument('--img_prefix', type=str,
help='prefix for the saved image file names. If None, use '
'the model file name')
args = parser.parse_args(gen_be=True)
assert args.model_file is not None, "need a model file to do Fast R-CNN testing"
if args.img_prefix is None:
args.img_prefix = os.path.splitext(os.path.basename(args.model_file))[0]
output_dir = os.path.join(args.data_dir, 'frcn_output')
if not os.path.isdir(output_dir):
os.mkdir(output_dir)
# hyperparameters
args.batch_size = 1
n_mb = 40
img_per_batch = args.batch_size
from neon.layers import Conv, Pooling, MergeBroadcast, BranchNode, Affine, Tree, Dropout
from neon.layers import GeneralizedCost, Multicost
from neon.initializers import Constant, Xavier
from neon.optimizers import GradientDescentMomentum, PolySchedule, MultiOptimizer
from neon.transforms import Rectlin, Softmax, CrossEntropyMulti
from neon.models import Model
from neon.data import ImageLoader
from neon.callbacks.callbacks import Callbacks
from neon.util.argparser import NeonArgparser
from neon.util.persist import load_obj, save_obj
from neon import logger as neon_logger
# parse the command line arguments
parser = NeonArgparser(__doc__)
parser.add_argument(
'--resume',
action='store_true',
help='first run without this and then run with so resume from saved file')
args = parser.parse_args()
if args.backend == 'mgpu':
batch_size = 32 * 8 # for some reason bs 128 does not work with bias layers
else:
batch_size = 32
# subset pct is set to make sure that every epoch has the same mb count
img_set_options = dict(repo_dir=args.data_dir,
inner_size=224,
dtype=np.float32,
subset_pct=0.09990891117239205)
train = ImageLoader(set_name='train',
scale_range=(256, 256),
from ssd_dataloader import build_dataloader
from callbacks import MAP_Callback, ssd_image_callback
from mboxloss import MBoxLoss
from collections import OrderedDict
import json
"""
Trains the SSD model on the provided dataset.
The ingest scripts for datasets are stored in the /datasets directory. We
include ingest scripts for PASCALVOC, KITTI, and SPACENET datasets.
"""
# Parse the command line arguments
arg_defaults = {'batch_size': 0}
parser = NeonArgparser(__doc__, default_overrides=arg_defaults)
parser.add_argument('--height', type=int, help='image height')
parser.add_argument('--width', type=int, help='image width')
parser.add_argument('--subset_pct', type=float, default=100.0,
help='fraction of full training data set to use')
parser.add_argument('--ssd_config', action='append', required=True, help='path to ssd json file')
parser.add_argument('--lr_scale', type=float, default=1.0, help='scale lr by this amount')
parser.add_argument('--image_sample_dir', type=str, help='path to save image samples')
parser.add_argument('--num_images', type=int, help='number of images to save')
parser.add_argument('--lr_step', type=int, action='append', help='epochs to step lr')
args = parser.parse_args(gen_be=False)
if args.ssd_config:
args.ssd_config = {k: v for k, v in [ss.split(':') for ss in args.ssd_config]}
# directory to store VGG weights
cache_dir = get_data_cache_dir(args.data_dir, subdir='ssd_cache')
import os
from neon.util.argparser import NeonArgparser
from neon.optimizers import GradientDescentMomentum, Schedule
from neon.transforms import TopKMisclassification
from neon.callbacks.callbacks import Callbacks, BatchNormTuneCallback
from data import make_msra_train_loader, make_validation_loader, make_tuning_loader
from network_msra import create_network
# parse the command line arguments (generates the backend)
train_config = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'train.cfg')
config_files = [train_config] if os.path.exists(train_config) else []
parser = NeonArgparser(__doc__, default_config_files=config_files)
parser.add_argument('--depth',
type=int,
default=0,
help='network configuration')
parser.add_argument(
'--bottleneck',
action="store_true",
help="use bottleneck modules compared to double 3x3 modules")
parser.add_argument('--subset_pct',
type=float,
default=100,
help='subset of training dataset to use (percentage)')
args = parser.parse_args()
model, cost = create_network(args.depth, args.bottleneck)
rseed = 0 if args.rng_seed is None else args.rng_seed
# setup data provider
from glob import glob
import functools
import gzip
from multiprocessing import Pool
import numpy as np
import os
import tarfile
import struct
from PIL import Image as PILImage
from neon.util.compat import range, StringIO
from neon.util.persist import load_obj, save_obj
from neon.data import load_i1kmeta
from neon.util.argparser import NeonArgparser
parser = NeonArgparser(__doc__)
parser.add_argument('--set_type', help='(i1k|directory)', required=True,
choices=['i1k', 'directory'])
parser.add_argument('--image_dir', help='Directory to find images', required=True)
parser.add_argument('--target_size', type=int, default=256,
help='Size in pixels to scale images (Must be 256 for i1k dataset)')
parser.add_argument('--macro_size', type=int, default=5000, help='Images per processed batch')
parser.add_argument('--file_pattern', default='*.jpg', help='Image extension to include in'
'directory crawl')
args = parser.parse_args()
logger = logging.getLogger(__name__)
# NOTE: We have to leave this helper function out of the class to use multiprocess pool.map
def proc_img(target_size, squarecrop, is_string=False, imgfile=None):
imgfile = StringIO(imgfile) if is_string else imgfile
im = PILImage.open(imgfile)
from neon.backends import gen_backend
from neon.data.dataloader_transformers import TypeCast
import numpy as np
from neon.initializers import Kaiming, IdentityInit, Constant
from neon.layers import Conv, Pooling, GeneralizedCost, Affine, Activation, Dropout
from neon.layers import MergeSum, SkipNode, BatchNorm
from neon.data.datasets import Dataset
from neon.util.persist import load_obj
import os
# parse the command line arguments
parser = NeonArgparser(__doc__)
parser.add_argument('--depth',
type=int,
default=18,
help='choices 18, 34, 50, 101, 152')
args = parser.parse_args()
# hyperparameters
num_epochs = args.epochs
# Next line gets rid of the deterministic warning
args.deterministic = None
if (args.rng_seed is None):
args.rng_seed = 16
print('Batch size = {}'.format(args.batch_size))
from neon.transforms import TopKMisclassification
from neon.callbacks.callbacks import Callbacks
from data import make_alexnet_train_loader, make_validation_loader
from network_allcnn import create_network
# parse the command line arguments (generates the backend)
train_config = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'train.cfg')
config_files = [train_config] if os.path.exists(train_config) else []
parser = NeonArgparser(__doc__,
default_config_files=config_files,
default_overrides=dict(batch_size=64))
parser.add_argument('--deconv',
action='store_true',
help='save visualization data from deconvolution')
parser.add_argument('--subset_pct',
type=float,
default=100,
help='subset of training dataset to use (percentage)')
args = parser.parse_args()
model, cost = create_network()
rseed = 0 if args.rng_seed is None else args.rng_seed
# setup data provider
assert 'train' in args.manifest, "Missing train manifest"
assert 'val' in args.manifest, "Missing validation manifest"
train = make_alexnet_train_loader(args.manifest['train'], args.manifest_root,
model.be, args.subset_pct, rseed)
from neon import logger as neon_logger
from neon.util.argparser import NeonArgparser
from neon.initializers import Kaiming
from neon.layers import Conv, Pooling, GeneralizedCost, Activation
from neon.layers import MergeSum, SkipNode
from neon.optimizers import GradientDescentMomentum, Schedule
from neon.transforms import Rectlin, Softmax, CrossEntropyMulti, TopKMisclassification
from neon.models import Model
from neon.data import ImageLoader
from neon.callbacks.callbacks import Callbacks, BatchNormTuneCallback
import itertools as itt
import sys
# parse the command line arguments (generates the backend)
parser = NeonArgparser(__doc__)
parser.add_argument('--depth', type=int, default=0,
help='network configuration')
parser.add_argument('--bottleneck', action="store_true",
help="use bottleneck modules compared to double 3x3 modules")
parser.add_argument('--subset_pct', type=float, default=100,
help='subset of training dataset to use (percentage)')
args = parser.parse_args()
if args.depth in (0, 18):
stages = (2, 2, 2, 2)
elif args.depth in (1, 34, 50):
stages = (3, 4, 6, 3)
elif args.depth in (2, 68, 101):
stages = (3, 4, 23, 3)
elif args.depth in (3, 102, 152):
stages = (3, 8, 36, 3)
elif args.depth in (4, 98, 138):
from neon import logger as neon_logger
from neon.models import Model
from aeon import DataLoader
from neon.util.argparser import NeonArgparser, extract_valid_args
from neon.transforms import Misclassification, PrecisionRecall
from neon.backends import gen_backend
from neon.data.dataloader_transformers import TypeCast, OneHot
import numpy as np
import pandas as pd
from neon.data import HDF5IteratorOneHot, HDF5Iterator
import h5py
# parse the command line arguments
parser = NeonArgparser(__doc__)
parser.add_argument('--subset', type=int, default=9)
args = parser.parse_args()
subset = args.subset
testFileName = '/mnt/data/medical/luna16/luna16_roi_subset{}_ALL.h5'.format(subset)
print('Using test file: {}'.format(testFileName))
# Next line gets rid of the deterministic warning
args.deterministic = None
if (args.rng_seed is None):
args.rng_seed = 16
print('Batch size = {}'.format(args.batch_size))
from neon import logger as neon_logger
from neon.backends import gen_backend
from neon.data import PASCALVOCTrain
from neon.transforms import CrossEntropyMulti, SmoothL1Loss, ObjectDetection
from neon.util.argparser import NeonArgparser, extract_valid_args
from neon.optimizers import GradientDescentMomentum, MultiOptimizer
from neon.callbacks.callbacks import Callbacks
from neon.layers import Multicost, GeneralizedCostMask
from neon.util.persist import save_obj
from util import load_vgg_weights, create_frcn_model, scale_bbreg_weights
# main script
# parse the command line arguments
parser = NeonArgparser(__doc__, default_overrides=dict(batch_size=4))
parser.add_argument('--subset_pct', type=float, default=100,
help='subset of training dataset to use (percentage)')
args = parser.parse_args(gen_be=False)
# Override save path if None
if args.save_path is None:
args.save_path = 'frcn_vgg.pkl'
if args.callback_args['save_path'] is None:
args.callback_args['save_path'] = args.save_path
if args.callback_args['serialize'] is None:
args.callback_args['serialize'] = min(args.epochs, 10)
# hyperparameters
args.batch_size = 4
from __future__ import division
from neon.backends import gen_backend
from neon.util.argparser import NeonArgparser, extract_valid_args
from neon.optimizers import GradientDescentMomentum, MultiOptimizer, StepSchedule
from neon.callbacks.callbacks import Callbacks, TrainMulticostCallback
from neon.util.persist import save_obj
from objectlocalization import PASCAL
from neon.transforms import CrossEntropyMulti, SmoothL1Loss
from neon.layers import Multicost, GeneralizedCostMask
import util
# parse the command line arguments
parser = NeonArgparser(__doc__, default_overrides={'batch_size': 1})
parser.add_argument('--subset_pct', type=float, default=100,
help='subset of training dataset to use (percentage)')
args = parser.parse_args(gen_be=False)
# hyperparameters
assert args.batch_size is 1, "Faster-RCNN only supports batch size 1"
n_mb = None
rpn_rois_per_img = 256 # number of rois to sample to train rpn
frcn_rois_per_img = 128 # number of rois to sample to train frcn
# setup backend
be = gen_backend(**extract_valid_args(args, gen_backend))
be.enable_winograd = 4 # default to winograd 4 for fast autotune
year = '2007'
from neon.optimizers import GradientDescentMomentum, MultiOptimizer, StepSchedule
from neon.callbacks.callbacks import Callbacks
from neon.util.persist import save_obj, get_data_cache_dir
from objectlocalization import PASCALVOC
from neon.transforms import CrossEntropyMulti, SmoothL1Loss
from neon.layers import Multicost, GeneralizedCostMask
import util
import faster_rcnn
import os
train_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'pascalvoc.cfg')
config_files = [train_config] if os.path.exists(train_config) else []
# parse the command line arguments
parser = NeonArgparser(__doc__, default_config_files=config_files)
parser.add_argument('--width', type=int, default=1000, help='Width of input image')
parser.add_argument('--height', type=int, default=1000, help='Height of input image')
parser.add_argument('--subset_pct', type=float, default=100,
help='subset of training dataset to use (percentage)')
args = parser.parse_args(gen_be=False)
# hyperparameters
assert args.batch_size is 1, "Faster-RCNN only supports batch size 1"
assert 'train' in args.manifest
rpn_rois_per_img = 256 # number of rois to sample to train rpn
frcn_rois_per_img = 128 # number of rois to sample to train frcn
# setup backend
be = gen_backend(**extract_valid_args(args, gen_backend))
be.enable_winograd = 4 # default to winograd 4 for fast autotune
self.batch_index += 1
yield self.X_dev, self.y_dev
# replicate neon's mse error metric
def err(y, t):
feature_axis = 1
return (0.5 * np.square(y - t).mean(axis=feature_axis).mean())
if __name__ == '__main__':
# parse the command line arguments
parser = NeonArgparser(__doc__)
parser.add_argument('--curvetype', default='Lissajous1', choices=['Lissajous1', 'Lissajous2'],
help='type of input curve data to use (Lissajous1 or Lissajous2)')
args = parser.parse_args(gen_be=False)
# network hyperparameters
hidden = 32
args.batch_size = 1
# The following flag will switch between 2 training strategies:
# 1. return_sequence True:
# Inputs are sequences, and target outputs will be sequences.
# The RNN layer's output at EVERY step will be used for errors and optimized.
# The RNN model contains a RNN layer and an Affine layer
# The data iterator will format the data accordingly, and will stride along the
# whole series with no overlap
# 2. return_sequence False:
# Inputs are sequences, and target output will be a single step.
"""
import os
from neon.util.argparser import NeonArgparser
from neon.layers import Conv, Pooling, MergeBroadcast, BranchNode, Affine, Tree, Dropout
from neon.layers import GeneralizedCost, Multicost
from neon.initializers import Constant, Xavier
from neon.backends import gen_backend
from neon.optimizers import GradientDescentMomentum, MultiOptimizer
from neon.transforms import Rectlin, Softmax, CrossEntropyMulti, TopKMisclassification
from neon.models import Model
from neon.data import ImageLoader
parser = NeonArgparser(__doc__)
parser.add_argument('--subset_pct', type=float, default=100,
help='subset of training dataset to use (percentage)')
parser.add_argument('--test_only', action='store_true',
help='skip fitting - evaluate metrics on trained model weights')
args = parser.parse_args()
# setup data provider
img_set_options = dict(repo_dir=args.data_dir, inner_size=224,
dtype=args.datatype, subset_pct=args.subset_pct)
test = ImageLoader(set_name='validation', scale_range=(256, 256),
do_transforms=False, **img_set_options)
init1 = Xavier(local=False)
initx = Xavier(local=True)
bias = Constant(val=0.20)
relu = Rectlin()
from neon.data.dataiterator import ArrayIterator
from neon.data.text_preprocessing import pad_data
from neon.initializers import Uniform, GlorotUniform
from neon.layers import (GeneralizedCost, LSTM, Affine, Dropout, LookupTable,
RecurrentSum, Recurrent, DeepBiLSTM, DeepBiRNN)
from neon.models import Model
from neon.optimizers import Adagrad
from neon.transforms import Logistic, Tanh, Softmax, CrossEntropyMulti, Accuracy
from neon.callbacks.callbacks import Callbacks
from neon.util.argparser import NeonArgparser, extract_valid_args
# parse the command line arguments
parser = NeonArgparser(__doc__)
parser.add_argument(
'--rlayer_type',
default='lstm',
choices=['bilstm', 'lstm', 'birnn', 'bibnrnn', 'rnn'],
help='type of recurrent layer to use (lstm, bilstm, rnn, birnn, bibnrnn)')
args = parser.parse_args(gen_be=False)
# hyperparameters from the reference
args.batch_size = 128
gradient_clip_value = 15
vocab_size = 20000
sentence_length = 128
embedding_dim = 128
hidden_size = 128
reset_cells = True
# setup backend
from neon.util.persist import load_obj
from neon.util.argparser import NeonArgparser, extract_valid_args
from neon.initializers import Uniform, Orthonormal, Constant
from neon.layers import GeneralizedCostMask, Multicost, GRU, SkipThought
from neon.models import Model
from neon.transforms import CrossEntropyMulti, Logistic, Tanh
from neon.callbacks.callbacks import Callbacks, MetricCallback
from neon.optimizers import Adam
from neon import logger as neon_logger
from data_loader import load_data
from data_iterator import SentenceHomogenous
# parse the command line arguments
parser = NeonArgparser(__doc__)
parser.add_argument('--output_dir', default='output/',
help='choose the directory to save the files')
parser.add_argument('--max_vocab_size', default=20000,
help='number of (most frequent) words to use from vocabulary')
parser.add_argument('--max_len_w', default=30,
help='maximum sentence length for training')
parser.add_argument('--subset_pct', type=float, default=100,
help='subset of training dataset to use (percentage)')
args = parser.parse_args(gen_be=False)
# hyperparameters from the reference
args.batch_size = 64
embed_dim = 620
valid_split = None
# setup backend
be = gen_backend(**extract_valid_args(args, gen_backend))
def caption_video(infile, caption, outfile):
cmd = '''ffmpeg -i {0} -an \
-vf drawtext="textfile={1}: fontcolor=white: fontsize=16: box=1: [email protected]" \
-y {2}'''
proc = subprocess.Popen(cmd.format(infile, caption, outfile), shell=True)
proc.communicate()
# parse the command line arguments
demo_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'test.cfg')
config_files = [demo_config] if os.path.exists(demo_config) else []
parser = NeonArgparser(__doc__, default_config_files=config_files)
parser.add_argument('--input_video', help='video file')
parser.add_argument('--output_video', help='Video file with overlayed inference hypotheses')
args = parser.parse_args()
assert args.model_file is not None, "need a model file for testing"
model = Model(args.model_file)
assert 'categories' in args.manifest, "Missing categories file"
category_map = {t[0]: t[1] for t in np.genfromtxt(args.manifest['categories'],
dtype=None, delimiter=',')}
# Make a temporary directory and clean up afterwards
outdir = mkdtemp()
atexit.register(shutil.rmtree, outdir)
caption_file = os.path.join(outdir, 'caption.txt')
import spacy
from neon.backends import gen_backend
from neon.models import Model
from neon.util.argparser import NeonArgparser, extract_valid_args
from nlp_architect.contrib.neon.text_iterators import TaggedTextSequence, MultiSequenceDataIterator
from nlp_architect.utils.generic import get_paddedXY_sequence
from nlp_architect.utils.embedding import load_word_embeddings
from nlp_architect.utils.io import validate_existing_filepath
from utils import extract_nps
if __name__ == '__main__':
parser = NeonArgparser()
parser.add_argument('--model',
type=validate_existing_filepath,
required=True,
help='Path to model file')
parser.add_argument('--settings',
type=validate_existing_filepath,
required=True,
help='Path to model settings file')
parser.add_argument(
'--input',
type=validate_existing_filepath,
required=True,
help='Input texts file path (samples to pass for inference)')
parser.add_argument('--embedding_model',
type=validate_existing_filepath,
help='Pre-trained word embedding model file path')
parser.add_argument('--print_only_nps',
default=True,
from future import standard_library
standard_library.install_aliases() # triggers E402, hence noqa below
from builtins import input # noqa
import numpy as np # noqa
from neon.backends import gen_backend # noqa
from neon.initializers import Uniform, GlorotUniform # noqa
from neon.layers import LSTM, Affine, Dropout, LookupTable, RecurrentSum # noqa
from neon.models import Model # noqa
from neon.transforms import Logistic, Tanh, Softmax # noqa
from neon.util.argparser import NeonArgparser, extract_valid_args # noqa
from neon.util.compat import pickle # noqa
from neon.data.text_preprocessing import clean_string # noqa
# parse the command line arguments
parser = NeonArgparser(__doc__)
parser.add_argument(
'--model_weights', required=True, help='pickle file of trained weights')
parser.add_argument('--vocab_file', required=True, help='vocabulary file')
args = parser.parse_args()
# hyperparameters from the reference
batch_size = 1
clip_gradients = True
gradient_limit = 5
vocab_size = 20000
sentence_length = 128
embedding_dim = 128
hidden_size = 128
reset_cells = True
num_epochs = args.epochs
# setup backend
"""
logging information and command parsing
Can specify these argument on command line or here
"""
parser = NeonArgparser(__doc__)
argss=[ "--file_name","--train_num_p","--valid_num_p","--test_num_p",
"--train_num_n","--valid_num_n","--test_num_n","--nclass",
#data dependent arguments
"--data_dict","--norm_type",
#output dependent arguments
"--out_dir"]
for a in argss:
parser.add_argument(a)
parser.set_defaults(
#constant arguments
rng_seed=2,
backend= "cpu",
progress_bar=True,
verbose=4,
evaluation_freq=2,
#data
epochs= 10,
batch_size=128,
data_dir="/global/project/projectdirs/nervana/yunjie/climate_neon1.0run/conv/DATA/",
file_name="hurricanes.h5",
"""
from neon.backends import gen_backend
from neon.data import Text, load_text
from neon.initializers import Uniform
from neon.layers import GeneralizedCost, LSTM, Affine, GRU, LookupTable
from neon.models import Model
from neon.optimizers import GradientDescentMomentum, Schedule
from neon.transforms import Logistic, Tanh, Softmax, CrossEntropyMulti
from neon.callbacks.callbacks import Callbacks
from neon.util.argparser import NeonArgparser, extract_valid_args
# parse the command line arguments
parser = NeonArgparser(__doc__)
parser.add_argument('--rlayer_type', default='lstm', choices=['gru', 'lstm'],
help='type of recurrent layer to use (gru or lstm)')
args = parser.parse_args(gen_be=False)
# hyperparameters from the reference
args.batch_size = 20
time_steps = 20
hidden_size = 200
gradient_clip_norm = 5
# setup backend
be = gen_backend(**extract_valid_args(args, gen_backend))
# download penn treebank
train_path = load_text('ptb-train', path=args.data_dir)
valid_path = load_text('ptb-valid', path=args.data_dir)
import numpy as np
from neon.util.argparser import NeonArgparser
from neon.initializers import Gaussian, GlorotUniform
from neon.layers import Conv, Pooling, GeneralizedCost, Affine
from neon.layers import DeepBiRNN, RecurrentMean
from neon.optimizers import Adagrad
from neon.transforms import Tanh, Rectlin, Softmax, CrossEntropyBinary
from neon.models import Model
from neon.data import DataLoader, AudioParams
from neon.callbacks.callbacks import Callbacks
from sklearn import metrics
from indexer import Indexer
parser = NeonArgparser(__doc__)
parser.add_argument('-elec', '--electrode', default=0, help='electrode index')
parser.add_argument('-out', '--out_dir', default='preds', help='directory to write output files')
parser.add_argument('-test', '--test_mode', action="store_true", help="testing mode")
args = parser.parse_args()
pattern = '*.' + str(args.electrode) + '.wav'
data_dir = args.data_dir
out_dir = args.out_dir
if not os.path.exists(out_dir):
os.makedirs(out_dir)
if data_dir[-1] != '/':
data_dir += '/'
subj = int(data_dir[-2])
assert subj in [1, 2, 3]
indexer = Indexer()
tain_idx, test_idx = indexer.run(data_dir, pattern, testing=args.test_mode)
self.subj = subj
self.data_dir = data_dir
self.eval_set = eval_set
def on_epoch_end(self, callback_data, model, epoch):
preds = model.get_outputs(self.eval_set)[:, 1]
idx_file = os.path.join(
self.data_dir,
'eval-' + str(self.subj) + '-' + str(0) + '-index.csv')
labels = np.loadtxt(idx_file, delimiter=',', skiprows=1, usecols=[1])
logger.display('Eval AUC for subject %d epoch %d: %.4f\n' %
(self.subj, epoch, score(labels, preds)))
parser = NeonArgparser(__doc__)
parser.add_argument('-elec', '--electrode', default=0, help='electrode index')
parser.add_argument('-out',
'--out_dir',
default='preds',
help='directory to write output files')
parser.add_argument('-validate',
'--validate_mode',
action="store_true",
help="validate on training data")
args = parser.parse_args()
data_dir = os.path.normpath(args.data_dir)
out_dir = args.out_dir
if not os.path.exists(out_dir):
os.makedirs(out_dir)
subj = int(data_dir[-1])
--model_weights <trained_pickle_file>
"""
from builtins import map, zip
import cv2
import numpy as np
import os
from neon.backends import gen_backend
from neon.data import DataLoader, VideoParams, ImageParams
from neon.util.argparser import NeonArgparser, extract_valid_args
from network import create_network
# parse the command line arguments
parser = NeonArgparser(__doc__)
parser.add_argument('--class_ind_file',
help='Path of two column file mapping integer'
'class labels to their canonical names.')
parser.add_argument('--model_weights', help='Pickle file of trained model weights.')
args = parser.parse_args(gen_be=False)
# setup backend
be = gen_backend(**extract_valid_args(args, gen_backend))
be.bsz = 32
# setup data provider
shape = dict(channel_count=3, height=112, width=112, scale_min=128, scale_max=128)
testParams = VideoParams(frame_params=ImageParams(center=True, flip=False, **shape),
frames_per_clip=16)
common = dict(target_size=1, nclasses=101, datum_dtype=np.uint8)
from neon.data import ArrayIterator
from neon.initializers import Uniform, GlorotUniform
from neon.layers import GeneralizedCost, Affine, Dropout, LookupTable, LSTM, RecurrentSum
from neon.models import Model
from neon.optimizers import Adagrad
from neon.transforms import Logistic, Tanh, Softmax, CrossEntropyMulti, Accuracy
from neon.util.argparser import NeonArgparser, extract_valid_args
from neon.callbacks.callbacks import Callbacks
from neon.data.text_preprocessing import get_paddedXY, get_google_word2vec_W
import h5py
import cPickle
# parse the command line arguments
parser = NeonArgparser(__doc__)
parser.add_argument('-f', '--review_file',
default='labeledTrainData.tsv',
help='input movie review file')
parser.add_argument('--vocab_file',
default='labeledTrainData.tsv.vocab',
help='output file to save the processed vocabulary')
parser.add_argument('--use_w2v', action='store_true',
help='use downloaded Google Word2Vec')
parser.add_argument('--w2v',
default='GoogleNews-vectors-negative300.bin',
help='the pre-built Word2Vec')
args = parser.parse_args()
# hyperparameters
hidden_size = 128
embedding_dim = 128
def caption_video(infile, caption, outfile):
cmd = '''ffmpeg -i {0} -an \
-vf drawtext="textfile={1}: fontcolor=white: fontsize=16: box=1: [email protected]" \
-y {2}'''
proc = subprocess.Popen(cmd.format(infile, caption, outfile), shell=True)
proc.communicate()
# parse the command line arguments
demo_config = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'test.cfg')
config_files = [demo_config] if os.path.exists(demo_config) else []
parser = NeonArgparser(__doc__, default_config_files=config_files)
parser.add_argument('--input_video', help='video file')
parser.add_argument('--output_video',
help='Video file with overlayed inference hypotheses')
args = parser.parse_args()
assert args.model_file is not None, "need a model file for testing"
model = Model(args.model_file)
assert 'categories' in args.manifest, "Missing categories file"
category_map = {
t[0]: t[1]
for t in np.genfromtxt(
args.manifest['categories'], dtype=None, delimiter=',')
}
# Make a temporary directory and clean up afterwards
for idx in range(data.shape[0]):
im = np.pad(data[idx].reshape((3, 32, 32)), self.pad_width, mode='mean')
im = np.uint8(np.transpose(im, axes=[1, 2, 0]).copy())
im = Image.fromarray(im)
path = os.path.join(img_dir, str(labels[idx][0]), str(idx) + '.png')
im.save(path, format='PNG')
if setn == 'train':
self.pixel_mean = list(data.mean(axis=0).reshape(3, -1).mean(axis=1))
self.pixel_mean.reverse() # We will see this in BGR order b/c of opencv
if __name__ == "__main__":
from neon.util.argparser import NeonArgparser
parser = NeonArgparser(__doc__)
parser.add_argument('--set_type', help='(i1k|cifar10|directory|csv)', required=True,
choices=['i1k', 'cifar10', 'directory', 'csv'])
parser.add_argument('--image_dir', help='Directory to find images', default=None)
parser.add_argument('--target_size', type=int, default=0,
help='Size in pixels to scale shortest side DOWN to (0 means no scaling)')
parser.add_argument('--macro_size', type=int, default=5000, help='Images per processed batch')
parser.add_argument('--file_pattern', default='*.jpg', help='Image extension to include in'
'directory crawl')
args = parser.parse_args()
logger = logging.getLogger(__name__)
if args.set_type == 'i1k':
args.target_size = 256 # (maybe 512 for Simonyan's methodology?)
bw = BatchWriterI1K(out_dir=args.data_dir, image_dir=args.image_dir,
target_size=args.target_size, macro_size=args.macro_size,
file_pattern="*.JPEG")
import sys
from neon.util.argparser import NeonArgparser
from neon.backends import gen_backend
from neon.initializers import Constant, Gaussian
from neon.layers import Conv, Dropout, Pooling, GeneralizedCost, Affine
from neon.optimizers import GradientDescentMomentum, MultiOptimizer, Schedule
from neon.transforms import Rectlin, Softmax, CrossEntropyMulti, TopKMisclassification
from neon.models import Model
from neon.data import ImgMaster
from neon.callbacks.callbacks import Callbacks, Callback
# For running complete alexnet
# alexnet.py -e 90 -val 1 -s <save-path> -w <path-to-saved-batches>
# parse the command line arguments
parser = NeonArgparser(__doc__)
parser.add_argument('--model_file', help='load model from pkl file')
args = parser.parse_args()
# hyperparameters
batch_size = 128
# setup backend
be = gen_backend(backend=args.backend, rng_seed=args.rng_seed, device_id=args.device_id,
batch_size=batch_size, default_dtype=args.datatype)
try:
train = ImgMaster(repo_dir=args.data_dir, inner_size=224, set_name='train')
test = ImgMaster(repo_dir=args.data_dir, inner_size=224, set_name='validation',
do_transforms=False)
except (OSError, IOError, ValueError) as err:
print err
from neon.callbacks.callbacks import Callbacks, GANCostCallback
from neon.callbacks.plotting_callbacks import GANPlotCallback
from neon.initializers import Gaussian
from neon.layers import GeneralizedGANCost, Sequential, Conv, Deconv, Dropout, Pooling
from neon.layers.layer import Linear, Reshape
from neon.layers.container import GenerativeAdversarial
from neon.models.model import GAN
from neon.transforms import Rectlin, Logistic, GANCost
from neon.util.argparser import NeonArgparser
from neon.util.persist import ensure_dirs_exist
from neon.layers.layer import Dropout
# parse the command line arguments
parser = NeonArgparser(__doc__)
parser.add_argument('--kbatch',
type=int,
default=1,
help='number of data batches per noise batch in training')
args = parser.parse_args()
# load up the data set
# setup weight initialization function
init = Gaussian()
# discriminiator using convolution layers
lrelu = Rectlin(slope=0.1) # leaky relu for discriminator
# sigmoid = Logistic() # sigmoid activation function
conv1 = dict(
init=init, batch_norm=False,
activation=lrelu) # what's about BatchNorm Layer and batch_norm parameter?
conv2 = dict(init=init, batch_norm=True, activation=lrelu, padding=2)
from neon.backends import gen_backend
from neon.data.text import Text
from neon.data.dataloaders import load_ptb_train, load_ptb_test
from neon.initializers import Uniform
from neon.layers import GeneralizedCost, LSTM, Affine, GRU, LookupTable
from neon.models import Model
from neon.optimizers import GradientDescentMomentum, Schedule
from neon.transforms import Logistic, Tanh, Softmax, CrossEntropyMulti
from neon.callbacks.callbacks import Callbacks
from neon.util.argparser import NeonArgparser, extract_valid_args
# parse the command line arguments
parser = NeonArgparser(__doc__)
parser.add_argument('--rlayer_type',
default='lstm',
choices=['gru', 'lstm'],
help='type of recurrent layer to use (gru or lstm)')
args = parser.parse_args(gen_be=False)
# hyperparameters from the reference
args.batch_size = 20
time_steps = 20
hidden_size = 200
gradient_clip_norm = 5
# setup backend
be = gen_backend(**extract_valid_args(args, gen_backend))
# download penn treebank
train_path = load_ptb_train(path=args.data_dir)
valid_path = load_ptb_test(path=args.data_dir)
.. _[Springenberg2014]: http://arxiv.org/pdf/1412.6806.pdf
"""
from neon.util.argparser import NeonArgparser
from neon.backends import gen_backend
from neon.initializers import GlorotUniform
from neon.optimizers import GradientDescentMomentum, Schedule
from neon.layers import Conv, Dropout, Activation, Pooling, GeneralizedCost, DataTransform
from neon.transforms import Rectlin, Softmax, CrossEntropyMulti, Normalizer
from neon.models import Model
from neon.callbacks.callbacks import Callbacks
from neon.data import ImgMaster, ImageLoader
# parse the command line arguments
parser = NeonArgparser(__doc__)
parser.add_argument('--deconv', action='store_true',
help='save visualization data from deconvolution')
parser.add_argument('--loader_version', default='old', choices=['old', 'new'],
help='whether to use old dataloader (ImgMaster) or new (ImageLoader)')
args = parser.parse_args()
# hyperparameters
batch_size = 64
# setup backend
be = gen_backend(backend=args.backend,
batch_size=batch_size,
rng_seed=args.rng_seed,
device_id=args.device_id,
datatype=args.datatype)
--model_file <saved weights file>
"""
from neon.util.argparser import NeonArgparser
from neon.initializers import Constant, Gaussian
from neon.layers import Conv, Dropout, Pooling, GeneralizedCost, Affine, LRN
from neon.optimizers import GradientDescentMomentum, MultiOptimizer, Schedule
from neon.transforms import Rectlin, Softmax, CrossEntropyMulti, TopKMisclassification
from neon.models import Model
from neon.data import ImageLoader
from neon.callbacks.callbacks import Callbacks
# parse the command line arguments (generates the backend)
parser = NeonArgparser(__doc__)
parser.add_argument('--subset_pct',
type=float,
default=100,
help='subset of training dataset to use (percentage)')
parser.add_argument(
'--test_only',
action='store_true',
help='skip fitting - evaluate metrics on trained model weights')
args = parser.parse_args()
if args.test_only:
if args.model_file is None:
raise ValueError('To test model, trained weights need to be provided')
# setup data provider
img_set_options = dict(repo_dir=args.data_dir,
inner_size=224,
subset_pct=args.subset_pct)
