name='train') valid = ArrayIterator(self.valid_x, self.valid_y, lshape=self.shape, make_onehot=False, name='valid') self._data_dict = {'train': train, 'valid': valid} return self._data_dict def r_score(y_true, y_pred): r2 = r2_score(y_true, y_pred) return (np.sign(r2) * math.sqrt(math.fabs(r2))) parser = NeonArgparser(__doc__) parser.add_argument('-quick', '--quick_mode', action="store_true", help="use a small subset of the data") parser.add_argument('-ts', '--time_steps', default=7, help='number of time steps') args = parser.parse_args() dataset = Fin(nlags=int(args.time_steps), path=args.data_dir, quick=args.quick_mode) train = dataset.train_iter valid = dataset.valid_iter
""" import util from objectlocalization import PASCALVOC from neon.backends import gen_backend from neon.util.persist import get_data_cache_dir, save_obj from neon.util.argparser import NeonArgparser, extract_valid_args from neon.util.compat import range from neon import logger as neon_logger from voc_eval import voc_eval import numpy as np import faster_rcnn from tqdm import tqdm # parse the command line arguments parser = NeonArgparser(__doc__, default_overrides={'batch_size': 1}) parser.add_argument('--normalize', action='store_true', help='Normalize the final bounding box regression layers.') parser.add_argument('--output', default=None, help='File to save inference results (optional)') parser.add_argument('--width', type=int, default=1000, help='Width of input image') parser.add_argument('--height', type=int, default=1000, help='Height of input image')
# evaluation error_rate = mlp_model.eval(valid_set) logger.info('Mis-classification error on validation set= %0.1f', error_rate * 100) reslts = mlp_model.get_outputs(valid_set) return reslts # -------------------------------------------------------------------------------------# if __name__ == "__main__": # parse the command line arguments parser = NeonArgparser() parser.add_argument('--data_set_file', default='data/data_set.pkl', type=validate_existing_filepath, help='train and validation sets path') parser.add_argument('--model_prm', default='data/mcs_model.prm', type=validate_parent_exists, help='trained model full path') args = parser.parse_args() # generate backend, it is optional to change to backend='mkl' be = gen_backend(backend='cpu', batch_size=10) # read training and validation data file
def main(): # larger batch sizes may not fit on GPU parser = NeonArgparser(__doc__, default_overrides={'batch_size': 4}) parser.add_argument("--bench", action="store_true", help="run benchmark instead of training") parser.add_argument("--num_classes", type=int, default=12, help="number of classes in the annotation") parser.add_argument("--height", type=int, default=256, help="image height") parser.add_argument("--width", type=int, default=512, help="image width") args = parser.parse_args(gen_be=False) # check that image dimensions are powers of 2 if((args.height & (args.height - 1)) != 0): raise TypeError("Height must be a power of 2.") if((args.width & (args.width - 1)) != 0): raise TypeError("Width must be a power of 2.") (c, h, w) = (args.num_classes, args.height, args.width) # need to use the backend with the new upsampling layer implementation be = NervanaGPU_Upsample(rng_seed=args.rng_seed, device_id=args.device_id) # set batch size be.bsz = args.batch_size # couple backend to global neon object NervanaObject.be = be shape = dict(channel_count=3, height=h, width=w, subtract_mean=False) train_params = ImageParams(center=True, flip=False, scale_min=min(h, w), scale_max=min(h, w), aspect_ratio=0, **shape) test_params = ImageParams(center=True, flip=False, scale_min=min(h, w), scale_max=min(h, w), aspect_ratio=0, **shape) common = dict(target_size=h*w, target_conversion='read_contents', onehot=False, target_dtype=np.uint8, nclasses=args.num_classes) train_set = PixelWiseImageLoader(set_name='train', repo_dir=args.data_dir, media_params=train_params, shuffle=False, subset_percent=100, index_file=os.path.join(args.data_dir, 'train_images.csv'), **common) val_set = PixelWiseImageLoader(set_name='val', repo_dir=args.data_dir,media_params=test_params, index_file=os.path.join(args.data_dir, 'val_images.csv'), **common) # initialize model object layers = gen_model(c, h, w) segnet_model = Model(layers=layers) # configure callbacks callbacks = Callbacks(segnet_model, eval_set=val_set, **args.callback_args) opt_gdm = GradientDescentMomentum(1.0e-6, 0.9, wdecay=0.0005, schedule=Schedule()) opt_biases = GradientDescentMomentum(2.0e-6, 0.9, schedule=Schedule()) opt_bn = GradientDescentMomentum(1.0e-6, 0.9, schedule=Schedule()) opt = MultiOptimizer({'default': opt_gdm, 'Bias': opt_biases, 'BatchNorm': opt_bn}) cost = GeneralizedCost(costfunc=CrossEntropyMulti()) if args.bench: segnet_model.initialize(train_set, cost=cost) segnet_model.benchmark(train_set, cost=cost, optimizer=opt) sys.exit(0) else: segnet_model.fit(train_set, optimizer=opt, num_epochs=args.epochs, cost=cost, callbacks=callbacks) # get the trained segnet model outputs for valisation set outs_val = segnet_model.get_outputs(val_set) with open('outputs.pkl', 'w') as fid: pickle.dump(outs_val, fid, -1)
import os from neon import logger as neon_logger from neon.optimizers import GradientDescentMomentum, Schedule, MultiOptimizer from neon.transforms import Accuracy from neon.callbacks.callbacks import Callbacks from neon.util.argparser import NeonArgparser from data import make_train_loader, make_test_loader from network import create_network # parse the command line arguments train_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'train.cfg') config_files = [train_config] if os.path.exists(train_config) else [] parser = NeonArgparser(__doc__, default_config_files=config_files) parser.add_argument('--subset_pct', type=float, default=100, help='subset of training dataset to use (percentage)') args = parser.parse_args() random_seed = 0 if args.rng_seed is None else args.rng_seed model, cost = create_network() # setup data provider assert 'train' in args.manifest, "Missing train manifest" assert 'test' in args.manifest, "Missing validation manifest" train = make_train_loader(args.manifest['train'], args.manifest_root, model.be, args.subset_pct, random_seed)
train_X[i, :] = wave_data train_y = ReadCSV("data/train.csv") train_y = np.array(train_y[0:1000]) # test_X = np.zeros((test_dim, nframes), dtype=np.float32) # for i in range(0, test_dim): # filename = path+"test/test%d.aiff" % (i+1) # wave_data_gram = ReadAIFF(filename) # test_X[i, :] = wave_data_gram.reshape(nframes) train_X, train_y = shuffle(train_X, train_y) return train_X[0:700], train_y[0:700], train_X[700:1000], train_y[700:1000] parser = NeonArgparser(__doc__) # Use gpu args = parser.parse_args() args.epochs = 50 train_set_x, train_set_y, valid_set_x, valid_set_y = load_data("data/") train_set = ArrayIterator(train_set_x, train_set_y, nclass=2) valid_set = ArrayIterator(valid_set_x, valid_set_y, nclass=2) init_uni = Gaussian(loc=0.0, scale=0.01) # setup model layers layers = [DeepBiRNN(4000, init=init_uni, activation=Rectlin()), DeepBiRNN(1000, init=init_uni, activation=Rectlin()), DeepBiRNN(200, init=init_uni, activation=Rectlin()), Affine(nout=2, init=init_uni, activation=Logistic(shortcut=True))]
python examples/imagenet_allcnn.py -w </path/to/ImageNet/macrobatches> """ from neon.util.argparser import NeonArgparser from neon.initializers import Kaiming from neon.optimizers import GradientDescentMomentum, Schedule from neon.layers import Conv, Dropout, Activation, Pooling, GeneralizedCost, DataTransform from neon.transforms import Rectlin, Softmax, CrossEntropyMulti, Normalizer from neon.models import Model from neon.callbacks.callbacks import Callbacks from neon.data import ImageLoader # parse the command line arguments parser = NeonArgparser(__doc__, default_overrides=dict(batch_size=64)) parser.add_argument('--deconv', action='store_true', help='save visualization data from deconvolution') parser.add_argument('--subset_pct', type=float, default=100, help='subset of training dataset to use (percentage)') args = parser.parse_args() # setup data provider img_set_options = dict(repo_dir=args.data_dir, inner_size=224, scale_range=256, subset_pct=args.subset_pct) train = ImageLoader(set_name='train', **img_set_options)
import os from neon.util.argparser import NeonArgparser from neon.optimizers import GradientDescentMomentum, Schedule from neon.transforms import TopKMisclassification from neon.callbacks.callbacks import Callbacks from data import make_alexnet_train_loader, make_validation_loader from network_allcnn import create_network # parse the command line arguments (generates the backend) train_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'train.cfg') config_files = [train_config] if os.path.exists(train_config) else [] parser = NeonArgparser(__doc__, default_config_files=config_files, default_overrides=dict(batch_size=64)) parser.add_argument('--deconv', action='store_true', help='save visualization data from deconvolution') parser.add_argument('--subset_pct', type=float, default=100, help='subset of training dataset to use (percentage)') args = parser.parse_args() model, cost = create_network() rseed = 0 if args.rng_seed is None else args.rng_seed # setup data provider assert 'train' in args.manifest, "Missing train manifest"
from datetime import datetime from neon.callbacks.callbacks import Callbacks, GANCostCallback from neon.callbacks.plotting_callbacks import GANPlotCallback from neon.optimizers import RMSProp from neon.util.argparser import NeonArgparser from neon.util.persist import ensure_dirs_exist from network_gan import create_model from lsun_data import make_loader # parse the command line arguments train_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'train.cfg') config_files = [train_config] if os.path.exists(train_config) else [] parser = NeonArgparser(__doc__, default_config_files=config_files, default_overrides={ 'rng_seed': 0, 'batch_size': 64 }) parser.add_argument('-D', '--dmodel', type=str, default='dc', help='discriminator model type: dc or mlp, default dc') parser.add_argument('-G', '--gmodel', type=str, default='dc', help='generator model type: dc or mlp, default dc') parser.add_argument( '--subset_pct', type=float,
def main(): # parse the command line arguments parser = NeonArgparser(__doc__) args = parser.parse_args() logger = logging.getLogger() logger.setLevel(args.log_thresh) #Set up batch iterator for training images train = ImgMaster(repo_dir='spectroDataTmp', set_name='train', inner_size=400, subset_pct=100) val = ImgMaster(repo_dir='spectroDataTmp', set_name='validation', inner_size=400, subset_pct=100, do_transforms=False) test = ImgMaster(repo_dir='spectroTestDataTmp', set_name='validation', inner_size=400, subset_pct=100, do_transforms=False) train.init_batch_provider() test.init_batch_provider() print "Constructing network..." model = constuct_network() model.load_weights(args.model_file) #Optimizer opt = Adadelta() # configure callbacks valmetric = TopKMisclassification(k=5) callbacks = Callbacks(model, train, eval_set=val, metric=valmetric, **args.callback_args) cost = GeneralizedCost(costfunc=CrossEntropyMulti()) #flag = input("Press Enter if you want to begin training process.") print "Training network..." print args.epochs model.fit(train, optimizer=opt, num_epochs=args.epochs, cost=cost, callbacks=callbacks) mets = model.eval(test, metric=valmetric) print 'Validation set metrics:' print 'LogLoss: %.2f, Accuracy: %.1f %%0 (Top-1), %.1f %% (Top-5)' % ( mets[0], (1.0 - mets[1]) * 100, (1.0 - mets[2]) * 100) test.exit_batch_provider() train.exit_batch_provider()
from neon import logger as neon_logger from neon.data import Shakespeare from neon.initializers import Uniform from neon.layers import GeneralizedCost, LSTM, Affine from neon.models import Model from neon.optimizers import RMSProp from neon.transforms import Logistic, Tanh, Softmax, CrossEntropyMulti from neon.callbacks.callbacks import Callbacks from neon.util.argparser import NeonArgparser # parse the command line arguments default_overrides = dict(save_path='rnn_text_gen.pickle', serialize=1, batch_size=64) parser = NeonArgparser(__doc__, default_overrides=default_overrides) args = parser.parse_args() # hyperparameters time_steps = 64 hidden_size = 512 gradient_clip_value = 5 # download shakespeare text dataset = Shakespeare(time_steps, path=args.data_dir) train_set = dataset.train_iter valid_set = dataset.valid_iter # weight initialization init = Uniform(low=-0.08, high=0.08)
file_name = os.path.join(image_dir, 'image_{}.jpg'.format(n)) img.save(file_name) n = n + 1 return (all_boxes, all_gt_boxes) if __name__ == '__main__': """ Simple example of using the dataloader with pre-generated augmentation data """ arg_defaults = {'batch_size': 0} parser = NeonArgparser(__doc__, default_overrides=arg_defaults) parser.add_argument('--ssd_config', action='append', required=True, help='ssd json file path') parser.add_argument('--height', type=int, help='image height') parser.add_argument('--width', type=int, help='image width') parser.add_argument('--num_images', type=int, default=0, help='number of images to plot') parser.add_argument('--image_dir', type=str, help='folder to save sampled images') parser.add_argument('--score_threshold', type=float, help='threshold for predicted scores.') parser.add_argument('--output', type=str, help='file to save detected boxes.') args = parser.parse_args(gen_be=False) if args.model_file is None: parser.print_usage() exit('You need to specify model file to evaluate.') if args.ssd_config: args.ssd_config = {k: v for k, v in [ss.split(':') for ss in args.ssd_config]}