Esempio n. 1
0
def create_data_stream(args):
    print(args)
    sw = StopWatch()

    if not args.no_copy:
        with sw:
            print('Copying data to local machine...')
            rsync = Rsync(args.tmpdir)
            rsync.sync(args.data_path)

        args.data_path = os.path.join(args.tmpdir,
                                      os.path.basename(args.data_path))
    return fuel_utils.get_datastream(path=args.data_path,
                                     which_set=args.dataset,
                                     batch_size=args.batch_size)
Esempio n. 2
0
            print('Previously trained model detected: {}'.format(reload_path))
            print('Training continues')
            args.reload_model = reload_path

    ##############
    # print args #
    ##############
    print(args)

    sw = StopWatch()

    if not args.no_copy:
        print('Loading data streams from {}'.format(args.data_path))
        print('Copying data to local machine...')
        rsync = Rsync(args.tmpdir)
        rsync.sync(args.data_path)
        args.data_path = os.path.join(args.tmpdir,
                                      os.path.basename(args.data_path))
        sw.print_elapsed()

    ####################
    # load data stream #
    ####################
    train_datastream = get_datastream(path=args.data_path,
                                      which_set=args.train_dataset,
                                      batch_size=args.batch_size)
    valid_datastream = get_datastream(path=args.data_path,
                                      which_set=args.valid_dataset,
                                      batch_size=args.batch_size)
    test_datastream = get_datastream(path=args.data_path,
                                     which_set=args.test_dataset,