示例#1
0
    def __init__(self,
                 dataset_path,
                 batch_size,
                 mode,
                 x_keys,
                 y_keys,
                 split_name,
                 valid_split_number=0,
                 p=None):

        # Setting the random generator seed
        np.random.seed(cfg.RANDOM_SEED)

        self.images_path = os.path.join(dataset_path, 'images')
        if p is None:
            self.annotations_path = os.path.join(dataset_path, 'annotations',
                                                 '100')
            self.missing_annotations_path = os.path.join(
                dataset_path, 'missing_organs', '100')
        else:
            self.annotations_path = os.path.join(dataset_path, 'annotations',
                                                 str(p))
            self.missing_annotations_path = os.path.join(
                dataset_path, 'missing_organs', str(p))

        self.valid_split_number = valid_split_number
        self.split_name = split_name

        Dataset.__init__(self, dataset_path, batch_size, mode, x_keys, y_keys,
                         p)
def train(net_type, generator_fn_str, dataset_file, build_net_fn, featurized=True):
    d = Dataset(dataset_file + 'train.pgn')
    generator_fn = getattr(d, generator_fn_str)
    d_test = Dataset(dataset_file + 'test.pgn')

    X_val, y_val = d_test.load(generator_fn.__name__,
        featurized = featurized,
        refresh    = False,
        board = net_type)

    board_num_channels = X_val[0].shape[1] if net_type == 'to' else X_val[0].shape[0]
    model = build_net_fn(board_num_channels=board_num_channels, net_type=net_type)
    start_time = str(int(time.time()))
    try:
        plot_model(model, start_time, net_type)
    except:
        print("Skipping plot")
    from keras.callbacks import ModelCheckpoint
    checkpointer = ModelCheckpoint(
        filepath       = get_filename_for_saving(start_time, net_type),
        verbose        = 2,
        save_best_only = True)

    model.fit_generator(generator_fn(featurized=featurized, board=net_type),
        samples_per_epoch = SAMPLES_PER_EPOCH,
        nb_epoch          = NUMBER_EPOCHS,
        callbacks         = [checkpointer],
        validation_data   = (X_val, y_val),
        verbose           = VERBOSE_LEVEL)
def example_test(model,
                 images_dir="./data/images",
                 annotations_dir="./data/annotations",
                 demo_dir="./examples",
                 name="example01.png"):

    if not (os.path.exists(demo_dir)):
        os.mkdir(demo_dir)

    dataset = Dataset(training=False,
                      images_dir=images_dir,
                      annotations_dir=annotations_dir)

    image_name = np.random.choice(dataset.get_images_list())
    image_path = os.path.join(images_dir, image_name + ".jpg")
    mask_path = os.path.join(annotations_dir, "trimaps", image_name + ".png")

    orig_img = cv2.imread(image_path)[..., ::-1]
    true_mask = (cv2.imread(mask_path) * 100)[..., ::-1]
    pred_mask = process(image_path, model)

    labels = ["Original Image", "True Mask", "Predicted Mask"]
    collection = [orig_img, true_mask, pred_mask]

    creat_example(collection, labels, demo_dir, name, 3)
def main():
    """
    Creates a temporary file for the given input which is
    used to create a dataset, that is then evaluated on the given model.
    The generated summary is printed to standard out.
    """
    args, unknown_args = prepare_arg_parser().parse_known_args()
    model_file = args.model_file

    with suppress_stdout_stderr():
        model, _optimizer, vocab, _stats, cfg = train.load_model(
            model_file, unknown_args
        )

    _, filename = tempfile.mkstemp()
    try:
        with open(filename, "a") as f:
            input_ = sys.stdin.read()
            article = preprocess.parse(input_)
            print(f"{article}\tSUMMARY_STUB", file=f)

        with suppress_stdout_stderr():
            dataset = Dataset(filename, vocab, cfg)

        batch = next(dataset.generator(1, cfg.pointer))

        # don't enforce any min lengths (useful for short cmdline summaries")
        setattr(cfg, "min_summary_length", 1)
        bs = BeamSearch(model, cfg=cfg)
        summary = evaluate.batch_to_text(bs, batch)[0]
        print(f"SUMMARY:\n{summary}")
    finally:
        os.remove(filename)
class ParseState(object):
    """Support object for read()."""

    def __init__(self, config=defaults, name=None):
        self.config = config
        self.dataset = Dataset(name=name)
        self.document = Document()
        self.texts = []
        self.tags = []

    def sentence_break(self):
        if len(self.texts) == 0:
            return
        if self.config.iobes:
            self.tags = iob_to_iobes(self.tags)
        tokens = [Token(t, g) for t, g in zip(self.texts, self.tags)]
        self.document.add_child(Sentence(tokens=tokens))
        self.texts = []
        self.tags = []

    def document_break(self):
        self.sentence_break()
        if len(self.document) == 0:
            return
        self.dataset.add_child(self.document)
        self.document = Document()

    def finish(self):
        self.document_break()
示例#6
0
class ParseState(object):
    """Support object for read()."""
    def __init__(self, config=defaults, name=None):
        self.config = config
        self.dataset = Dataset(name=name)
        self.document = Document()
        self.texts = []
        self.tags = []

    def sentence_break(self):
        if len(self.texts) == 0:
            return
        if self.config.iobes:
            self.tags = iob_to_iobes(self.tags)
        tokens = [Token(t, g) for t, g in zip(self.texts, self.tags)]
        self.document.add_child(Sentence(tokens=tokens))
        self.texts = []
        self.tags = []

    def document_break(self):
        self.sentence_break()
        if len(self.document) == 0:
            return
        self.dataset.add_child(self.document)
        self.document = Document()

    def finish(self):
        self.document_break()
示例#7
0
def main(args):

    dataset = Dataset(batch_size)
    class_num = dataset.class_num()
    image_batch, label_batch = dataset.get_train_batch()
    image_batch = tf.reshape(image_batch, [-1, image_size, image_size, 3])

    glaph_ = Graph(batch_size, class_num)
    train_op = glaph_.inference(image_batch, label_batch)

    gpu_options = tf.GPUOptions(
        per_process_gpu_memory_fraction=gpu_memory_fraction)
    sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options,
                                            log_device_placement=False))

    sess.run(tf.global_variables_initializer())
    sess.run(tf.local_variables_initializer())
    # with sess.as_default():
    # sess.run(iterator.initializer)
    for epoch in range(epoch_num):
        i_batch, l_batch = sess.run([image_batch, label_batch])
        # logger.debug('i_batch: %s, l_batch: %s' % (i_batch, l_batch))
        _, loss_t, loss_p_adv, loss_p, los_re = sess.run(train_op)
        logger.debug('loss_t: %s, loss_p_adv: %s, loss_p: %s, los_re: %s' %
                     (loss_t, loss_p_adv, loss_p, los_re))

    sess.close()
示例#8
0
	def __init__(self, model, config):
		self.model   = model
		self.lr      = config['lr']
		self.epoches = config['epochs']
		self.batches = config['batchs']
		self.samples = config['samples']
		self.dataset = Dataset(self.samples, self.batches)
示例#9
0
文件: main.py 项目: hfxunlp/ATR
def eval(model, loss, dataset, params, batch_size=None):
    model.eval()

    if batch_size is None:
        batch_size = params.batch_size

    total_loss = 0.
    total_token = 0.
    hidden = model.init_hidden(batch_size)
    eval_data = Dataset(dataset, params.vocab)
    start_time = time.time()
    with torch.no_grad():
        for bidx, batch in enumerate(
                eval_data.batcher(
                    batch_size,
                    params.num_steps,
                )):
            x, t = batch

            x = x.to(params.device)
            t = t.to(params.device)

            hidden = repackage_hidden(hidden)
            logits, hidden = model(x, state=hidden)
            gloss = loss(logits.view(-1, logits.size(-1)), t.view(-1))

            num_token = (x > 0).float().sum()
            total_loss += gloss.item() * num_token
            total_token += num_token

    return total_loss / total_token, time.time() - start_time
    def __init__(self, model, optimizer, vocab, cfg, stats=defaultdict(int)):
        """
        Create a trainer instance for the given model.
        This includes creating datasets based on the `input_file` and
        `valid_file` in the given `cfg`.

        :param model: The Seq2Seq model to train
        :param optimizer: The optimizer for the `model`
        :param vocab: A `Vocabulary` instance to be used
        :param cfg: The current Config from which we get epochs/batch_size/files etc.
        :param stats:
            A dict with values such as epoch/running_avg_loss etc. when resuming training
        """

        self.cfg = cfg
        self.dataset = Dataset(cfg.train_file, vocab, cfg)
        self.validation_dataset = Dataset(cfg.valid_file,
                                          vocab,
                                          cfg,
                                          evaluation=True)

        self.model = model
        self.optimizer = optimizer
        self.validator = cfg.validator.lower()
        # if cfg.validate_every == 0, we validate once every epoch
        self.validate_every = (
            math.ceil(len(self.dataset) / cfg.batch_size)  # batches/epoch
            if cfg.validate_every == 0 else cfg.validate_every)
        self.rouge_valid = self.validator == "rouge"
        self.loss_valid = self.validator == "loss"
        self.scheduler = (
            None if cfg.learning_rate_decay >= 1.0 else ReduceLROnPlateau(
                optimizer,
                mode="min" if self.validator == "loss" else "max",
                factor=cfg.learning_rate_decay,
                patience=cfg.learning_rate_patience,
                verbose=True,
            ))
        self.coverage_loss_weight_decay = cfg.coverage_loss_weight_decay
        assert self.validator in ["loss", "rouge"]
        self.early_stopping = cfg.early_stopping
        self.patience = cfg.patience

        self.epoch = stats["epoch"]
        self.iteration = stats["iteration"]
        self.running_avg_loss = stats["running_avg_loss"]
        self.running_avg_cov_loss = stats["running_avg_cov_loss"]
        self.best_validation_score = stats["best_validation_score"]
        self.current_validation_score = stats.get("current_validation_score",
                                                  0)
        self.current_patience = stats.get("current_patience", 0)
        self.model_identifier = stats["model_identifier"]
        self.time_training = stats["time_training"]

        # Updated and managed from train function and context
        self.training_start_time = None
        self.writer = None
        self.pbar = None
示例#11
0
    def __init__(self, embeddings, evalita=None, twitter=False, debug=False):
        """
        This class define the neural network model as well
        as its operations for training, prediction and evaluation
        :param embeddings: Embeddings
        :param evalita: (optional) tuple => (evalita_test_path, evalita_train_path)
        :param twitter: (optional) to enable twitter char '#' '@' in dataset
        :param debug: (optional) to enable debug info and logs 
        """
        # initialization parameters
        self.hidden_size = embeddings.embed_dim
        self._embeddings = embeddings
        self.dataset = Dataset(embeddings, evalita, twitter)
        self.ntags = len(self.dataset.labels)

        # fixed parameters
        # TODO it is necessary to find a right forget_bias and learning_rate
        self.forget_bias = 1.0

        # initializing future variables
        self.embeddings = None
        self.labels = None
        self.output = None
        self.predictions = None
        self.logits = None
        self.loss = None
        self.transition_params = None

        # dropout keep probability
        self.keep_prob = None

        # utils
        self.batch_size = BATCH_SIZE
        self.labels_len = self.dataset.max_sequence

        # training utilities
        self.optimizer = None
        self.train_op = None

        # initializing feed dictioanry
        self.feed_dict = dict()

        # initializer
        self.init_g = None
        self.init_l = None

        # saver - initialization after variable init
        self.saver = None

        # debug
        self.debug = debug

        # tensorboard
        self.file_writer = tf.summary.FileWriter(LOG_DIR)
        self.summary = None

        # metrics
        self.metrics = None
示例#12
0
def evaluate(model_path, batch_size=128):
    print("Evaluating model {}".format(model_path))
    model = load_model(model_path)
    ds = Dataset()
    X, y = ds.get_test_data()

    gen = DataLoader(X, y, IMG_WIDTH, IMG_HEIGHT, batch_size)
    metrics = model.evaluate_generator(gen, steps=len(gen), workers=8)
    pprint(dict(zip(model.metrics_names, metrics)))
示例#13
0
 def load_train_val(cls, normalize=True):
     ds = loaders.load_cifar10(cls.data_path, 'train')
     if normalize:
         cls.mean, cls.std = pp.get_normalization_statistics(ds.images)
         ds = Dataset(pp.normalize(ds.images, cls.mean, cls.std), ds.labels,
                      ds.class_count)
     ds.shuffle()
     ds_train, ds_val = ds.split(0, int(ds.size * 0.8))
     return ds_train, ds_val
示例#14
0
def predict(df):
    basepath = '/media/user1/preprocessed'
    features = []
    truth_label = []
    df.sort()
    print(df)
    label = np.zeros((1, ))
    for name in df:
        print(name)
        label[0] = 0
        vector = np.zeros(800)
        if 'COR' in name or 'SUB' in name:
            label[0] = 1

        labels_df = pd.read_csv('./Book1.csv')
        path = labels_df['ID'].tolist()

        if name in path:
            label[0] = 1

        basedir = os.path.normpath(basepath)
        print(basedir)
        files = glob(basedir + '/' + name + '*.npy')
        files.sort()
        l = len(files)

        if l == 0:
            break

        for i in range(l):
            max_pro = 0
            img = np.load(files[i])
            s = img.shape

            if s[0] != 224:
                img = misc.imresize(img, (224, 224), 'bilinear')
            img = np.stack((img, img, img), axis=2)
            img = img[np.newaxis, :, :, :]

            test_generator = Dataset(
                img,
                label,
                augment=False,
                shuffle=False,
                input_form='t1',
                seed=seed,
            )

            test_generator.reset()
            test_results = evaluate.get_results(model, test_generator)
            probabilities = list(
                evaluate.transform_binary_probabilities(test_results))
            vector[i] = probabilities[0]

        features.append(vector)
        truth_label.append(label[0])
    return features, truth_label
示例#15
0
    def nfold_training(self, N=N_FOLD, model_path=MODEL_CKPT):
        """
        Ad-hoc routine to run nfold cross validation over tweets dataset
        :param N: int, (optional) number of folds
        :param model_path: string, (optional) path to model 
        """
        # re instantiating dataset
        self.dataset = Dataset(self._embeddings,
                               twitter=TWITTER_CHARS,
                               n_fold=N)

        metrics = []
        nfold_counter = 0
        # running nfold
        for training, testing_batch in zip(self.dataset.trainingset,
                                           self.dataset.testingset):

            nfold_counter += 1
            print("---- N FOLD RUN ", nfold_counter, "----")

            # instantiating a new session
            with tf.Session() as sess:
                # preparing model
                sess.run(self.init_g)
                sess.run(self.init_l)

                # restoring previous checkpoint
                self.restore_model(sess, path=model_path)

                # batching
                for i in range(0, int(len(training) / BATCH_SIZE) + 1):
                    print("[TRAIN TWEETS] RUN ", i)

                    # retrieving training batch
                    training_batch = training[i * BATCH_SIZE:(i + 1) *
                                              BATCH_SIZE]
                    # feeding dictionary
                    (sentences,
                     labels), sequence_lengths = self.dataset.build_batch(
                         training_batch, self._embeddings.vocabulary,
                         self.dataset.labels, self.dataset.max_sequence,
                         self.dataset.word_vector_len)

                    self.set_feed_dictionary(embeddings=(sentences,
                                                         sequence_lengths),
                                             labels=labels)

                    # training step
                    self.training_step(sess)

                # evaluating
                print("[EVALUATION TWEETS] running...")
                metrics.append(self.evaluate_model(sess,
                                                   testing=testing_batch))

        if LOG_METRICS is not None:
            self.metrics.write_log_metric_nfold(metrics, N, LOG_METRICS)
示例#16
0
    def __init__(self):
        self.dataset = Dataset()
        self.cap = None

        self.showMoments = False
        self.showEvaluation = False

        self.bufferFace = Buffer(self.ROLLING_WINDOW_LENGTH)
        self.bufferLeftEye = Buffer(self.ROLLING_WINDOW_LENGTH)
        self.bufferRightEye = Buffer(self.ROLLING_WINDOW_LENGTH)
示例#17
0
def prepare(args):
    logger = logging.getLogger("rc")
    logger.info('train test split...')
    #    train_test_split(args.all_file, args.train_file, args.test_file, args.train_rate)
    sen_data = Dataset(args)
    sen_data.build_vocab()

    with open(args.data_path, 'wb') as fout:
        pickle.dump(sen_data, fout)
    logger.info('Done with preparing!')
示例#18
0
def build_data(args):
    datafile = h5py.File(args.datafile, 'r')
    train_inputs = datafile['train_inputs']  # TODO stupid file name
    train_targs = datafile['train_targets']
    valid_inputs = datafile['valid_inputs']
    valid_targs = datafile['valid_targets']
    train = Dataset(train_inputs, train_targs, args.batch_size)
    valid = Dataset(valid_inputs, valid_targs, args.batch_size)
    return {'train':train, 'valid':valid}, \
            {'gram_size':datafile['gram_size'][0] - 1, 'vocab_size':datafile['vocab_size'][0], \
                'hid_size':args.d_hid, 'emb_size':args.d_emb}
示例#19
0
    def get_data(self, data_path):
        self.data_path = data_path
        train_path = data_path + '/test.csv'  # train_path = data_path + '/train.csv'
        test_path = data_path + '/test.csv'

        self.train_data = Dataset(train_path)
        self.test_data = Dataset(test_path)

        self.input_dim = self.train_data.x_dim
        self.num_label = self.train_data.num_label
        self.num_samples = self.train_data.__len__()
示例#20
0
def main(params):
    filepath = params['filepath']
    date = params['date']
    forecastcol = params['forecastcolumn']
    epochs = params['epochs']
    bs = params['batch_size']
    ws = params['sequence_length']

    #read the filepath
    df = pd.read_csv(filepath, sep=",")
    dataobj = Dataset(df, date, forecastcol)

    #normal trian and eval split
    #this split doesn't belong to model data preparation
    train_split, eval_split = dataobj.normal_split()

    model_train, model_labels = model_data_prep(train_split, ws)

    logging.info("Train Data Shape:{}".format(model_train.shape))
    logging.info("Train Label Shape:{}".format(model_labels.shape))
    
    #call a model file
    logging.info("============= MODEL TRAINING STARTED =============")
    network, modeldump = model.conv1d(model_train, 
                model_labels, epochs=epochs, 
                batch_size = bs)
    
    #model.plot_loss(modeldump)

    predictions = model.forecast_fn(network, 
                    model_train[-1],
                    len(eval_split), 
                    eval_split)
    

    assert len(eval_split) == len(predictions), "Length Mismatch between Actuals and Predictions"
    score = model.evaluate_model_performance(eval_split, predictions) #mse square

    #sve the model
    model_name = 'conv1d_{}.h5'.format(datetime.datetime.now().strftime('%d-%m-%Y'))
    network.save(model_name)

    #plot the figure
    plt.figure(figsize=(15, 7))
    plt.plot(eval_split, color='orange', label='Actual Values')
    plt.plot(predictions, color='red', label='Forecasted Values')
    plt.legend()
    plt.title("Model Error Rate:{0:2f}".format(score))
    plt.xticks(rotation=90)
    plt.savefig("conv1d_model_forecast.png")
    plt.show()
    
    logging.info("Model Score on Test Data:{}".format(score))
def detect(path,**kwargs):
    d = Dataset('./data')
    image = d.read(path)
    global rgb
    rgb = d.read(path)
    pupil = find_pupil(image, 15)[0]
    e = Ellipse()
    e._axes = (pupil[2], pupil[2])
    e._center = pupil[0:2]
    x, y = circle_perimeter(pupil[0], pupil[1], pupil[2])
    #rgb[x,y] = 255
    return e, image
示例#22
0
def main(epoch_count=1):
    from data import Dataset, loaders
    from data.preparers import Iccv09Preparer
    import ioutils
    from ioutils import console

    print("Loading and deterministically shuffling data...")
    data_path = os.path.join(
        ioutils.path.find_ancestor(os.path.dirname(__file__), 'projects'),
        'datasets/cifar-10-batches-py')
    ds = loaders.load_cifar10_train(data_path)
    labels = np.array([np.bincount(l.flat).argmax() for l in ds.labels])
    ds = Dataset(ds.images, labels, ds.class_count)
    ds.shuffle(order_determining_number=0.5)

    print("Splitting dataset...")
    ds_train, ds_val = ds.split(0, int(ds.size * 0.8))
    print(ds_train.size, ds_val.size)

    print("Initializing model...")
    model = ClfBaselineB(
        input_shape=ds.image_shape,
        class_count=ds.class_count,
        batch_size=128,
        learning_rate_policy=1e-6,
        #learning_rate_policy={
        #    'boundaries': [3, 5, 7],
        #    'values': [10**-i for i in range(1, 5)]
        #},
        ortho_penalty=0,
        use_multiclass_hinge_loss=False,
        training_log_period=100)

    def handle_step(i):
        text = console.read_line(impatient=True, discard_non_last=True)
        if text == 'd':
            viz.display(ds_val, lambda im: model.predict([im])[0])
        elif text == 'q':
            return True
        return False

    model.training_step_event_handler = handle_step

    #viz = visualization.Visualizer()
    print("Starting training and validation loop...")
    #model.test(ds_val)
    #ds_train = ds_train[:128]
    for i in range(epoch_count):
        model.train(ds_train, epoch_count=1)
        model.test(ds_train, "Training")
        model.test(ds_val, "Validation")
    model.save_state()
示例#23
0
def single_image_upload():
    # Access the user-sent image from the request object.
    fname = request.files['input-file'].filename
    fpath = os.path.join("imgs", fname)
    request.files['input-file'].save(fpath)

    # Create a dataset to contain the image.
    ds = Dataset(name='dataset_' + str(int(random() * 1000000)),
                 prefix='./imgs',
                 batch_size=1,
                 images=True)
    ds.initialize(sources=[fname])
    ds_id = STATE.add_dataset(ds)
    return json.dumps({'datasetId': ds_id})
示例#24
0
def load_dataset(file_name):
    dataset = Dataset(logger, file_name)
    columns = ['date', 'security', 'feature']
    dataset.drop_na(columns)
    dataset.factorize_label()
    dataset.sort_date()
    return dataset
示例#25
0
def main(args):

    # metadata for training
    trajectory_length = 20
    prediction_length = trajectory_length // 2

    print("************* Loading Dataset ***************")
    dataset = Dataset()
    dataset.load_data(args.dataset)
    training_data, testing_data = dataset.get_train_validation_batch(
        trajectory_length)

    # # reduce the size of training data..
    if args.truncated:
        print("Using truncated data")
        training_data = training_data[:10]
        testing_data = testing_data[:10]

    # experiment configs
    # experiment_embedding_size = [64]
    # experiment_hidden_size = [128]
    experiment_embedding_size = args.embedding_size
    experiment_hidden_size = args.hidden_size

    for embedding_size in experiment_embedding_size:
        for hidden_size in experiment_hidden_size:

            social_model = SocialModel(hidden_size=hidden_size,
                                       embedding_size=embedding_size)

            experiment(social_model,
                       training_data,
                       testing_data,
                       hidden_size,
                       embedding_size,
                       args,
                       model_name='social_lstm')

            # lstm_model = VanillaLSTMModel(hidden_size, embedding_size)
            # experiment(
            #     lstm_model, training_data, testing_data,
            #     hidden_size = hidden_size,
            #     embedding_size = embedding_size,
            #     num_epochs = 100,
            #     lr = args.lr,
            #     model_name = 'vanilla_lstm',
            # )

    print("done!")
def validate(model_hdf5, net_type, generator_fn_str, dataset_file, featurized=True):
    from keras.models import load_model
    import data

    d_test = Dataset(dataset_file + 'test.pgn')
    X_val, y_val = d_test.load(generator_fn_str,
        featurized = featurized,
        refresh    = False,
        board      = "both")
    boards = data.board_from_state(X_val)

    if net_type == "from":
        model_from = load_model("saved/" + model_hdf5)
        y_hat_from = model_from.predict(X_val)
        num_correct = 0
        for i in range(len(boards)):
            if y_val[0][i,np.argmax(y_hat_from[i])] > 0:
                num_correct += 1
        print(num_correct / len(boards))

    elif net_type == "to":
        model_to = load_model("saved/" + model_hdf5)
        y_hat_to = model_to.predict([X_val, y_val[0].reshape(y_val[0].shape[0],1,X_val.shape[2],X_val.shape[3])])
        num_correct = 0
        for i in range(len(boards)):
            if y_val[1][i,np.argmax(y_hat_to[i])] > 0:
                num_correct += 1
        print(num_correct / len(boards))

    elif net_type == "from_to":
        model_from = load_model("saved/" + model_hdf5[0])
        model_to = load_model("saved/" + model_hdf5[1])
        y_hat_from = model_from.predict(X_val)

        for i in range(len(boards)):
            from_square = np.argmax(y_hat_from[i])
            y_max_from = np.zeros((1,1,X_val.shape[2],X_val.shape[3]))
            y_max_from.flat[from_square] = 1

            y_hat_to = model_to.predict([np.expand_dims(X_val[i], 0), y_max_from])
            to_square = np.argmax(y_hat_to)
            move_attempt = data.move_from_action(from_square, to_square)
            if boards[i].is_legal(move_attempt):
                print("YAY")
            else:
                print("BOO")
            print(move_attempt)
            move = data.move_from_action(np.argmax(y_val[0]), np.argmax(y_val[1]))
            print(move)
示例#27
0
    def treino(self, input_train):
        self.train = input_train
        p = Parameters()
        d = Dataset()
        with tf.Session(graph = self.graph) as session:
            
           # Tensorboard area
            self._loss_gen = tf.placeholder(tf.float32, shape=())
            self._loss_dis = tf.placeholder(tf.float32, shape=())
            self.img_pl = tf.placeholder(tf.float32, shape=self.shape_out)

            self.score_summary_op = tf.summary.merge([
                tf.summary.scalar('Generator_loss', self._loss_gen),
                tf.summary.scalar('Discriminator_loss_real', self._loss_dis),
                tf.summary.image('Generated_images', self.img_pl, 128)
            ])
            logdir = self.param.TENSORBOARD_DIR + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + "/"
            self.writer = tf.summary.FileWriter(logdir, session.graph)

            # weight initialization
            session.run(tf.global_variables_initializer())
            saver = tf.train.Saver()

            # full optimization
            for epoch in range(self.param.NUM_EPOCHS_FULL):
                print('Epoch: '+ str(epoch+1), end=' ')
                self._training_epoch(session, epoch+1)

            path_model = self.param.LOG_DIR_MODEL  + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + '_gan.ckpt'
            saver.save(session, path_model)
            print("The model has saved in: " + path_model)
示例#28
0
def main():
    path = argv[1]
    dr_config = Config(constants.DREAM_CONFIG)
    with open(path, 'rb') as f:
        dr_model = torch.load(f)

    bc = BasketConstructor(constants.RAW_DATA_DIR, constants.FEAT_DATA_DIR)
    ub_basket = bc.get_baskets('prior', reconstruct=False)

    # this function needs a sample input to infer types
    baskets, lens, users = Dataset(ub_basket)[0: dr_config.batch_size]

    baskets, lens, users = sort_batch_of_lists(baskets, lens, users)
    baskets = pad_batch_of_lists(baskets, lens[0])

    dr_hidden = dr_model.init_hidden(dr_config.batch_size)
 
    ub_seqs = [] # users' basket sequence

    for ubaskets in baskets:
        x = dr_model.embed_baskets(ubaskets)
        ub_seqs.append(torch.cat(x, 0).unsqueeze(0))

    ub_seqs = torch.cat(ub_seqs, 0)

    r_model = dr_model.rnn

    print("Converting model of type: " + str(type(dr_model)))
    m_in = (ub_seqs, dr_hidden)
    m_out = r_model(*m_in)

    torch.onnx.export(r_model, ub_seqs, './models/model.onnx', verbose=True)
示例#29
0
文件: convert.py 项目: kaimast/DREAM
def main():
    torch_tvm.enable(opt_level=3,
                     device_type="gpu",
                     device="cuda",
                     host="llvm")

    path = argv[1]
    dr_config = Config(constants.DREAM_CONFIG)
    with open(path, 'rb') as f:
        dr_model = torch.load(f)

    bc = BasketConstructor(constants.RAW_DATA_DIR, constants.FEAT_DATA_DIR)
    ub_basket = bc.get_baskets('prior', reconstruct=False)

    # this function needs a sample input to infer types
    baskets, lens, users = Dataset(ub_basket)[0:dr_config.batch_size]

    baskets, lens, users = sort_batch_of_lists(baskets, lens, users)
    baskets = pad_batch_of_lists(baskets, lens[0])

    dr_hidden = dr_model.init_hidden(dr_config.batch_size)

    ub_seqs = []  # users' basket sequence

    for ubaskets in baskets:
        x = dr_model.embed_baskets(ubaskets)
        ub_seqs.append(torch.cat(x, 0).unsqueeze(0))

    ub_seqs = torch.cat(ub_seqs, 0)

    arg = [ub_seqs, dr_hidden]

    relay_graph = torch_tvm.to_relay(dr_model.rnn, arg)
示例#30
0
def main(config_file,mode,distributed):
    config = check_params(config_file)
    if mode in ["Train","train"]:
        train_dataset = Dataset(config["train_params"]["input_path"],config["train_params"]["imsize"])
        if distributed:
            import horovod as hvd
            hvd.init()
            if hvd.rank()==0:
                writer = setup_tensorboard(get_params(config["train_params"],"tensorboard_location","./summary/"))
            train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset, num_replicas=hvd.size(),
                                                                            rank=hvd.rank())

            train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=config["train_params"]["batch_size"],
                                                       sampler=train_sampler, shuffle=True)
            model = ConvLSTM(**config["model_params"])
            optimizer = hvd.DistributedOptimizer(model.optimizer, named_parameters=model.named_parameters())
            hvd.broadcast_parameters(model.state_dict(), root_rank=0)
            train_distributed(model,train_loader,optimizer,config,writer)
        else:
            writer = setup_tensorboard(get_params(config["train_params"], "tensorboard_location", "./summary/"))
            train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=config["train_params"]["batch_size"],
                                                     shuffle = True)
            model = ConvLSTM(**config["model_params"])
            train(model,train_loader,model.optimizer,config,writer)
    elif mode in ["infer","Infer"]:
        model = ConvLSTM(**config["model_params"])
        model.load_state_dict(config["infer_params"]["model_save_path"])
        output_file = open(config["infer_params"]["output_path"])
示例#31
0
def train(auto_stop, steps=50, threshold=0.5):
    dataset = Dataset()
    discriminator_feeder = TrainFeeder(dataset)
    generator_feeder = TrainFeeder(dataset)
    discriminator = Discriminator(len(dataset.ci2n)).cuda()
    generator = Generator(len(dataset.ci2n)).cuda()
    criterion = torch.nn.BCEWithLogitsLoss(size_average=False)
    discriminator_optimizer = torch.optim.Adam(discriminator.parameters(), lr=1e-4)
    generator_optimizer = torch.optim.Adam(generator.parameters(), lr=1e-4)
    discriminator_feeder.prepare('train')
    generator_feeder.prepare('train')
    if os.path.isfile(ckpt_path):
        ckpt = torch.load(ckpt_path)
        discriminator.load_state_dict(ckpt['discriminator'])
        generator.load_state_dict(ckpt['generator'])
        discriminator_optimizer.load_state_dict(ckpt['discriminator_optimizer'])
        generator_optimizer.load_state_dict(ckpt['generator_optimizer'])
        discriminator_feeder.load_state(ckpt['discriminator_feeder'])
        generator_feeder.load_state(ckpt['generator_feeder'])
    loss = 1
    while True:
        #run_generator_epoch(generator, discriminator, generator_feeder, criterion, generator_optimizer, 0.2, 100)
        loss = run_discriminator_epoch(generator if loss < threshold else None, discriminator, discriminator_feeder, criterion, discriminator_optimizer, steps)
        if loss < threshold:
            run_generator_epoch(generator, discriminator, generator_feeder, criterion, generator_optimizer, 0.2, 100)
        utils.mkdir(config.checkpoint_folder)
        torch.save({
            'discriminator': discriminator.state_dict(),
            'generator': generator.state_dict(),
            'discriminator_optimizer': discriminator_optimizer.state_dict(),
            'generator_optimizer': generator_optimizer.state_dict(),
            'discriminator_feeder': discriminator_feeder.state(),
            'generator_feeder': generator_feeder.state()
            }, ckpt_path)
        print('MODEL SAVED.')
示例#32
0
def train(dataset_path, ratio=(1, 1)):
    g_loss_sum = d_loss_sum = 0

    for i, batch in enumerate(Dataset(dataset_path).batches(64)):
        real = torch.ones(64, 1).to(device)
        fake = torch.zeros(64, 1).to(device)

        #  train g
        z = torch.randn(64, 64, requires_grad=True).to(device)
        gen_x = g(z)
        gen_y = d(gen_x)

        g_loss = loss_func(gen_y, real)
        g_opt.zero_grad()
        (g_loss * ratio[0]).backward()
        g_opt.step()

        # train d
        real_x = batch.reshape(64, 1, 64, 64)
        real_x = torch.tensor(real_x, requires_grad=True).to(device)
        real_y = d(real_x)
        fake_y = d(gen_x.detach())

        d_loss = (loss_func(real_y, real) + loss_func(fake_y, fake)) / 2
        d_opt.zero_grad()
        (d_loss * ratio[1]).backward()
        d_opt.step()

        g_loss_sum += g_loss.item()
        d_loss_sum += d_loss.item()

        if (i + 1) % 100 == 0:
            print(f'g: {g_loss_sum / 100}, d: {d_loss_sum / 100}')
            g_loss_sum = d_loss_sum = 0
示例#33
0
def detect(path,**kwargs):
    d = Dataset('./data')
    image = d.read(path)
    global rgb
    rgb = d.read(path)
    pupil = find_pupil(image, 15)[0]
    e = Ellipse()
    e._axes = (pupil[2], pupil[2])
    e._center = pupil[0:2]
    #img, points, ellipse = find_iris(image, pupil, **kwargs)
    x, y = circle_perimeter(pupil[0], pupil[1], pupil[2])
    rgb[x,y] = 255
    #ex, ey = ellipse.center
    #major, minor = ellipse.axes
    #orientation = ellipse.orientation
    #imshow(rgb)
    #x, y = ellipse_perimeter(int(ex), int(ey), int(major), int(minor), orientation)
    #rgb[x,y] = (220, 40, 40)
    #imshow(rgb)
    return e, image
示例#34
0
import numpy as np
import matplotlib.pyplot as plt

from skimage import data, filter, color
from skimage.transform import hough_circle
from skimage.feature import peak_local_max
from skimage.draw import circle_perimeter
from skimage.util import img_as_ubyte
from data import Dataset


# Load picture and detect edges
d = Dataset('data')
image = d.read(d.images[0])
edges = filter.canny(image, sigma=2, low_threshold=10, high_threshold=50)

fig, ax = plt.subplots(ncols=1, nrows=1, figsize=(6, 6))

# Detect two radii
hough_radii = np.arange(15, 30, 2)
hough_res = hough_circle(edges, hough_radii)

centers = []
accums = []
radii = []

for radius, h in zip(hough_radii, hough_res):
    # For each radius, extract two circles
    peaks = peak_local_max(h, num_peaks=2)
    centers.extend(peaks)
    accums.extend(h[peaks[:, 0], peaks[:, 1]])
 def __init__(self, config=defaults, name=None):
     self.config = config
     self.dataset = Dataset(name=name)
     self.document = Document()
     self.texts = []
     self.tags = []