示例#1
0
def evaluate(valloader, model, criterion, save_image):

    # switch the model to evaluate mode ( important for dropout layers or batchnorm layers )
    model.evaluate()
    loss_sum = 0
    for i, data in enumerate(valloader):
        # get train and label data
        val, label = data
        # put on gpu or cpu
        val = val.to(device)
        # label is of type TensorLong

        label = label.to(device)

        # for the CrossEntropyLoss:
        # outputs needs to be of size: Minibatch, classsize, dim 1, dim 2 , ...
        # outputs  are 2 classes with 2d images. Channelsize = class size
        # label needs to be of format: Minibatch, dim 1, dim 2, ...
        # i cut the channel info for it to work, because it is only a 2d image.
        # as an alternative, one could add 1 channel for class in train, than label does not need any change
        # label normally looks like: ( minibatchsize, 1, width, height )
        label = label.view(label.size(0), label.size(2), label.size(3))
        # forward + backward + optimize
        outputs = model(val, padding=args.pad)
        loss = criterion(torch.log(outputs), label)
        running_loss = loss.item()
        loss_sum = loss_sum + running_loss

        if save_image:
            save_images_eval(outputs, './eval/', index=i + 1)

        del outputs, val, label, loss

    loss_avg = loss_sum / (i + 1)
    return loss_avg
示例#2
0
def run():
    print('loading training data...')
    X, y = load_training_data()
    #min_max_scaler = preprocessing.MinMaxScaler()
    #X = min_max_scaler.fit_transform(X)

    util.count(y)
    print('loading data completed.')
    print('loading models...')
    models = md.getClassifiers()

    print('training models...')
    md.train(models, X, y)
    print('training models completed...')

    print('loading test data...')
    X_test, y_test = load_test_data()
    #min_max_scaler = preprocessing.MinMaxScaler()
    #X_test = min_max_scaler.fit_transform(X_test)

    util.count(y_test)
    print('loading test data completed...')

    print('eval models...')
    md.evaluate(models, X_test, y_test)
    print('eval models completed...')
示例#3
0
def pred(data, countries, start):
	# ***** Get predictions for all country *****#
	open('predictions.txt', 'w').close()
	for country in countries:
		record_predictions('\n' + country + ':')
		print('\n', country, ':')
		df = data[['Week', country]]
		df = df[np.isfinite(df[country])] # remove empty rows
		df = df.rename(index=str, columns={country: "Now"})

		for i in range(3):
			df_ = df.copy()
			df_ = prepare(df_, pred=True)
			# train the model using all data except for the latest record
			X = df_.iloc[:,:-1]
			y = df_.iloc[:,-1]
			X,y = model.preprocess_data(X,y)
			X_train, y_train = X[:-1], y[:-1]
			# printX_train, '\n', y_train
			lr, label = model.lr(X_train,y_train)
			model.evaluate(lr,X,y,X_train,X_train,y_train,y_train,label,graph=False)

			fea = X[-1]
			y_pred = lr.predict([fea])
			print('features:', fea, '\npredictions:', y_pred[0])
			record_predictions(str(y_pred[0]))

			rec = pd.DataFrame([[start+i, y_pred[0]]], columns=['Week','Now'])
			# print'\nbefore appending\n', df
			df = df.append(rec)
示例#4
0
def main():
    ###############################################################################
    # Load data
    ###############################################################################
    d = util.Dictionary()
    if args.task == "train":
        logging.info("Reading train...")
        trncorpus = util.read_corpus(args.ftrn, d, True)
        d.freeze()  # no new word types allowed
        vocab_size = d.size()
        # save dict
        d.save_dict(fprefix + ".dict")
        logging.info("Reading dev...")
        devcorpus = util.read_corpus(args.fdev, d, False)

    elif args.task == "test":
        logging.info("Reading test...")
        d.load_dict(args.fdct)
        d.freeze()
        vocab_size = d.size()
        # load test corpus
        tstcorpus = util.read_corpus(args.ftst, d, False)

    ###############################################################################
    # Build the model
    ###############################################################################
    if args.task == "train":
        model_fname = fprefix + ".model"
        pretrained_model = None
        if args.fmod:
            # load pre-trained model
            pretrained_model = model.load_model(args.fmod, vocab_size,
                                                args.nclass, args.inputdim,
                                                args.hiddendim, args.nlayer,
                                                args.droprate)
            logging.info("Successfully loaded pretrained model.")

        trained = model.train(trncorpus, devcorpus, vocab_size, args.nclass,
                              args.inputdim, args.hiddendim, args.nlayer,
                              args.trainer, args.lr, args.droprate, args.niter,
                              args.logfreq, args.verbose, model_fname,
                              pretrained_model)
        dev_accuracy = model.evaluate(trained, devcorpus.docs)
        logging.info("Final Accuracy on dev: %s", dev_accuracy)
        model.save_model(trained, model_fname)

    else:
        trained_model = model.load_model(args.fmod, vocab_size, args.nclass,
                                         args.inputdim, args.hiddendim,
                                         args.nlayer, args.droprate)
        tst_accuracy = model.evaluate(trained_model, tstcorpus.docs)
        logging.info("Final Accuracy on test: %s", tst_accuracy)
示例#5
0
def main(config):
    data_gainer = Data(config)
    train_x, train_y = data_gainer.get_train_data()
    valid_x, valid_y = data_gainer.get_valid_data()
    test_x = data_gainer.get_test_data()
    train_label = data_gainer.return_label("train")
    valid_label = data_gainer.return_label("valid")
    test_label = data_gainer.return_label("test")

    # train
    _, tuning_loss_mean = train(train_x, train_y, config)
    loss_plot(tuning_loss_mean)
    # evaluate
    y_pred_valid = predict(valid_x, config)
    evaluate(y_pred_valid, valid_label, data_gainer)
示例#6
0
def wave(num):
    """
    The basic calculating and drawing function.
    The internal function is 'cosine' >> (x, y) values.

    The function uses slider values to variate the output.
    Slider values are defined by <slider name>.val
    """
    for i, letter in enumerate('abcdefghijklmnopqrstuvwxyz'):
        evaluate(i, np.array([s.val for s in slides]))

        sprite = pygame.image.load('out.png')
        sprite = pygame.transform.scale2x(sprite)
        rect = sprite.get_rect()
        rect.center = (50 + i % 13 * 62, 50 + i//13 * 90)
        screen.blit(sprite, rect)
示例#7
0
def run_training(train_file, base_path, device, bidirectional=False):
    corpus_name = os.path.split(train_file)[1].replace('.train', '')
    print(f"Training with {corpus_name}")
    model_name = corpus_name + ("_LSTM" if not bidirectional else "_BiLSTM")
    model_path = base_path.joinpath(model_name)
    model_path.mkdir(exist_ok=True)
    dev_file = train_file.replace(".train", ".dev")
    test_file = train_file.replace(".train", ".test")
    print(f"Loading {train_file}")
    train_data = EosDataset(train_file, split_dev=False, min_freq=10000, save_vocab=model_path.joinpath("vocab.pickle"),
                            remove_duplicates=False, shuffle_input=False)
    print(f"Loading {dev_file}")
    dev_data = EosDataset(dev_file, split_dev=False, min_freq=10000, load_vocab=model_path.joinpath("vocab.pickle"),
                          remove_duplicates=False, shuffle_input=False)
    print(f"Loading {test_file}")
    test_data = EosDataset(test_file, split_dev=False, min_freq=10000, load_vocab=model_path.joinpath("vocab.pickle"),
                           remove_duplicates=False, shuffle_input=False)
    model = DeepEosModel(max_features=20000, rnn_bidirectional=bidirectional)
    model.to(device)
    print(model)
    print(f"Number of trainable parameters: {sum(p.numel() for p in model.parameters() if p.requires_grad)}")

    optimizer = optim.Adam(model.parameters(), lr=0.001)
    print("Starting Traning")
    train(model, train_data, dev_data, base_path=model_path, optimizer=optimizer, epochs=5, device=device)
    print("Evaluating")
    with open(model_path.joinpath("evaluation.txt"), 'w', encoding='UTF-8') as f:
        precision, recall, f1, accuracy = evaluate(model, test_data, device=device)
        f.write(f"Precision: {precision}\nRecall: {recall}\nF1: {f1}\nAccuracy: {accuracy}\n")
示例#8
0
def test():
    model = edsr(scale=scale, num_res_blocks=depth, num_filters=channels)
    checkpoint = tf.train.Checkpoint(step=tf.Variable(0),
                                     psnr=tf.Variable(-1.0),
                                     optimizer=Adam(1e-04),
                                     model=model)
    checkpoint_manager = tf.train.CheckpointManager(checkpoint=checkpoint,
                                                    directory='./ckpt',
                                                    max_to_keep=3)
    restore(checkpoint, checkpoint_manager)

    video_valid = video_ds(subset='valid')
    valid_ds = video_valid.dataset(batch_size=1,
                                   random_transform=False,
                                   repeat_count=1)

    psnr, ssim = evaluate(checkpoint.model, valid_ds)
    print('PSNR:%.3f, SSIM:%.3f' % (psnr, ssim))

    psnr_b = bilinear_upscale('../image_240', '../image_960', scale=scale)
    print('bilinear upscale PSNR:%.3f' % psnr_b)

    lr = load_image('../image_240/frame1500.jpg')
    sr = resolve_single(checkpoint.model, lr)
    plt.imshow(sr)
    plt.show()
示例#9
0
    def __evaluate(self, sess, model, name, data, idToTag, logger):
        evaluateResult = dict()
        logger.info("evaluate:{}".format(name))
        nerResults = model.evaluate(sess, data,
                                    idToTag)  # 预测结果,格式【原始字符 正确标注 预测标注结果】
        evalResult, evalResultDict = self.curDep.testNer(
            nerResults, self.FLAGS.resultPath)
        for line in evalResult:  #打印出来
            logger.info(line)

        wholeDict = evalResultDict['wholeClass']
        subClassDict = evalResultDict["subClass"]

        if name == 'dev':
            evaluateResult['TrainWhole'] = wholeDict
            evaluateResult['TrainSubclass'] = subClassDict
        if name == 'test':
            evaluateResult['DevWhole'] = wholeDict
            evaluateResult['DevSubclass'] = subClassDict

        f1 = evalResultDict["wholeClass"]['MacroF1']
        if name == "dev":
            bestDevF1 = model.bestDevF1.eval()
            if f1 > bestDevF1:
                tf.assign(model.bestDevF1,
                          f1).eval()  # tf.assign()更新赋值,把best更新为f1.
                logger.info("new best dev f1 score:{:>.3f}".format(f1))
            return f1 > bestDevF1, evaluateResult
        elif name == "test":
            bestTestF1 = model.bestTestF1.eval()
            if f1 > bestTestF1:
                tf.assign(model.bestTestF1, f1).eval()
                logger.info("new best test f1 score:{:>.3f}".format(f1))

            return f1 > bestTestF1, evaluateResult
示例#10
0
def train(model, training_data, callback=True, batch_size=256, epochs=10):

    (x_train, y_train), (x_test, y_test), mapping, nb_classes = training_data

    # convert class vectors to binary class matrices
    y_train = np_utils.to_categorical(y_train, nb_classes)
    y_test = np_utils.to_categorical(y_test, nb_classes)

    if callback == True:
        # Callback for analysis in TensorBoard
        tbCallBack = keras.callbacks.TensorBoard(log_dir='./Graph', histogram_freq=0, write_graph=True, write_images=True)

    model.fit(x_train, y_train,
              batch_size=batch_size,
              epochs=epochs,
              verbose=1,
              validation_data=(x_test, y_test),
              callbacks=[tbCallBack] if callback else None)

    score = model.evaluate(x_test, y_test, verbose=0)
    print('Test score:', score[0])
    print('Test accuracy:', score[1])

    # Offload model to file
    model_yaml = model.to_yaml()
    with open("bin/model.yaml", "w") as yaml_file:
        yaml_file.write(model_yaml)
    save_model(model, 'bin/model.h5')
示例#11
0
def main(data, model):
    print("Reading images...")
    x_train = data.read_images('./data/trainImage256.txt', TRAIN_DATA_SIZE)
    x_test = data.read_images('./data/testImage256.txt', TEST_DATA_SIZE)
    y_train = data.read_labels('./data/trainLABEL256.txt', TRAIN_DATA_SIZE)
    y_test = data.read_labels('./data/testLABEL256.txt', TEST_DATA_SIZE)
    
    print("Creating model...")
    model.create_model(multi_gpu=False)

    print("Now training...")
    history = model.training(x_train, y_train, x_test, y_test)
    accuracy = history.history["accuracy"]
    loss = history.history["loss"]
    eval = model.evaluate(x_test, y_test)
    
    print("accuracy = " + str(eval))
    model.save('./model.h5')

    if not os.path.exists('./result_keras'):
        os.mkdir('./result_keras')
    for i in range(TEST_DATA_SIZE):
        ret = model.predict(x_test[i, :, :, 0].reshape([1, IMG_SIZE, IMG_SIZE, 1]), 1)
        np.savetxt('./result_keras/' + str(i) + '.txt', ret[0, :, :, 0])
    
    with open("training_log.txt", "w") as f:
        for i in range(training_epochs):
            f.write(str(loss[i]) + "," + str(accuracy[i]) + "\n")
    ax1 = plt.subplot()
    ax1.plot(loss, color="blue")
    ax2 = ax1.twinx()
    ax2.plot(accuracy, color="orange")
    plt.show()
示例#12
0
            def after_run(self, run_context, run_values):
                if self._step % FLAGS.log_frequency == 0:
                    current_time = time.time()
                    duration = current_time - self._start_time
                    self._start_time = current_time

                    loss_value = run_values.results
                    examples_per_sec = FLAGS.log_frequency * FLAGS.batch_size / duration
                    sec_per_batch = float(duration / FLAGS.log_frequency)

                    if self._step % FLAGS.eval_frequency == 0:
                        precision = model.evaluate(run_context.session,
                                                   top_k_op,
                                                   FLAGS.num_valid_examples)
                        format_str = (
                            '%s: step %d, loss = %.2f (%.1f examples/sec; %.3f '
                            'sec/batch), precision = %.2f%%')
                        print(format_str %
                              (datetime.now(), self._step, loss_value,
                               examples_per_sec, sec_per_batch, precision))
                    else:
                        format_str = (
                            '%s: step %d, loss = %.2f (%.1f examples/sec; %.3f '
                            'sec/batch)')
                        print(format_str %
                              (datetime.now(), self._step, loss_value,
                               examples_per_sec, sec_per_batch))
示例#13
0
    def evaluate(self, model_pickle, test_data) -> dict:
        print("Evaluating model")
        model = pickle.loads(model_pickle)

        score = model_package.evaluate(model, test_data)

        return {"SMAPE": score}
示例#14
0
def inference_benchmark(validation_pipeline, dlrm, timer, splitter, FLAGS):
    if FLAGS.max_steps == -1:
        FLAGS.max_steps = 1000

    if FLAGS.saved_model_input_path:
        cast_dtype = tf.float16 if FLAGS.amp else tf.float32
    else:
        cast_dtype = None

    auc, test_loss, latencies = evaluate(validation_pipeline, dlrm,
                               timer, auc_thresholds=FLAGS.auc_thresholds,
                               data_parallel_splitter=splitter,
                               max_steps=FLAGS.max_steps, cast_dtype=cast_dtype)

    # don't benchmark the first few warmup steps
    latencies = latencies[10:]
    result_data = {
        'mean_inference_throughput': FLAGS.valid_batch_size / np.mean(latencies),
        'mean_inference_latency': np.mean(latencies)
    }

    for percentile in [90, 95, 99]:
        result_data[f'p{percentile}_inference_latency'] = np.percentile(latencies, percentile)
    result_data['auc'] = auc

    if hvd.rank() == 0:
        dllogger.log(data=result_data, step=tuple())
示例#15
0
 def check_and_log(self, learning, dataset, sz):
     if self.conditional_log is None: return
     if sz == 0 or sz >= self.conditional_log[0]:
         test_err = MModel.evaluate(learning.model, dataset.test_data)
         add_to_dict(self.err_log, sz,
                     [np.array([learning.cnt_labels, test_err])])
     if sz >= self.conditional_log[0]:
         self.conditional_log[0] *= self.conditional_log[1]
示例#16
0
def evaluateAndShowAttention(input_sentence, input_lang, output_lang, encoder1,
                             attn_decoder1):
    output_words, attentions = model.evaluate(encoder1, attn_decoder1,
                                              input_sentence, input_lang,
                                              output_lang)
    print('input =', input_sentence)
    print('output =', ' '.join(output_words))
    showAttention(input_sentence, output_words, attentions)
示例#17
0
def fine_tune(model: Union[DeepEosModel, str], vocab_path: Union[str, Path], cross_validation_set=None):
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    window_size = 6

    if type(model) is str:
        model = torch.load(str(model), map_location=device)

    # BIOfid fine-tuning
    biofid_train = EosDataset('data/bioFID_train_cleaned.txt', split_dev=False, window_size=window_size,
                              load_vocab=vocab_path, shuffle_input=False)
    biofid_test = EosDataset('data/bioFID_test.txt', split_dev=False, window_size=window_size, load_vocab=vocab_path,
                             shuffle_input=False)

    print("\nBIOfid test scores prior to fine-tuning")
    evaluate(model, biofid_test, device=device)
    print(flush=True)

    optimizer = optim.Adam(model.parameters(), lr=0.0001)
    train(model, biofid_train, biofid_test, base_path=Path('biofid_models/'),
          optimizer=optimizer, epochs=10, device=device)

    print("\nBIOfid test")
    evaluate(model, biofid_test, device=device)

    if cross_validation_set is not None:
        print("\nCross validation")
        evaluate(model, cross_validation_set, device=device)

    model_name = 'de.biofid'
    torch.save(model, Path('biofid_models/').joinpath(model_name + '.pt'))
示例#18
0
def aggregate():
    if len(selected_clients) <= 0:
        return {"success": False}

    print("Waiting for updates...")
    updates = []

    global version

    selected_clients_copy = selected_clients.copy()
    while len(selected_clients_copy) > 0:
        update = aggregator.recv_pyobj()

        print("Recieved update %d/%d" %
              (len(selected_clients_copy), len(selected_clients)))

        client_id = update["client_id"]
        model_version = update["version"]
        client_metrics = update["metrics"]

        if client_id in selected_clients_copy and model_version == version:
            print("Received update on version %s from client %s" %
                  (model_version, client_id))
            print("Metrics: %s" %
                  json.dumps(client_metrics, indent=4, sort_keys=True))
            updates.append(update)
            selected_clients_copy.remove(client_id)

    total_points = 0
    for update in updates:
        total_points += update["points"]

    print(total_points)

    weighted_avg = np.array(
        [np.zeros(layer.shape) for layer in model.get_weights()])

    for update in updates:
        points = update["points"]
        weights = update["weights"]
        weighted_avg += (points / total_points) * np.array(weights)

    model.set_weights(weighted_avg.tolist())

    version += 1
    print("Current version: %s" % version)
    print("Evaluating...")
    history = model.evaluate(x=x_test, y=y_test, batch_size=32)
    return {"success": True, "loss": history[0], "accuracy": history[1]}
示例#19
0
def main(args):

    print("Loading Dataset...")
    dataset = testset_dataloader(args.tokenizer, args)

    print("Dataset Loaded...")
    print(f"Dataset size: {len(dataset.dataset)}")

    pred = evaluate(args.model, dataset, args)

    result = pd.DataFrame(pred)
    result.index += 1
    result.to_csv(args.output_path, header=False)

    print(f"Results successfully saved at {args.output_path}")
示例#20
0
async def echo(message):

    # media = types.MediaGroup()

    WIDTH = message.photo[-1].width
    HEIGHT = message.photo[-1].height

    byteImgIO = BytesIO()
    await message.photo[-1].download(byteImgIO)
    byteImgIO.seek(0)
    byteImg = byteImgIO.read()

    dataBytesIO = BytesIO(byteImg)

    # im = Image.frombuffer('RGB', (WIDTH, HEIGHT), dataBytesIO.getvalue())
    im = np.array(Image.open(dataBytesIO))
    res_l = evaluate(im)
    res_l = [x for x in res_l if x not in ['<unk>', '<end>']]
    result = ' '.join(res_l)
    result_ru = translate(result)
    print('result: ', result, result_ru)

    try:
        r = requests.post('https://ttsmp3.com/makemp3_new.php',
                          data={
                              'msg': result_ru,
                              'lang': 'Maxim',
                              'source': 'ttsmp3'
                          },
                          headers={'User-agent': 'Mozilla/5.0'},
                          config={'verbose': sys.stderr})
        print(r)
        print(r.json())
        r1 = requests.get(r.json()['URL'])
        with open('./audio/audio.ogg', 'wb') as f:
            f.write(r1.content)
    except:
        var = gTTS(text=result_ru, lang='ru')
        var.save('./audio/audio.ogg')

    await message.answer_voice(types.InputFile('./audio/audio.ogg'))
    await message.answer("''" + result_ru + "''")
    await message.answer(__text)
def inference_benchmark(validation_pipeline, dlrm, timer, splitter, FLAGS):
    if FLAGS.max_steps == -1:
        FLAGS.max_steps = 1000

    _, _, latencies = evaluate(validation_pipeline,
                               dlrm,
                               timer,
                               auc_thresholds=None,
                               data_parallel_splitter=splitter,
                               max_steps=FLAGS.max_steps)

    # don't benchmark the first few warmup steps
    latencies = latencies[10:]
    result_data = {
        'mean_inference_throughput':
        FLAGS.valid_batch_size / np.mean(latencies),
        'mean_inference_latency': np.mean(latencies)
    }

    for percentile in [90, 95, 99]:
        result_data[f'p{percentile}_inference_latency'] = np.percentile(
            latencies, percentile)
    dllogger.log(data=result_data, step=tuple())
    def test(self):
        """Computing AUC for the unseen test set """
        # Load trained parameters
        G_path = os.path.join(self.model_save_path,
                              '{}_G.pth'.format(self.test_model))
        self.device = torch.device("cuda:0")
        self.G.load_state_dict(torch.load(G_path))
        self.G.eval()

        data_loader = self.img_data_loader
        # Create big error tensor for the test set.
        self.an_scores = torch.zeros(size=(len(data_loader.valid.dataset), ),
                                     dtype=torch.float32,
                                     device=self.device)
        self.gt_labels = torch.zeros(size=(len(data_loader.valid.dataset), ),
                                     dtype=torch.long,
                                     device=self.device)

        with torch.no_grad():
            for i, (real_x, org_c) in enumerate(data_loader.valid):
                real_x = self.to_var(real_x, volatile=True)
                fake_x, enc_feat, rec_feat = self.G(real_x)
                error = torch.mean(
                    torch.pow(
                        (enc_feat - rec_feat), 2), dim=(1, 2, 3)) + torch.mean(
                            torch.abs(real_x - fake_x), dim=(1, 2, 3))
                self.an_scores[i * self.batch_size:i * self.batch_size +
                               error.size(0)] = error.reshape(error.size(0))
                self.gt_labels[i * self.batch_size:i * self.batch_size +
                               error.size(0)] = org_c.reshape(error.size(0))
            self.an_scores = (self.an_scores - torch.min(self.an_scores)) / (
                torch.max(self.an_scores) - torch.min(self.an_scores))
            self.an_scores = self.an_scores.detach()
            self.an_scores = self.an_scores.cpu().numpy()
            self.gt_labels = self.gt_labels.detach()
            self.gt_labels = self.gt_labels.cpu().numpy()
            auc = evaluate(self.gt_labels, self.an_scores, metric='roc')
示例#23
0
import data
import model
import korali

k = korali.Engine()
e = korali.Experiment()

# Defining Concurrent Jobs
popSize = 512
if (len(sys.argv) > 1): popSize = int(sys.argv[1])

# Setting up the reference likelihood for the Bayesian Problem
e["Problem"]["Type"] = "Bayesian/Reference"
e["Problem"]["Likelihood Model"] = "Normal"
e["Problem"]["Reference Data"] = data.getReferenceData().tolist()
e["Problem"]["Computational Model"] = lambda koraliData: model.evaluate( koraliData, data.getReferencePoints(), korali.getMPIComm() )

e["Distributions"][0]["Name"] = "Uniform 0"
e["Distributions"][0]["Type"] = "Univariate/Uniform"
e["Distributions"][0]["Minimum"] = 0.2
e["Distributions"][0]["Maximum"] = 0.6

e["Distributions"][1]["Name"] = "Uniform 1"
e["Distributions"][1]["Type"] = "Univariate/Uniform"
e["Distributions"][1]["Minimum"] = 10.0
e["Distributions"][1]["Maximum"] = 40.0
 
e["Distributions"][2]["Name"] = "Uniform 2"
e["Distributions"][2]["Type"] = "Univariate/Uniform"
e["Distributions"][2]["Minimum"] = 1e-5
e["Distributions"][2]["Maximum"] = 2.0
示例#24
0
            'params':[par for par in to_be_trained if par in no_decay],
            'weight_decay':0
            }
        ]


    #optimizer = Adam(grouped_params, lr=LR)
    optimizer = AdamW(grouped_params, lr=LR)

    best_valid_accuracy = 0
    train_list, valid_list = [], []
    for epoch in range(EPOCHS):
        # tarin step over all batches
        model.train_step(network, train_data_loader, loss_function, optimizer, device)

        train_targets, train_outputs = model.evaluate(network, train_data_loader, device)
        valid_targets, valid_outputs = model.evaluate(network, valid_data_loader, device)

        train_accuracy = accuracy_score(train_targets, [int(i>0.5) for i in train_outputs])
        valid_accuracy = accuracy_score(valid_targets, [int(i>0.5) for i in valid_outputs])

        train_list.append(train_accuracy)
        valid_list.append(valid_accuracy)

        print("Train Accuracy", train_accuracy)
        print("Valid Accuracy", valid_accuracy)

        if valid_accuracy > best_valid_accuracy:
            best_valid_accuracy = valid_accuracy
            #torch.save(network.state_dict(), MODEL_PATH)
示例#25
0
文件: main.py 项目: flaska/tflow
import data
import model
import display

(train_images, train_labels), (test_images, test_labels) = data.get_data()

class_names = [
    'T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt',
    'Sneaker', 'Bag', 'Ankle boot'
]

# display.display_sample(train_images, class_names, train_labels)

model = model.get_model()
model.fit(train_images, train_labels, epochs=5)

test_loss, test_acc = model.evaluate(test_images, test_labels)

print('Test accuracy:', test_acc)

predictions = model.predict(test_images)

display.plot_results(predictions, test_images, test_labels, class_names)
示例#26
0
    stats.to_csv("data/wine_stats.csv", sep=',', encoding='utf-8')

    # Step 4: Training
    # Create a function that saves the model's weights
    cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath = model_name,
                                                     save_weights_only=True,
                                                     verbose=0, save_freq="epoch")
    # model.load_weights(model_name)
    model.fit(X_train, Y_train,
              batch_size     = batch_size,
              epochs         = epochs,
              validation_data= (X_valid,Y_valid),
              callbacks      = [cp_callback])

    # Step 6: Evaluation
    mae, mse = model.evaluate(X_valid, Y_valid, verbose = 2)
    print("Evaluation, MAE: {:2.4f}, MSE: {:2.4f}".format(mae, mse))

elif sys.argv[1] == "predict":
    # Step 3: Loads the weights
    model.load_weights(model_name)
    my_model = tf.keras.Sequential([model])

    # Step 4: Get Normalization values
    stats   = pd.read_csv("data/wine_stats.csv", sep = ',', header = 0)

    # Step 5: Prepare the input AND predict
    input = sys.argv[2].split(",")
    input = np.array([float(x) for x in input])
    input  = loader.normalize(input, stats).values
    print(input)
示例#27
0
 def evaluate(self, dataset):
     return evaluate(self.checkpoint.model, dataset)
示例#28
0
# create required directory
required_dirs = ["param", "result", "mnist"]
hoge.make_dir(required_dirs)

mnist_data = MNIST('./mnist/',
                   train=True,
                   download=True,
                   transform=transforms.ToTensor())
dataloader = DataLoader(mnist_data, batch_size=mini_batch_num, shuffle=True)

print("\n")
# train restart
if interrupt_flag:
    f = open("./param/tmp.pickle", mode="rb")
    init_epoch = pickle.load(f)
    model = model.GAN(dataloader, interrupting=True)
else:
    init_epoch = 1
    model = model.GAN(dataloader)
del dataloader

for epoch in range(init_epoch, epochs + 1):
    print("Epoch[%d/%d]:" % (epoch, epochs))
    model.study(epoch)
    model.evaluate()
    model.save_tmp_weight(epoch)
    model.eval_pic(epoch)
    model.output(epoch)
#model.output()
示例#29
0
            print('no model sess path specified. Can not preceed.')
            sys.exit(1)
        else:
            result_path = cfg['sess_path']

    valid_batch_data = get_batch_data(valid_text,
                                      word2id_dict=word2id_dict,
                                      batch_size=cfg['batch_size'],
                                      max_length=cfg['max_length'])

    pepfile_path = os.path.join(result_path, 'perplexity.txt')
    result_ptr = open(pepfile_path, 'w')

    print("start evaluate the language model")
    evaluate(sess_path=result_path,
             eva_data=valid_batch_data,
             result_ptr=result_ptr)
    print("evaluation phase completed")

if cfg['g']:
    if not cfg['t']:
        if len(cfg['sess_path']) == 0:
            print('no model sess path specified. Can not preceed.')
            sys.exit(1)
        else:
            result_path = cfg['sess_path']

    # get id2word_dict
    tf.reset_default_graph()
    contin_path = "data/sentences.continuation"
    contin_text = load_data(contin_path)
示例#30
0
	# Print best validation accuracy and epoch in valid_set
	max_val_acc, idx = max((val, idx) for (idx, val) in enumerate(history.history['val_acc']))
	print('Maximum accuracy at epoch', '{:d}'.format(idx + 1), '=', '{:.4f}'.format(max_val_acc))

	# plot the result
	import matplotlib.pyplot as plt

	plt.figure()
	plt.plot(history.epoch, history.history['acc'], label="acc")
	plt.plot(history.epoch, history.history['val_acc'], label="val_acc")
	plt.scatter(history.epoch, history.history['acc'], marker='*')
	plt.scatter(history.epoch, history.history['val_acc'])
	plt.legend(loc='lower right')
	plt.show()

	plt.figure()
	plt.plot(history.epoch, history.history['loss'], label="loss")
	plt.plot(history.epoch, history.history['val_loss'], label="val_loss")
	plt.scatter(history.epoch, history.history['loss'], marker='*')
	plt.scatter(history.epoch, history.history['val_loss'], marker='*')
	plt.legend(loc='lower right')
	plt.show()

	score, acc = model.evaluate(X_test, test_label, batch_size=BATCH_SIZE)
	print('Test score:', score)
	print('Test accuracy:', acc)

	predictions = model.predict(X_test)
	preditFval(predictions, test_label)
示例#31
0
import model


doc = {'dataset': {'path': 'data/sample1000/'},
       # 'dataset': {'path': 'data/full/'},
       'features': ['a', 'a1', 'a1-2', 'b', 'b1-3', 'd', 'd1'],
       'fm_param': {'n_iter': 350, 'stdev': .001, 'rank': 4},
       'seeds': [123, 345, 231, 500, 442, 873, 921, 111, 222],
       'output': {},
       'threshold': .84,
       'shift_features': ['a', 'b', 'd'],
       'max_shift': 5,
       'min_freq': [1, 5, 10, 20],
       'min_size': 10,
       'lower_proba': .7,
       'upper_proba': .9,
       'submission_path': 'data/full/submission/submission.txt'
       }

print model.evaluate(doc)
#model.submission(doc)
示例#32
0
文件: run.py 项目: thanhmaikmt/pjl
#try:
dxdt_guess=zeros(model.n_eq)
dxdt=zeros(model.n_eq)
x=zeros(model.n_eq)

for i in range(n_step):
   
    It=I_at_time(time+dt)
    
    flag=True
    cnt=0
    
    while True :
        
        err=model.evaluate(dxdt,x,dxdt_guess,It,dt)
        
        print cnt,"  Err=",err
        cnt+=1
        if err < tol or cnt>5:
            break
  
  
        
    dxdt[:]=dxdt_guess
    x[:]=x+dt*dxdt
    tt=copy.deepcopy(dxdt[model.n_node:model.n_node+n_plot_x])
    plot.drawX(0,xx,tt)
    
    times.append(time)
    jj.append(tt)
示例#33
0
def main(argv):
    hvd.init()
    validate_cmd_line_flags()
    init_logging(log_path=FLAGS.log_path, FLAGS=FLAGS)
    init_tf(FLAGS)

    train_pipeline, validation_pipeline, dataset_metadata, multi_gpu_metadata = create_input_pipelines(FLAGS)

    dlrm = Dlrm.load_model_if_path_exists(FLAGS.saved_model_input_path)

    if dlrm is None:
        if FLAGS.dummy_model:
            dlrm = DummyDlrm(FLAGS=FLAGS, dataset_metadata=dataset_metadata,
                             multi_gpu_metadata=multi_gpu_metadata)
        else:
            dlrm = Dlrm(FLAGS=FLAGS, dataset_metadata=dataset_metadata,
                        multi_gpu_metadata=multi_gpu_metadata)
            dlrm = dlrm.restore_checkpoint_if_path_exists(FLAGS.restore_checkpoint_path)

    if FLAGS.optimizer == 'sgd':
        embedding_optimizer = tf.keras.optimizers.SGD(learning_rate=FLAGS.learning_rate, momentum=0)
        if FLAGS.amp:
            embedding_optimizer = LossScaleOptimizer(embedding_optimizer,
                                                     initial_scale=FLAGS.loss_scale,
                                                     dynamic=False)
        mlp_optimizer = embedding_optimizer
        optimizers = [mlp_optimizer]

    elif FLAGS.optimizer == 'adam':
        embedding_optimizer = tfa.optimizers.LazyAdam(learning_rate=FLAGS.learning_rate)
        mlp_optimizer = tf.keras.optimizers.Adam(learning_rate=FLAGS.learning_rate)
        if FLAGS.amp:
            embedding_optimizer = LossScaleOptimizer(embedding_optimizer,
                                                     initial_scale=FLAGS.loss_scale,
                                                     dynamic=False)
            mlp_optimizer = LossScaleOptimizer(mlp_optimizer,
                                               initial_scale=FLAGS.loss_scale,
                                               dynamic=False)
        optimizers = [mlp_optimizer, embedding_optimizer]

    scheduler = LearningRateScheduler(optimizers,
                                      warmup_steps=FLAGS.warmup_steps,
                                      base_lr=FLAGS.learning_rate,
                                      decay_start_step=FLAGS.decay_start_step,
                                      decay_steps=FLAGS.decay_steps)

    timer = IterTimer(train_batch_size=FLAGS.batch_size, test_batch_size=FLAGS.valid_batch_size,
                      optimizer=embedding_optimizer, print_freq=FLAGS.print_freq, enabled=hvd.rank() == 0)

    splitter = DataParallelSplitter(batch_size=FLAGS.batch_size)

    if FLAGS.mode == 'inference':
        inference_benchmark(validation_pipeline, dlrm, timer, splitter, FLAGS)
        return
    elif FLAGS.mode == 'deploy':
        dlrm.save_model_if_path_exists(FLAGS.saved_model_output_path,
                                       save_input_signature=FLAGS.save_input_signature)
        print('deployed to: ', FLAGS.saved_model_output_path)
        return

    elif FLAGS.mode == 'eval':
        test_auc, test_loss, _ = evaluate(validation_pipeline, dlrm,
                                          timer, auc_thresholds=FLAGS.auc_thresholds,
                                          data_parallel_splitter=splitter)
        if hvd.rank() == 0:
            dllogger.log(data=dict(auc=test_auc, test_loss=test_loss), step=tuple())
        return

    eval_points = compute_eval_points(train_batches=len(train_pipeline),
                                      evals_per_epoch=FLAGS.evals_per_epoch)

    trainer = DlrmTrainer(dlrm, embedding_optimizer=embedding_optimizer,
                          mlp_optimizer=mlp_optimizer, amp=FLAGS.amp,
                          lr_scheduler=scheduler, dp_splitter=splitter,
                          data_parallel_bottom_mlp=FLAGS.data_parallel_bottom_mlp,
                          pipe=train_pipeline, cpu=FLAGS.cpu)

    best_auc = 0
    train_begin = time.time()
    for epoch in range(FLAGS.epochs):
        for step in range(len(train_pipeline)):
            if step == FLAGS.profiler_start_step and hvd.rank() == FLAGS.profiled_rank:
                tf.profiler.experimental.start('logdir')

            if FLAGS.profiler_start_step and step == FLAGS.profiler_start_step + 100 and hvd.rank() == FLAGS.profiled_rank:
                tf.profiler.experimental.stop()

            loss = trainer.train_step()

            timer.step_train(loss=loss)

            if FLAGS.max_steps != -1 and step > FLAGS.max_steps:
                dist_print(f'Max steps of {FLAGS.max_steps} reached, exiting')
                break

            if step in eval_points:
                test_auc, test_loss, _ = evaluate(validation_pipeline, dlrm,
                                                  timer, FLAGS.auc_thresholds,
                                                  data_parallel_splitter=splitter)
                dist_print(f'Evaluation completed, AUC: {test_auc:.6f}, test_loss: {test_loss:.6f}')
                timer.test_idx = 0
                best_auc = max(best_auc, test_auc)

    elapsed = time.time() - train_begin
    dlrm.save_checkpoint_if_path_exists(FLAGS.save_checkpoint_path)
    dlrm.save_model_if_path_exists(FLAGS.saved_model_output_path,
                                   save_input_signature=FLAGS.save_input_signature)

    if hvd.rank() == 0:
        dist_print(f'Training run completed, elapsed: {elapsed:.0f} [s]')
        results = {
            'throughput': FLAGS.batch_size / timer.mean_train_time(),
            'mean_step_time_ms': timer.mean_train_time() * 1000,
            'auc': best_auc
        }
        dllogger.log(data=results, step=tuple())
示例#34
0
def main(args):
    model_id = build_model_id(args)
    model_path = build_model_path(args, model_id)
    setup_model_dir(args, model_path)
    sys.stdout, sys.stderr = setup_logging(args, model_path)

    x_train, y_train = load_model_data(args.train_file,
            args.data_name, args.target_name)
    x_validation, y_validation = load_model_data(
            args.validation_file,
            args.data_name, args.target_name)

    rng = np.random.RandomState(args.seed)

    if args.n_classes > -1:
        n_classes = args.n_classes
    else:
        n_classes = max(y_train)+1

    n_classes, target_names, class_weight = load_target_data(args, n_classes)

    if len(class_weight) == 0:
        n_samples = len(y_train)
        print('n_samples', n_samples)
        print('classes', range(n_classes))
        print('weights', n_samples / (n_classes * np.bincount(y_train)))
        class_weight = dict(zip(range(n_classes),
            n_samples / (n_classes * np.bincount(y_train))))
    print('class_weight', class_weight)

    logging.debug("n_classes {0} min {1} max {2}".format(
        n_classes, min(y_train), max(y_train)))

    y_train_one_hot = np_utils.to_categorical(y_train, n_classes)
    y_validation_one_hot = np_utils.to_categorical(y_validation, n_classes)

    logging.debug("y_train_one_hot " + str(y_train_one_hot.shape))
    logging.debug("x_train " + str(x_train.shape))

    min_vocab_index = np.min(x_train)
    max_vocab_index = np.max(x_train)
    logging.debug("min vocab index {0} max vocab index {1}".format(
        min_vocab_index, max_vocab_index))

    json_cfg = load_model_json(args, x_train, n_classes)

    logging.debug("loading model")

    sys.path.append(args.model_dir)
    import model
    from model import build_model

    #######################################################################      
    # Subsetting
    #######################################################################      
    if args.subsetting_function:
        subsetter = getattr(model, args.subsetting_function)
    else:
        subsetter = None

    def take_subset(subsetter, path, x, y, y_one_hot, n):
        if subsetter is None:
            return x[0:n], y[0:n], y_one_hot[0:n]
        else:
            mask = subsetter(path)
            idx = np.where(mask)[0]
            idx = idx[0:n]
        return x[idx], y[idx], y_one_hot[idx]

    x_train, y_train, y_train_one_hot = take_subset(
            subsetter, args.train_file,
            x_train, y_train, y_train_one_hot,
            n=args.n_train)

    x_validation, y_validation, y_validation_one_hot = take_subset(
            subsetter, args.validation_file,
            x_validation, y_validation, y_validation_one_hot,
            n=args.n_validation)

    #######################################################################      
    # Preprocessing
    #######################################################################      
    if args.preprocessing_class:
        preprocessor = getattr(model, args.preprocessing_class)(seed=args.seed)
    else:
        preprocessor = modeling.preprocess.NullPreprocessor()

    logging.debug("y_train_one_hot " + str(y_train_one_hot.shape))
    logging.debug("x_train " + str(x_train.shape))

    model_cfg = ModelConfig(**json_cfg)
    logging.info("model_cfg " + str(model_cfg))
    model = build_model(model_cfg)
    setattr(model, 'stop_training', False)

    logging.info('model has {n_params} parameters'.format(
        n_params=count_parameters(model)))

    if len(args.extra_train_file) > 1:
        callbacks = keras.callbacks.CallbackList()
    else:
        callbacks = []

    save_model_info(args, model_path, model_cfg)

    if not args.no_save:
        if args.save_all_checkpoints:
            filepath = model_path + '/model-{epoch:04d}.h5'
        else:
            filepath = model_path + '/model.h5'
        callbacks.append(ModelCheckpoint(
            filepath=filepath,
            verbose=1,
            save_best_only=not args.save_every_epoch))

    callback_logger = logging.info if args.log else callable_print

    if args.n_epochs < sys.maxsize:
        # Number of epochs overrides patience.  If the number of epochs
        # is specified on the command line, the model is trained for
        # exactly that number; otherwise, the model is trained with
        # early stopping using the patience specified in the model 
        # configuration.
        callbacks.append(EarlyStopping(
            monitor='val_loss', patience=model_cfg.patience, verbose=1))

    if args.classification_report:
        cr = ClassificationReport(x_validation, y_validation,
                callback_logger,
                target_names=target_names)
        callbacks.append(cr)

    if model_cfg.optimizer == 'SGD':
        callbacks.append(SingleStepLearningRateSchedule(patience=10))

    if len(args.extra_train_file) > 1:
        args.extra_train_file.append(args.train_file)
        logging.info("Using the following files for training: " +
                ','.join(args.extra_train_file))

        train_file_iter = itertools.cycle(args.extra_train_file)
        current_train = args.train_file

        callbacks._set_model(model)
        callbacks.on_train_begin(logs={})

        epoch = batch = 0

        while True:
            x_train, y_train_one_hot = preprocessor.fit_transform(
                    x_train, y_train_one_hot)
            x_validation, y_validation_one_hot = preprocessor.transform(
                    x_validation, y_validation_one_hot)

            iteration = batch % len(args.extra_train_file)

            logging.info("epoch {epoch} iteration {iteration} - training with {train_file}".format(
                    epoch=epoch, iteration=iteration, train_file=current_train))
            callbacks.on_epoch_begin(epoch, logs={})

            n_train = x_train.shape[0]

            callbacks.on_batch_begin(batch, logs={'size': n_train})

            index_array = np.arange(n_train)
            if args.shuffle:
                rng.shuffle(index_array)

            batches = keras.models.make_batches(n_train, model_cfg.batch_size)
            logging.info("epoch {epoch} iteration {iteration} - starting {n_batches} batches".format(
                    epoch=epoch, iteration=iteration, n_batches=len(batches)))

            avg_train_loss = avg_train_accuracy = 0.
            for batch_index, (batch_start, batch_end) in enumerate(batches):
                batch_ids = index_array[batch_start:batch_end]

                if isinstance(model, keras.models.Graph):
                    data = {
                            'input': x_train[batch_ids],
                            'output': y_train_one_hot[batch_ids]
                            }
                    train_loss = model.train_on_batch(data, class_weight=class_weight)
                    train_accuracy = 0.
                else:
                    train_loss, train_accuracy = model.train_on_batch(
                            x_train[batch_ids], y_train_one_hot[batch_ids],
                            accuracy=True, class_weight=class_weight)

                batch_end_logs = {'loss': train_loss, 'accuracy': train_accuracy}

                avg_train_loss = (avg_train_loss * batch_index + train_loss)/(batch_index + 1)
                avg_train_accuracy = (avg_train_accuracy * batch_index + train_accuracy)/(batch_index + 1)

                callbacks.on_batch_end(batch,
                        logs={'loss': train_loss, 'accuracy': train_accuracy})

            logging.info("epoch {epoch} iteration {iteration} - finished {n_batches} batches".format(
                    epoch=epoch, iteration=iteration, n_batches=len(batches)))

            logging.info("epoch {epoch} iteration {iteration} - loss: {loss} - acc: {acc}".format(
                    epoch=epoch, iteration=iteration, loss=avg_train_loss, acc=avg_train_accuracy))

            batch += 1

            # Validation frequency (this if-block) doesn't necessarily
            # occur in the same iteration as beginning of an epoch
            # (next if-block), so model.evaluate appears twice here.
            kwargs = { 'verbose': 0 if args.log else 1 }
            pargs = []
            validation_data = {}
            if isinstance(model, keras.models.Graph):
                validation_data = {
                        'input': x_validation,
                        'output': y_validation_one_hot
                        }
                pargs = [validation_data]
            else:
                pargs = [x_validation, y_validation_one_hot]
                kwargs['show_accuracy'] = True

            if (iteration + 1) % args.validation_freq == 0:
                if isinstance(model, keras.models.Graph):
                    val_loss = model.evaluate(*pargs, **kwargs)
                    y_hat = model.predict(validation_data)
                    val_acc = accuracy_score(y_validation, np.argmax(y_hat['output'], axis=1))
                else:
                    val_loss, val_acc = model.evaluate(
                            *pargs, **kwargs)
                logging.info("epoch {epoch} iteration {iteration} - val_loss: {val_loss} - val_acc: {val_acc}".format(
                        epoch=epoch, iteration=iteration, val_loss=val_loss, val_acc=val_acc))
                epoch_end_logs = {'iteration': iteration, 'val_loss': val_loss, 'val_acc': val_acc}
                callbacks.on_epoch_end(epoch, epoch_end_logs)

            if batch % len(args.extra_train_file) == 0:
                if isinstance(model, keras.models.Graph):
                    val_loss = model.evaluate(*pargs, **kwargs)
                    y_hat = model.predict(validation_data)
                    val_acc = accuracy_score(y_validation, np.argmax(y_hat['output'], axis=1))
                else:
                    val_loss, val_acc = model.evaluate(
                            *pargs, **kwargs)
                logging.info("epoch {epoch} iteration {iteration} - val_loss: {val_loss} - val_acc: {val_acc}".format(
                        epoch=epoch, iteration=iteration, val_loss=val_loss, val_acc=val_acc))
                epoch_end_logs = {'iteration': iteration, 'val_loss': val_loss, 'val_acc': val_acc}
                epoch += 1
                callbacks.on_epoch_end(epoch, epoch_end_logs)

            if model.stop_training:
                logging.info("epoch {epoch} iteration {iteration} - done training".format(
                    epoch=epoch, iteration=iteration))
                break

            current_train = next(train_file_iter)
            x_train, y_train = load_model_data(current_train,
                    args.data_name, args.target_name)
            y_train_one_hot = np_utils.to_categorical(y_train, n_classes)

            if epoch > args.n_epochs:
                break

        callbacks.on_train_end(logs={})
    else:
        x_train, y_train_one_hot = preprocessor.fit_transform(
                x_train, y_train_one_hot)
        x_validation, y_validation_one_hot = preprocessor.transform(
                x_validation, y_validation_one_hot)
        if isinstance(model, keras.models.Graph):
            data = {
                    'input': x_train,
                    'output': y_train_one_hot
                    }
            validation_data = {
                    'input': x_validation,
                    'output': y_validation_one_hot
                    }
            model.fit(data,
                shuffle=args.shuffle,
                nb_epoch=args.n_epochs,
                batch_size=model_cfg.batch_size,
                validation_data=validation_data,
                callbacks=callbacks,
                class_weight=class_weight,
                verbose=2 if args.log else 1)
            y_hat = model.predict(validation_data)
            print('val_acc %.04f' % 
                    accuracy_score(y_validation, np.argmax(y_hat['output'], axis=1)))
        else:
            model.fit(x_train, y_train_one_hot,
                shuffle=args.shuffle,
                nb_epoch=args.n_epochs,
                batch_size=model_cfg.batch_size,
                show_accuracy=True,
                validation_data=(x_validation, y_validation_one_hot),
                callbacks=callbacks,
                class_weight=class_weight,
                verbose=2 if args.log else 1)