def train(data_dir, arch, hidden_units, output_size, dropout, lr, epochs, gpu,
          checkpoint):

    print(
        'Dir: {},\t Arch:{},\t HiddenUints: {},\t lr: {},\t Epochs: {},\t gpu: {}\n'
        .format(data_dir, arch, hidden_units, lr, epochs, gpu))

    print('Loading Images from Directory...')
    trainloader, validloader, testloader, class_to_idx = get_loaders(data_dir)
    print('Images Loaded.\n')

    print('Building the Model...')
    model, criterion, optimizer = build_model(arch, hidden_units, output_size,
                                              dropout, lr)
    print('Model Built.\n')

    print('Beggining the Training...')
    model, optimizer = training(model, trainloader, validloader, epochs, 20,
                                criterion, optimizer, gpu)
    print('Training Done.\n')

    if checkpoint:
        print('Saving the Checkpoint...')
        save_checkpoint(checkpoint, model, optimizer, arch,
                        model.classifier[0].in_features, output_size,
                        hidden_units, dropout, class_to_idx, epochs, lr)
        print('Done.')
Example #2
0
    def train(self):
        """
        Train MemN2N model using training data for tasks.
        """
        np.random.seed(42)  # for reproducing
        assert self.data_dir is not None, "data_dir is not specified."
        print("Reading data from %s ..." % self.data_dir)

        # Parse training data
        train_data_path = glob.glob('%s/qa*_*_train.txt' % self.data_dir)
        dictionary = {"nil": 0}
        train_story, train_questions, train_qstory = parse_babi_task(train_data_path, dictionary, False)

        # Parse test data just to expand the dictionary so that it covers all words in the test data too
        test_data_path = glob.glob('%s/qa*_*_test.txt' % self.data_dir)
        parse_babi_task(test_data_path, dictionary, False)

        # Get reversed dictionary mapping index to word
        self.reversed_dict = dict((ix, w) for w, ix in dictionary.items())

        # Construct model
        self.general_config = BabiConfigJoint(train_story, train_questions, dictionary)
        self.memory, self.model, self.loss = build_model(self.general_config)

        # Train model
        if self.general_config.linear_start:
            train_linear_start(train_story, train_questions, train_qstory,
                               self.memory, self.model, self.loss, self.general_config)
        else:
            train(train_story, train_questions, train_qstory,
                  self.memory, self.model, self.loss, self.general_config)

        # Save model
        self.save_model()
def run_task(data_dir, task_id):
    """
    Train and test for each task
    """
    print("Train and test for task %d ..." % task_id)

    # Parse data
    train_files = glob.glob('%s/qa%d_*_train.txt' % (data_dir, task_id))
    test_files = glob.glob('%s/qa%d_*_test.txt' % (data_dir, task_id))

    dictionary = {"nil": 0}
    train_story, train_questions, train_qstory = parse_babi_task(
        train_files, dictionary, False)
    test_story, test_questions, test_qstory = parse_babi_task(
        test_files, dictionary, False)

    general_config = BabiConfig(train_story, train_questions, dictionary)

    memory, model, loss = build_model(general_config)

    if general_config.linear_start:
        train_linear_start(train_story, train_questions, train_qstory, memory,
                           model, loss, general_config)
    else:
        train(train_story, train_questions, train_qstory, memory, model, loss,
              general_config)

    test(test_story, test_questions, test_qstory, memory, model, loss,
         general_config)
Example #4
0
    def train(self):
        """
        Train MemN2N model using training data for tasks.
        """
        np.random.seed(42)  # for reproducing
        assert self.data_dir is not None, "data_dir is not specified."
        print("Reading data from %s ..." % self.data_dir)

        # Parse training data
        train_data_path = glob.glob('%s/qa*_*_train.txt' % self.data_dir)
        dictionary = {"nil": 0}
        train_story, train_questions, train_qstory = parse_babi_task(train_data_path, dictionary, False)

        # Parse test data just to expand the dictionary so that it covers all words in the test data too
        test_data_path = glob.glob('%s/qa*_*_test.txt' % self.data_dir)
        parse_babi_task(test_data_path, dictionary, False)

        # Get reversed dictionary mapping index to word
        self.reversed_dict = dict((ix, w) for w, ix in dictionary.items())

        # Construct model
        self.general_config = BabiConfigJoint(train_story, train_questions, dictionary)
        self.memory, self.model, self.loss = build_model(self.general_config)

        # Train model
        if self.general_config.linear_start:
            train_linear_start(train_story, train_questions, train_qstory,
                               self.memory, self.model, self.loss, self.general_config)
        else:
            train(train_story, train_questions, train_qstory,
                  self.memory, self.model, self.loss, self.general_config)

        # Save model
        self.save_model()
def main(training_from_scratch, args):

    if (training_from_scratch):

        text = open(args.filename, 'rb').read().decode(encoding='utf-8')
        text, char2idx, idx2char = preprocessing(
            text, args.checkpoint_dir,
            args.minocc)  # note that we are replacing the text here

        vocab_size = len(idx2char)
        config = Config(vocab_size, args.epochs)

        model = build_model(config)
    else:

        model = tf.keras.models.load_model(args.checkpoint)
        char2idx = unpickle(args.checkpoint_dir, 'char2idx')
        idx2char = unpickle(args.checkpoint_dir, 'idx2char')
        text = unpickle(args.checkpoint_dir, 'dataset')

        vocab_size = len(idx2char)
        config = Config(vocab_size, args.epochs, args.initepochs)

    text_as_int = np.array([char2idx[c] for c in text
                            ])  # works because text is a list of words
    train_model(args.checkpoint_dir, text_as_int, model, config)
Example #6
0
def train(max_eps=6000):
    env = gym.make('CartPole-v0')
    eval_env = gym.make('CartPole-v0')
    state_space_size = env.observation_space.shape[0]
    action_space_size = env.action_space.n
    print('action space size is {0}, state space size is {1}'.format(
        action_space_size, state_space_size))
    model = build_model(state_space_size, action_space_size)
    model.summary()
    gamma = 0.9
    optimizer = tf.optimizers.Adam(learning_rate=0.001)
    best_score = 0.0
    for eps in range(max_eps):
        done = False
        state = env.reset()
        rewards = []
        action_probs = []
        actions = []
        with tf.GradientTape() as tape:
            while not done:
                state_tensor = tf.convert_to_tensor([state], dtype=tf.float32)
                action_prob = model(state_tensor)
                action = sample_action(action_prob, action_space_size)
                next_state, reward, done, _ = env.step(action)
                rewards.append(reward)
                action_probs.append(action_prob[0])
                actions.append(action)
                state = next_state
            discounted_rewards = compute_discounted_rewards(rewards, gamma)
            normalized_discounted_rewards = tf.convert_to_tensor(
                normalize_discounted_rewards(discounted_rewards),
                dtype=tf.float32)
            probs = tf.stack(action_probs)
            clipped_probs = tf.clip_by_value(probs, 1e-8, 1.0 - 1e-8)
            onehot_actions = tf.one_hot(actions,
                                        action_space_size,
                                        dtype=tf.float32)
            log_likelihood = tf.multiply(onehot_actions,
                                         tf.math.log(clipped_probs))
            loss = tf.math.reduce_sum(
                tf.multiply(
                    -log_likelihood,
                    tf.expand_dims(normalized_discounted_rewards, axis=1)))
        policy_gradients = tape.gradient(loss, model.trainable_weights)
        optimizer.apply_gradients(
            zip(policy_gradients, model.trainable_weights))
        score = eval(model, eval_env, 10, action_space_size)
        if score >= best_score:
            best_score = score
            tf.saved_model.save(model, './best_model')
        print('Finished episode {0}/{1} with score {2}'.format(
            eps, max_eps, score))
    env.close()
    eval_env.close()
def run_joint_tasks(data_dir):
    """
    Train and test for all tasks but the trained model is built using training data from all tasks.
    """
    print("Jointly train and test for all tasks ...")
    tasks = range(20)

    # Parse training data
    train_data_path = []
    for t in tasks:
        train_data_path += glob.glob('%s/qa%d_*_train.txt' % (data_dir, t + 1))

    dictionary = {"nil": 0}
    train_story, train_questions, train_qstory = parse_babi_task(
        train_data_path, dictionary, False)

    # Parse test data for each task so that the dictionary covers all words before training
    for t in tasks:
        test_data_path = glob.glob('%s/qa%d_*_test.txt' % (data_dir, t + 1))
        parse_babi_task(test_data_path, dictionary,
                        False)  # ignore output for now

    general_config = BabiConfigJoint(train_story, train_questions, dictionary)
    memory, model, loss = build_model(general_config)

    if general_config.linear_start:
        train_linear_start(train_story, train_questions, train_qstory, memory,
                           model, loss, general_config)
    else:
        train(train_story, train_questions, train_qstory, memory, model, loss,
              general_config)

    # Test on each task
    for t in tasks:
        print("Testing for task %d ..." % (t + 1))
        test_data_path = glob.glob('%s/qa%d_*_test.txt' % (data_dir, t + 1))
        dc = len(dictionary)
        test_story, test_questions, test_qstory = parse_babi_task(
            test_data_path, dictionary, False)
        assert dc == len(
            dictionary
        )  # make sure that the dictionary already covers all words

        test(test_story, test_questions, test_qstory, memory, model, loss,
             general_config)
def main(training_from_scratch, args):

    ### opening the file ###
    text = open(args.filename, 'rb').read().decode(encoding='utf-8')
    vocab_size = len(set(text))

    config = Config(vocab_size, args.epochs, args.initepochs)

    ### looking at shit ###
    print('Length of text: {} characters'.format(len(text)))
    print(text[:250])

    ### creating vocab, converting text to long integer sequence ###
    char2idx, idx2char = create_vocab_from_file(text, args.checkpoint_dir)
    text_as_int = np.array([char2idx[c] for c in text])

    if (training_from_scratch):
        model = build_model(config)

    else:
        model = tf.keras.models.load_model(args.checkpoint)

    train_model(args.checkpoint_dir, text_as_int, model, config)
def run_joint_tasks(data_dir):
    """
    Train and test for all tasks but the trained model is built using training data from all tasks.
    """
    print("Jointly train and test for all tasks ...")
    tasks = range(20)

    # Parse training data
    train_data_path = []
    for t in tasks:
        train_data_path += glob.glob('%s/qa%d_*_train.txt' % (data_dir, t + 1))

    dictionary = {"nil": 0}
    train_story, train_questions, train_qstory = parse_babi_task(train_data_path, dictionary, False)

    # Parse test data for each task so that the dictionary covers all words before training
    for t in tasks:
        test_data_path = glob.glob('%s/qa%d_*_test.txt' % (data_dir, t + 1))
        parse_babi_task(test_data_path, dictionary, False)  # ignore output for now

    general_config = BabiConfigJoint(train_story, train_questions, dictionary)
    memory, model, loss = build_model(general_config)

    if general_config.linear_start:
        train_linear_start(train_story, train_questions, train_qstory, memory, model, loss, general_config)
    else:
        train(train_story, train_questions, train_qstory, memory, model, loss, general_config)

    # Test on each task
    for t in tasks:
        print("Testing for task %d ..." % (t + 1))
        test_data_path = glob.glob('%s/qa%d_*_test.txt' % (data_dir, t + 1))
        dc = len(dictionary)
        test_story, test_questions, test_qstory = parse_babi_task(test_data_path, dictionary, False)
        assert dc == len(dictionary)  # make sure that the dictionary already covers all words

        test(test_story, test_questions, test_qstory, memory, model, loss, general_config)
def run_tableQA(data_path, model_file):
    """
    Train and test for table QA
    """

    # Parse data
    train_files = glob.glob(data_path.format('train'))
    test_files = glob.glob(data_path.format('test'))
    # SV: init dict with pre-trained vectors, e.g. from fastText
    # dictionary = fasttext.load_model(EMBEDDINGS_MODEL_PATH)
    dictionary = {"nil": 0}
    train_story, train_questions, train_qstory = parse_babi_task(
        train_files, dictionary, False)
    test_story, test_questions, test_qstory = parse_babi_task(
        test_files, dictionary, False)
    # print test_questions
    print 'Dictionary:', len(dictionary)
    general_config = BabiConfig(train_story, train_questions, dictionary)

    memory, model, loss = build_model(general_config)

    if general_config.linear_start:
        train_linear_start(train_story, train_questions, train_qstory, memory,
                           model, loss, general_config)
    else:
        train(train_story, train_questions, train_qstory, memory, model, loss,
              general_config)

    test(test_story, test_questions, test_qstory, memory, model, loss,
         general_config)

    # save_model
    with gzip.open(model_file, "wb") as f:
        print("Saving model to file %s ..." % model_file)
        reversed_dict = dict((ix, w) for w, ix in dictionary.items())
        pickle.dump((reversed_dict, memory, model, loss, general_config), f)
def run_task(data_dir, task_id):
    """
    Train and test for each task
    """
    print("Train and test for task %d ..." % task_id)

    # Parse data
    train_files = glob.glob('%s/qa%d_*_train.txt' % (data_dir, task_id))
    test_files = glob.glob('%s/qa%d_*_test.txt' % (data_dir, task_id))

    dictionary = {"nil": 0}
    train_story, train_questions, train_qstory = parse_babi_task(train_files, dictionary, False)
    test_story, test_questions, test_qstory = parse_babi_task(test_files, dictionary, False)

    general_config = BabiConfig(train_story, train_questions, dictionary)

    memory, model, loss = build_model(general_config)

    if general_config.linear_start:
        train_linear_start(train_story, train_questions, train_qstory, memory, model, loss, general_config)
    else:
        train(train_story, train_questions, train_qstory, memory, model, loss, general_config)

    test(test_story, test_questions, test_qstory, memory, model, loss, general_config)
Example #12
0
n_mb = None
rpn_rois_per_img = 256  # number of rois to sample to train rpn
frcn_rois_per_img = 128  # number of rois to sample to train frcn

# setup backend
be = gen_backend(**extract_valid_args(args, gen_backend))
be.enable_winograd = 4  # default to winograd 4 for fast autotune

year = '2007'

train_set = PASCAL('trainval', year, path=args.data_dir, n_mb=n_mb,
                   rpn_rois_per_img=rpn_rois_per_img, frcn_rois_per_img=frcn_rois_per_img,
                   add_flipped=True, shuffle=True, rebuild_cache=True, subset_pct=args.subset_pct)

# build the Faster-RCNN model
model = util.build_model(train_set, frcn_rois_per_img, inference=False)

# set up cost different branches, respectively
weights = 1.0 / (rpn_rois_per_img)
roi_w = 1.0 / (frcn_rois_per_img)

frcn_tree_cost = Multicost(costs=[GeneralizedCostMask(costfunc=CrossEntropyMulti(), weights=roi_w),
                                  GeneralizedCostMask(costfunc=SmoothL1Loss(), weights=roi_w)
                                  ], weights=[1, 1])

cost = Multicost(costs=[GeneralizedCostMask(costfunc=CrossEntropyMulti(), weights=weights),
                        GeneralizedCostMask(costfunc=SmoothL1Loss(sigma=3.0), weights=weights),
                        frcn_tree_cost,
                        ],
                 weights=[1, 1, 1])
Example #13
0
 def modeling(self):
     self.embedding()
     model = build_model(len(self.word2int), 300, self.embeddingMatrix,
                         self.wordLimit)
     model.load_weights("./Microsoft_model.h5")
     return model
 def _load_model(self, file_path):
     self.network = build_model(self.network_config, num_classes=10)
     with open(file_path, "rb") as f_:
         self.network.load_state_dict(torch.load(f_))
Example #15
0
    def train_and_test(self, seed=None):
        """
        Train MemN2N model using training data for tasks.
        """
        if seed is None:
            np.random.seed(42)  # for reproducing
        else:
            np.random.seed(seed)
        train_data_arg = None
        test_data_arg = None
        if type(self.data_dir) is tuple:
            assert self.data_dir[
                0] is not None, "training data_dir is not specified."
            assert self.data_dir[
                1] is not None, "test data_dir is not specified."
            print("Reading training data from %s ..." % self.data_dir[0])
            print("Reading test data from %s ..." % self.data_dir[1])
            train_data_arg = '%s/qa*_train.txt' % self.data_dir[0]
            test_data_arg = '%s/qa*_valid.txt' % self.data_dir[1]
        else:
            assert self.data_dir is not None, "data_dir is not specified."
            print("Reading data from %s ..." % self.data_dir)
            train_data_arg = '%s/qa*_*_train.txt' % self.data_dir
            test_data_arg = '%s/qa*_*_test.txt' % self.data_dir
        assert train_data_arg is not None and test_data_arg is not None
        # Parse training data
        train_data_path = glob.glob(train_data_arg)
        dictionary = {"nil": 0}
        train_story, train_questions, train_qstory = parse_babi_task(
            train_data_path, dictionary, False)

        # Parse test data just to expand the dictionary so that it covers all words in the test data too
        test_data_path = glob.glob(test_data_arg)
        test_story, test_questions, test_qstory = parse_babi_task(
            test_data_path, dictionary, False)

        # Get reversed dictionary mapping index to word
        self.reversed_dict = dict((ix, w) for w, ix in dictionary.items())

        # Construct model
        self.general_config = BabiConfigJoint(train_story, train_questions,
                                              dictionary)

        #check for config switches format [initial learning rate, linear start option, hops]
        if self.config_switches is not None:
            self.general_config.train_config[
                "init_lrate"] = self.config_switches[0]

            #linear start option is passed to babi config constructor function so no need to set here

            # want equal of number of epochs for linear start and non linear start runs
            if self.general_config.linear_start is True:
                self.general_config.nepochs = 40
                self.general_config.ls_nepochs = 20
            else:
                self.general_config.nepochs = 60

            self.general_config.nhops = self.config_switches[2]

        self.memory, self.model, self.loss = build_model(self.general_config)

        # Train model
        train_val_results = []
        if self.general_config.linear_start:
            train_val_results += train_linear_start(train_story,
                                                    train_questions,
                                                    train_qstory, self.memory,
                                                    self.model, self.loss,
                                                    self.general_config)
        else:
            train_val_results += train(train_story, train_questions,
                                       train_qstory, self.memory, self.model,
                                       self.loss, self.general_config)

        test_error = test(test_story, test_questions, test_qstory, self.memory,
                          self.model, self.loss, self.general_config)

        model_test_accuracy = (1.0 - test_error) * 100.0

        train_val_file = self.model_file + 'train_val_accuracy.csv'
        with open(train_val_file, 'w') as f:
            f.write('epoch, TrainAccuracy, ValAccuracy\n')
            epoch = 1
            for item in train_val_results:
                line = '{}, {:.3f}, {:.3f}\n'.format(epoch, item[0], item[1])
                f.write(line)
                epoch += 1

        self.model_file += '_TestAcc{:.1f}percent_.pickle'.format(
            model_test_accuracy)
        # Save model
        self.save_model()
Example #16
0
year = '2007'

train_set = PASCAL('trainval',
                   year,
                   path=args.data_dir,
                   n_mb=n_mb,
                   rpn_rois_per_img=rpn_rois_per_img,
                   frcn_rois_per_img=frcn_rois_per_img,
                   add_flipped=True,
                   shuffle=True,
                   rebuild_cache=True,
                   subset_pct=args.subset_pct)

# build the Faster-RCNN model
model = util.build_model(train_set, frcn_rois_per_img, inference=False)

# set up cost different branches, respectively
weights = 1.0 / (rpn_rois_per_img)
roi_w = 1.0 / (frcn_rois_per_img)

frcn_tree_cost = Multicost(costs=[
    GeneralizedCostMask(costfunc=CrossEntropyMulti(), weights=roi_w),
    GeneralizedCostMask(costfunc=SmoothL1Loss(), weights=roi_w)
],
                           weights=[1, 1])

cost = Multicost(costs=[
    GeneralizedCostMask(costfunc=CrossEntropyMulti(), weights=weights),
    GeneralizedCostMask(costfunc=SmoothL1Loss(sigma=3.0), weights=weights),
    frcn_tree_cost,
Example #17
0
def run_task(data_dir, task_id):
    """
    Train and test for each task
    """
    print("Train and test for task %d ..." % task_id)

    # Parse data
    train_files = glob.glob('%s/qa%d_*_train.txt' % (data_dir, task_id))
    test_files  = glob.glob('%s/qa%d_*_test.txt' % (data_dir, task_id))
    #train_files = glob.glob('%s/qa%d_*train.txt' % (data_dir, task_id))
    #test_files  = glob.glob('%s/qa%d_*test.txt' % (data_dir, task_id))

    # #### empty dictionary
    dictionary = {"nil": 0}
    train_story, train_questions, train_qstory = parse_babi_task(train_files, dictionary, False)
    test_story, test_questions, test_qstory    = parse_babi_task(test_files, dictionary, False)
    

    general_config = BabiConfig(train_story, train_questions, dictionary)


    memory, model, loss = build_model(general_config)

    if general_config.linear_start:
        train_linear_start(train_story, train_questions, train_qstory, memory, model, loss, general_config)
    else:
        train(train_story, train_questions, train_qstory, memory, model, loss, general_config)
    
    with open('R_trained.txt', 'a') as outfile:
        json.dump(general_config.dictionary, outfile, indent=2)

    print("######## trained dictionary")
    print(general_config.dictionary)


    ans_index = test(test_story, test_questions, test_qstory, memory, model, loss, general_config)





    ####R this line load model
    memn2n = MemN2N(args.data_dir, args.model_file)
    #Try to load model
    memn2n.load_model()  

    dictionary2 = {"nil": 0}
    train_story2, train_questions2, train_qstory2 = parse_babi_task(train_files, memn2n.general_config.dictionary, False)
    test_story2, test_questions2, test_qstory2    = parse_babi_task(test_files, memn2n.general_config.dictionary, False)

    #print(len(test_questions2))
    #general_config2 = BabiConfig(train_story2, train_questions2,memn2n.general_config.dictionary)



    with open('R_loaded.txt', 'a') as outfile2:
        json.dump(memn2n.general_config.dictionary, outfile2, indent=2)

    print("???????? loaded dictionary")
    print(memn2n.general_config.dictionary)

    ans_index = test(test_story2, test_questions2, test_qstory2, memn2n.memory, memn2n.model, memn2n.loss, memn2n.general_config)
Example #18
0
  def train(self):
    """
    Train MemN2N model using training data for tasks.
    """
    #np.random.seed(42)  # for reproducing
    np.random.seed(self.rnd_seed)  # for reproducing
    print("np.random.seed: %d" % self.rnd_seed)
    assert self.data_dir is not None, "data_dir is not specified."
    print("Reading data from %s ..." % self.data_dir)

    # Parse training data
    train_data_path = glob.glob('%s/qa*_*_train.txt' % self.data_dir)
    dictionary = {"nil": 0}
    train_story, train_questions, train_qstory = parse_babi_task(train_data_path, dictionary, False)

    # Parse test data just to expand the dictionary so that it covers all words in the test data too
    test_data_path = glob.glob('%s/qa*_*_test.txt' % self.data_dir)
    parse_babi_task(test_data_path, dictionary, False)

    # Get reversed dictionary mapping index to word
    self.reversed_dict = dict((ix, w) for w, ix in dictionary.items())

    # Construct model
    #self.general_config = Babi10kConfigJoint(train_story, train_questions, dictionary)
    self.general_config = BabiConfigJoint(train_story, train_questions, dictionary)
    self.memory, self.model, self.loss = build_model(self.general_config)

    # Train model
    if self.general_config.linear_start:
      print('We will use LS training')
      self.best_model, self.best_memory = \
        train_linear_start(train_story, 
                           train_questions, 
                           train_qstory, 
                           self.memory, 
                           self.model, 
                           self.loss, 
                           self.general_config, 
                           self.log_path)
    else:
      train_logger = open(os.path.join(self.log_path, 'train.log'), 'w')
      train_logger.write('epoch batch_iter lr loss err\n')
      train_logger.flush()
      val_logger = open(os.path.join(self.log_path, 'val.log'), 'w')
      val_logger.write('epoch batch_iter lr loss err\n')
      val_logger.flush()
      global_batch_iter = 0
      train_logger, val_logger, self.best_model, self.best_memory, _ = \
        train(train_story, 
              train_questions, 
              train_qstory,
              self.memory, 
              self.model, 
              self.loss, 
              self.general_config,
              train_logger, 
              val_logger,
              global_batch_iter)

      train_logger.close()
      val_logger.close()

    # Save model
    self.save_model()
Example #19
0
    def train(self):
        """
    Train MemN2N model using training data for tasks.
    """
        #np.random.seed(42)  # for reproducing
        np.random.seed(self.rnd_seed)  # for reproducing
        print("np.random.seed: %d" % self.rnd_seed)
        assert self.data_dir is not None, "data_dir is not specified."
        print("Reading data from %s ..." % self.data_dir)

        # Parse training data
        train_data_path = glob.glob('%s/qa*_*_train.txt' % self.data_dir)
        dictionary = {"nil": 0}
        train_story, train_questions, train_qstory = parse_babi_task(
            train_data_path, dictionary, False)

        # Parse test data just to expand the dictionary so that it covers all words in the test data too
        test_data_path = glob.glob('%s/qa*_*_test.txt' % self.data_dir)
        parse_babi_task(test_data_path, dictionary, False)

        # Get reversed dictionary mapping index to word
        self.reversed_dict = dict((ix, w) for w, ix in dictionary.items())

        # Construct model
        #self.general_config = Babi10kConfigJoint(train_story, train_questions, dictionary)
        self.general_config = BabiConfigJoint(train_story, train_questions,
                                              dictionary)
        self.memory, self.model, self.loss = build_model(self.general_config)

        # Train model
        if self.general_config.linear_start:
            print('We will use LS training')
            self.best_model, self.best_memory = \
              train_linear_start(train_story,
                                 train_questions,
                                 train_qstory,
                                 self.memory,
                                 self.model,
                                 self.loss,
                                 self.general_config,
                                 self.log_path)
        else:
            train_logger = open(os.path.join(self.log_path, 'train.log'), 'w')
            train_logger.write('epoch batch_iter lr loss err\n')
            train_logger.flush()
            val_logger = open(os.path.join(self.log_path, 'val.log'), 'w')
            val_logger.write('epoch batch_iter lr loss err\n')
            val_logger.flush()
            global_batch_iter = 0
            train_logger, val_logger, self.best_model, self.best_memory, _ = \
              train(train_story,
                    train_questions,
                    train_qstory,
                    self.memory,
                    self.model,
                    self.loss,
                    self.general_config,
                    train_logger,
                    val_logger,
                    global_batch_iter)

            train_logger.close()
            val_logger.close()

        # Save model
        self.save_model()
Example #20
0
def run_task(data_dir, task_id, model_file, log_path):
    """
  Train and test for each task
  """
    print("Train and test for task %d ..." % task_id)

    train_files = glob.glob('%s/qa%d_*_train.txt' % (data_dir, task_id))
    test_files = glob.glob('%s/qa%d_*_test.txt' % (data_dir, task_id))

    dictionary = {"nil": 0}
    train_story, train_questions, train_qstory = \
      parse_babi_task(train_files, dictionary, False)
    test_story, test_questions, test_qstory = \
      parse_babi_task(test_files, dictionary, False)

    # Get reversed dictionary mapping index to word
    # NOTE: this needed to real-time testing
    reversed_dict = dict((ix, w) for w, ix in dictionary.items())

    general_config = BabiConfig(train_story, train_questions, dictionary)
    memory, model, loss_func = build_model(general_config)

    if general_config.linear_start:
        print('We will use LS training')
        best_model, best_memory = \
          train_linear_start(train_story,
                             train_questions,
                             train_qstory,
                             memory,
                             model,
                             loss_func,
                             general_config,
                             self.log_path)
    else:
        train_logger = open(os.path.join(self.log_path, 'train.log'), 'w')
        train_logger.write('epoch batch_iter lr loss err\n')
        train_logger.flush()
        val_logger = open(os.path.join(self.log_path, 'val.log'), 'w')
        val_logger.write('epoch batch_iter lr loss err\n')
        val_logger.flush()
        global_batch_iter = 0
        train_logger, val_logger, _, _, _ = \
          train(train_story,
                train_questions,
                train_qstory,
                memory,
                model,
                loss_func,
                general_config,
                train_logger,
                val_logger,
                global_batch_iter)
        train_logger.close()
        val_logger.close()

    model_file = os.path.join(log_path, model_file)
    with gzip.open(model_file, 'wb') as f:
        print('Saving model to file %s ...' % model_file)
        pickle.dump((reversed_dict, memory, model, loss_func, general_config),
                    f)

    print('Start to testing')
    test(test_story, test_questions, test_qstory, memory, model, loss_func,
         general_config)
Example #21
0
year = '2007'
valid_set = PASCAL(image_set,
                   year,
                   path=args.data_dir,
                   n_mb=n_mb,
                   rpn_rois_per_img=rpn_rois_per_img,
                   frcn_rois_per_img=frcn_rois_per_img,
                   add_flipped=False,
                   shuffle=False,
                   rebuild_cache=True)

num_classes = valid_set.num_classes

# build the Faster-RCNN network
(model, proposalLayer) = util.build_model(valid_set,
                                          frcn_rois_per_img,
                                          inference=True)

# load parameters and initialize model
model.load_params(args.model_file)
model.initialize(dataset=valid_set)

# normalize the model by the bbtarget mean and std if needed
# if a full training run was completed using train.py, then normalization
# was already performed prior to saving the model.
if args.normalize:
    model = util.scale_bbreg_weights(model, [0.0, 0.0, 0.0, 0.0],
                                     [0.1, 0.1, 0.2, 0.2], num_classes)

# run inference
# model.benchmark(valid_set, inference=True)
Example #22
0
rpn_rois_per_img = 256
frcn_rois_per_img = 128

# setup backend
be = gen_backend(**extract_valid_args(args, gen_backend))

image_set = 'test'
year = '2007'
valid_set = PASCAL(image_set, year, path=args.data_dir, n_mb=n_mb,
                   rpn_rois_per_img=rpn_rois_per_img, frcn_rois_per_img=frcn_rois_per_img,
                   add_flipped=False, shuffle=False, rebuild_cache=True)

num_classes = valid_set.num_classes

# build the Faster-RCNN network
(model, proposalLayer) = util.build_model(valid_set, frcn_rois_per_img, inference=True)

# load parameters and initialize model
model.load_params(args.model_file)
model.initialize(dataset=valid_set)

# normalize the model by the bbtarget mean and std if needed
# if a full training run was completed using train.py, then normalization
# was already performed prior to saving the model.
if args.normalize:
    model = util.scale_bbreg_weights(model, [0.0, 0.0, 0.0, 0.0],
                                     [0.1, 0.1, 0.2, 0.2], num_classes)

# run inference
# model.benchmark(valid_set, inference=True)
Example #23
0
x_train, x_test, y_train, y_test = train_test_split(newHeadlinesData, normalizedOpeningprice, test_size=0.2,
                                                    random_state=3)

# converting into array

x_train = np.array(x_train)
x_test = np.array(x_test)
y_train = np.array(y_train)
y_test = np.array(y_test)

print()
print("Train data shape : ", x_train.shape)
print("Test data shape : ", x_test.shape)

trainedModel = build_model(wordsCount, embeddingDimension, embeddingMatrix, wordLimit)
trainedModel.summary()

save_best_weights = 'model.h5'
callbacks = [ModelCheckpoint(save_best_weights, monitor='val_loss', save_best_only=True),
             EarlyStopping(monitor='val_loss', patience=5, verbose=1, mode='auto')]
history = trainedModel.fit([x_train, x_train], y_train, batch_size=128, epochs=100, validation_split=0.15, verbose=True,
                           shuffle=True, callbacks=callbacks)

# evaluation

trainedModel.load_weights("./model.h5")
predictions = list(chain.from_iterable(trainedModel.predict([x_test, x_test], verbose=True)))
error = mse(y_test, predictions)
print()
print("Mean Square Error : ", error)
Example #24
0
def run_joint_tasks(data_dir, model_file, log_path):
    """
  Train and test for all tasks but the trained model is built using training data from all tasks.
  """
    print("Jointly train and test for all tasks ...")
    tasks = range(20)

    # Parse training data
    train_data_path = []
    for t in tasks:
        train_data_path += glob.glob('%s/qa%d_*_train.txt' % (data_dir, t + 1))

    dictionary = {"nil": 0}
    train_story, train_questions, train_qstory = \
      parse_babi_task(train_data_path,
                      dictionary,
                      False)

    # Parse test data for each task so that the dictionary covers all words before training
    for t in tasks:
        test_data_path = glob.glob('%s/qa%d_*_test.txt' % (data_dir, t + 1))
        parse_babi_task(test_data_path, dictionary,
                        False)  # ignore output for now

    # Get reversed dictionary mapping index to word
    # NOTE: this needed to real-time testing
    reversed_dict = dict((ix, w) for w, ix in dictionary.items())

    general_config = BabiConfigJoint(train_story, train_questions, dictionary)
    memory, model, loss_func = build_model(general_config)

    if general_config.linear_start:
        print('We will use LS training')
        train_linear_start(train_story, train_questions, train_qstory, memory,
                           model, loss_func, general_config, log_path)
    else:
        train_logger = open(os.path.join(log_file, 'train.log'), 'w')
        train_logger.write('epoch batch_iter lr loss err\n')
        train_logger.flush()
        val_logger = open(os.path.join(log_file, 'val.log'), 'w')
        val_logger.write('epoch batch_iter lr loss err\n')
        val_logger.flush()
        train_logger, val_logger, best_model, best_memory = \
          train(train_story,
                train_questions,
                train_qstory,
                memory,
                model,
                loss_func,
                general_config,
                train_logger,
                val_logger)

        train_logger.close()
        val_logger.close()

    model_file = os.path.join(log_path, model_file)
    with gzip.open(model_file, 'wb') as f:
        print('Saving model to file %s ...' % model_file)
        pickle.dump((reversed_dict, memory, model, loss_func, general_config),
                    f)

    # Test on each task
    print('Start to testing')
    for t in tasks:
        print("Testing for task %d ..." % (t + 1))
        test_data_path = glob.glob('%s/qa%d_*_test.txt' % (data_dir, t + 1))
        dc = len(dictionary)
        test_story, test_questions, test_qstory = \
          parse_babi_task(test_data_path,
                          dictionary,
                          False)
        assert dc == len(
            dictionary
        )  # make sure that the dictionary already covers all words

        test(test_story, test_questions, test_qstory, memory, model, loss_func,
             general_config)
Example #25
0
import commons
import test
import train
import grl_util

######################################### SETUP #########################################
args = parser.parse_arguments()
start_time = datetime.now()
args.output_folder = f"runs/{args.exp_name}/{start_time.strftime('%Y-%m-%d_%H-%M-%S')}"
commons.setup_logging(args.output_folder)
commons.make_deterministic(args.seed)
logging.info(f"Arguments: {args}")
logging.info(f"The outputs are being saved in {args.output_folder}")

######################################### MODEL #########################################
model = util.build_model(args)

######################################### OPTIMIZER & LOSSES #########################################
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
criterion_netvlad = nn.TripletMarginLoss(margin=args.margin**0.5,
                                         p=2,
                                         reduction="sum")

######################################### RESUME #########################################
if args.resume:
    model, optimizer, best_score, start_epoch = util.resume_train(
        args, model, optimizer)
else:
    start_epoch = 0

######################################### DATASETS #########################################
Example #26
0
filename = 'shakespeare'

epochs = 20
batch_size = 64
seq_length = 100
rnn_units = 128  # 1024
embedding_dim = None  # 256

dataset, idx_to_char, char_to_idx, vocab = preprocess(filename, batch_size,
                                                      seq_length)

vocab_size = len(vocab)
checkpoint_dir = './training_checkpoints/' + filename
checkpoint_prefix = checkpoint_dir + '/ckpt'

### Train model (comment out if only generating)
model = build_model(vocab_size, embedding_dim, rnn_units, batch_size)
# model.load_weights(tf.train.latest_checkpoint(checkpoint_dir)) # use this to continue training where it left off
history = train_model(model,
                      dataset,
                      epochs=epochs,
                      checkpoint_prefix=checkpoint_prefix)

### Generate sample (comment out if only training)
#model = build_model(vocab_size, embedding_dim, rnn_units, batch_size=1)
#model.load_weights(tf.train.latest_checkpoint(checkpoint_dir))
#model.build(tf.TensorShape([1, None]))
#print(generate_text(model,char_to_idx,idx_to_char,
#                    start_string=u"Romeo:\n"))