def minibatch(self,
dataset,
subset,
use_datasets,
cache_data,
shift_ratio=-1):
if shift_ratio < 0:
shift_ratio = self.shift_ratio
with tf.name_scope('batch_processing'):
# Build final results per split.
images = [[] for _ in range(self.num_splits)]
labels = [[] for _ in range(self.num_splits)]
if use_datasets:
ds = data_utils.create_dataset(self.batch_size,
self.num_splits,
self.batch_size_per_split,
self.parse_and_preprocess,
dataset, subset, self.train,
cache_data)
ds_iterator = data_utils.create_iterator(ds)
for d in xrange(self.num_splits):
labels[d], images[d] = ds_iterator.get_next()
else:
record_input = data_flow_ops.RecordInput(
file_pattern=dataset.tf_record_pattern(subset),
seed=301,
parallelism=64,
buffer_size=10000,
batch_size=self.batch_size,
shift_ratio=shift_ratio,
name='record_input')
records = record_input.get_yield_op()
records = tf.split(records, self.batch_size, 0)
records = [tf.reshape(record, []) for record in records]
for idx in xrange(self.batch_size):
value = records[idx]
(label, image) = self.parse_and_preprocess(value, idx)
split_index = idx % self.num_splits
labels[split_index].append(label)
images[split_index].append(image)
for split_index in xrange(self.num_splits):
if not use_datasets:
images[split_index] = tf.parallel_stack(
images[split_index])
labels[split_index] = tf.concat(labels[split_index], 0)
images[split_index] = tf.reshape(images[split_index],
shape=[
self.batch_size_per_split,
self.height, self.width,
self.depth
])
labels[split_index] = tf.reshape(labels[split_index],
[self.batch_size_per_split])
return images, labels
def build_inputs(self):
with tf.variable_scope('inputs'):
print('Loading dataset...')
self.dataset = data_utils.create_dataset(
self.data_dir,
self.batch_size,
self.scale_dataset,
self.crop_dataset
)
print('Dataset: {}'.format('SPACE'))
print(' - size: {}'.format(self.dataset_size))
print(' - channels: {}'.format(self.channels))
self.data_iterator = self.dataset.make_one_shot_iterator()
self.data_init_op = None
self.real_images = self.data_iterator.get_next(
name='real_images'
)
self.zc_vectors = tf.placeholder(
tf.float32,
[None, self.z_size],
name='zc_vectors'
)
self.is_training = tf.placeholder_with_default(False, shape=[], name='is_training')
self.global_step = tf.Variable(0, trainable=False, name='global_step')
if self.regularizer_scale > 0.0:
self.generator_regularizer = tf.contrib.layers.l2_regularizer(self.regularizer_scale)
self.discriminator_regularizer = tf.contrib.layers.l2_regularizer(self.regularizer_scale)
self.mutual_info_regularizer = tf.contrib.layers.l2_regularizer(self.regularizer_scale)
else:
self.generator_regularizer = None
self.discriminator_regularizer = None
self.mutual_info_regularizer = None
batch_token_ids = []
for i in range(0, len(X)):
token_ids = X[i].tolist()
segment_ids = [0 for token in X[i]]
batch_token_ids.append(token_ids)
batch_segment_ids.append(segment_ids)
return [np.array(batch_token_ids), np.array(batch_segment_ids)]
model_path = "model_75_epochs_no_earlystopping.hd5"
print("Trained model path : %s" % model_path)
test_filename = "data/trial_data.txt"
print("Test dataset path : %s" % test_filename)
results_path = "data/res/submission.txt"
print("Results path : %s" % results_path)
# ALBERT predictions
print("\n === ALBERT predictions ===\n")
X_test, _, _ = create_dataset(test_filename)
model = load_model(model_path, custom_objects=get_custom_objects())
model.summary()
predictions = model.predict(data_creator(X_test))
word_id_lsts, post_lsts, _, _, _, _ = read_data(test_filename)
predictions_unpadded = unpad(np.array(predictions), word_id_lsts)
write_results(word_id_lsts, post_lsts, predictions_unpadded, results_path)
print("Results written")
if __name__=='__main__': args=parse_args() data_dir,model_net,dataset_name,batch=args.data_dir,args.model_net,args.dataset,args.batch #load dataset into data loader logit_predicted_train=torch.from_numpy(numpy.load(data_dir+model_net+"_"+dataset_name+'/'+dataset_name+"_"+model_net+"_logit_prediction_train.npy")).cuda() logit_predicted_valid=torch.from_numpy(numpy.load(data_dir+model_net+"_"+dataset_name+'/'+dataset_name+"_"+model_net+"_logit_prediction_valid.npy")).cuda() logit_predicted_test=torch.from_numpy(numpy.load(data_dir+model_net+"_"+dataset_name+'/'+dataset_name+"_"+model_net+"_logit_prediction_test.npy")).cuda() true_train=torch.from_numpy(numpy.load(data_dir+model_net+"_"+dataset_name+'/'+dataset_name+"_"+model_net+"_true_train.npy")).long().cuda() true_valid=torch.from_numpy(numpy.load(data_dir+model_net+"_"+dataset_name+'/'+dataset_name+"_"+model_net+"_true_valid.npy")).long().cuda() true_test=torch.from_numpy(numpy.load(data_dir+model_net+"_"+dataset_name+'/'+dataset_name+"_"+model_net+"_true_test.npy")).long().cuda() tr=[logit_predicted_train,logit_predicted_valid] te=[true_train,true_valid] dataset=[tr,te] trainLoader,validLoader=create_dataset(dataset,'FC',batch_size_tr=batch,batch_size_te=batch,tocuda=True) tr=[logit_predicted_train,logit_predicted_test] te=[true_train,true_test] dataset=[tr,te] _,testLoader=create_dataset(dataset,'FC',batch_size_tr=batch,batch_size_te=batch,tocuda=True) #networks if dataset_name in ['cifar10','svhn']: input_dim=10 elif dataset_name=='cifar100': input_dim=100 elif dataset_name in ['gender','vggface2']: input_dim=2 elif dataset_name=='cars': input_dim=196 elif dataset_name=='birds':
lr_new=-(lr_init/epochs_N)*e+lr_init
return lr_new
if __name__=='__main__':
MC_samples,model_dir,data_dir,dataset_name,model_net,valid_test=parse_args()
#load dataset into data loader
logit_predicted_test=torch.from_numpy(numpy.load(data_dir+model_net+"_"+dataset_name+'/'+dataset_name+"_"+model_net+"_logit_prediction_test.npy")).cuda()
logit_predicted_valid=torch.from_numpy(numpy.load(data_dir+model_net+"_"+dataset_name+'/'+dataset_name+"_"+model_net+"_logit_prediction_valid.npy")).cuda()
true_test=torch.from_numpy(numpy.load(data_dir+model_net+"_"+dataset_name+'/'+dataset_name+"_"+model_net+"_true_test.npy")).long().cuda()
true_valid=torch.from_numpy(numpy.load(data_dir+model_net+"_"+dataset_name+'/'+dataset_name+"_"+model_net+"_true_valid.npy")).long().cuda()
tr=[logit_predicted_test,logit_predicted_valid]
te=[true_test,true_valid]
dataset=[tr,te]
testLoader,validLoader=create_dataset(dataset,batch_size_tr=10000,batch_size_te=5000,tocuda=True)
#networks
variatonal = torch.load(model_dir+"models/"+"variatonal",map_location=lambda storage,loc:storage.cuda(0))
reconstruction = torch.load(model_dir+"models/"+"reconstruction",map_location=lambda storage,loc:storage.cuda(0))
q_m,q_logv=variatonal.forward()
'''
Measuring calibration
'''
n_bins=15
if valid_test=='validation':
#utilities
CE=nn.functional.cross_entropy
#before calibration
avg_confidence_before_BNN_valid=average_confidence(logit_predicted_valid)*100
if config.seed is not None:
np.random.seed(config.seed)
torch.manual_seed(config.seed)
results_folder = os.path.join(
config.results_folder, "_".join([
datetime.datetime.now().strftime("%d-%m-%Y_%H-%M-%S"), config.model
]))
os.mkdir(results_folder)
print("Results will be saved here : " + str(results_folder))
with open(os.path.join(results_folder, "config.yaml"), "w") as f:
f.write(yaml.dump(config_yaml))
train_dataset, val_dataset = create_dataset(config.file, config.train_per,
1, config.step_size,
config.data_transform,
config.standardization)
train_dataloader = DataLoader(train_dataset,
batch_size=config.batch_size,
shuffle=True)
val_dataloader = DataLoader(val_dataset,
batch_size=config.batch_size,
shuffle=False)
device = "cuda" if torch.cuda.is_available() else "cpu"
from models.rnn.lstm import CNN_autoencoder
model = CNN_autoencoder(config.step_size, config.embed_size).to(device)
loss = nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=config.lr)
"_logit_prediction_test.npy")).float().cuda()
true_valid = torch.from_numpy(
numpy.load(data_dir + model_net + "_" + dataset_name + '/' +
dataset_name + "_" + model_net +
"_true_valid.npy")).long().cuda()
true_test = torch.from_numpy(
numpy.load(data_dir + model_net + "_" + dataset_name + '/' +
dataset_name + "_" + model_net +
"_true_test.npy")).long().cuda()
tr = [logit_predicted_valid, logit_predicted_test]
te = [true_valid, true_test]
dataset = [tr, te]
trainLoader, validLoader = create_dataset(dataset,
'FC',
batch_size_tr=1000,
batch_size_te=1000,
tocuda=True)
#networks
if dataset_name in ['cifar10', 'svhn']:
input_dim = 10
elif dataset_name == 'cifar100':
input_dim = 100
elif dataset_name in ['gender', 'vggface2']:
input_dim = 2
elif dataset_name == 'cars':
input_dim = 196
elif dataset_name == 'birds':
input_dim = 200
else:
], batch_labels, batch_weights
batch_token_ids, batch_segment_ids, batch_labels, weights, batch_weights,labels = [], [], [], [], [], []
train_filename = "data/train.txt"
print("Data path : %s" % train_filename)
model_path = "model_100_epochs_batch64_earlystopping.hd5"
print("Trained model path : %s" % model_path)
config_path = "bert_base_cased/albert_config_base.json"
print("Config path : %s" % config_path)
checkpoint_path = "bert_base_cased/bert_model.ckpt"
print("Checkpoint path : %s\n" % checkpoint_path)
# Dataset generation
print(" === Dataset generation ===\n")
X_train, y_train, weights = create_dataset(train_filename)
maxlen = len(y_train[0])
#embedding_model = api.load("glove-twitter-25")
BATCH_SIZE = 64
INIT_LR = 10e-5
NB_EPOCHS = 100
# ALBERT model
print("\n === ALBERT model configuration ===\n")
bert = build_bert_model(config_path,
checkpoint_path,
with_pool=True,
albert=True,
return_keras_model=False)
'tf.Example protos for validation.')
parser.add_argument('w2v', help='The pre-trained word vector.')
parser.add_argument('vocab', help='The vocabulary file containing all words')
parser.add_argument('ckpt_dir',
help='Directory for saving and loading checkpoints.')
# parser.add_argument('model_dir', default='lstm_crf_tagger',
# help='Directory for saving and loading model.')
# parser.add_argument('model_version', default='0001',
# help='Version of saved model.')
args = parser.parse_args()
model_config = configuration.ModelConfig()
train_config = configuration.TrainingConfig()
train_dataset = data_utils.create_dataset(args.train_pattern,
train_config.batch_size)
dev_dataset = data_utils.create_dataset(args.dev_pattern,
train_config.batch_size)
vocab = data_utils.load_vocab(args.vocab)
w2v = data_utils.load_w2v(args.w2v, vocab)
embeddings = data_utils.load_vocab_embeddings(w2v, vocab, model_config.d_word)
tagger = model.Model(embeddings, len(vocab), model_config)
optimizer = tf.keras.optimizers.SGD(lr=train_config.learning_rate,
clipvalue=train_config.clip_gradients)
train_loss = tf.keras.metrics.Mean(name='train_loss')
train_accuracy = tf.keras.metrics.Accuracy(name='train_accuracy')
train_metric = metrics.TaggerMetric(model_config.n_tags)
if __name__=='__main__': model_net,MC_samples,dkl_scale_factor,dkl_after_epoch,data_dir,dataset_name,save_after_epoch,epochs_t,lr_t,batch,anneal,folder_name,layer_dim,n_layers=parse_args() #load dataset into data loader logit_predicted_train=torch.from_numpy(numpy.load(data_dir+model_net+"_"+dataset_name+'/'+dataset_name+"_"+model_net+"_logit_prediction_train.npy")).cuda() logit_predicted_valid=torch.from_numpy(numpy.load(data_dir+model_net+"_"+dataset_name+'/'+dataset_name+"_"+model_net+"_logit_prediction_valid.npy")).cuda() logit_predicted_test=torch.from_numpy(numpy.load(data_dir+model_net+"_"+dataset_name+'/'+dataset_name+"_"+model_net+"_logit_prediction_test.npy")).cuda() true_train=torch.from_numpy(numpy.load(data_dir+model_net+"_"+dataset_name+'/'+dataset_name+"_"+model_net+"_true_train.npy")).long().cuda() true_valid=torch.from_numpy(numpy.load(data_dir+model_net+"_"+dataset_name+'/'+dataset_name+"_"+model_net+"_true_valid.npy")).long().cuda() true_test=torch.from_numpy(numpy.load(data_dir+model_net+"_"+dataset_name+'/'+dataset_name+"_"+model_net+"_true_test.npy")).long().cuda() tr=[logit_predicted_train,logit_predicted_valid] te=[true_train,true_valid] dataset=[tr,te] trainLoader,testLoader=create_dataset(dataset,batch_size_tr=batch,batch_size_te=batch,tocuda=True) #networks if dataset_name in ['cifar10','svhn']: input_dim=10 elif dataset_name=='cifar100': input_dim=100 elif dataset_name in ['gender','vggface2']: input_dim=2 elif dataset_name == 'cars': input_dim = 196 elif dataset_name == 'birds': input_dim = 200 else: raise NotImplemented
np.random.seed(config.seed)
results_folder = os.path.join(
config.results_folder, "_".join([
datetime.datetime.now().strftime("%d-%m-%Y_%H-%M-%S"), config.model
]))
os.mkdir(results_folder)
print("Results will be saved here : " + str(results_folder))
with open(os.path.join(results_folder, "config.yaml"), "w") as f:
f.write(yaml.dump(config_yaml))
train_dataset, val_dataset = create_dataset(config.file,
config.train_per,
1,
config.sample_size,
config.data_transform,
config.standardization,
tensor=False)
D0 = np.random.randn(train_dataset.shape[2], config.embed_size)
opt = bpdndl.BPDNDictLearn.Options({
'Verbose': False,
'MaxMainIter': 100,
'BPDN': {
'rho': 10.0 * config.lambda_l + 0.1
},
'CMOD': {
'rho': train_dataset.shape[0] / 1e3
}