def Data_load(num_timesteps_input, num_timesteps_output):
A, X, means, stds, nodes = load_metr_la_data()
# A, X, max_X, min_X = load_metr_la_data()
# A, X, max_value, X_val = load_metr_la_data()
split_line1 = int(X.shape[2] * 0.6)
split_line2 = int(X.shape[2] * 0.8)
train_original_data = X[:, :, :split_line1]
val_original_data = X[:, :, split_line1:split_line2]
# val_original_data = X_val
test_original_data = X[:, :, split_line2:]
training_input, training_target = generate_dataset(
train_original_data,
num_timesteps_input=num_timesteps_input,
num_timesteps_output=num_timesteps_output)
val_input, val_target = generate_dataset(
val_original_data,
num_timesteps_input=num_timesteps_input,
num_timesteps_output=num_timesteps_output)
test_input, test_target = generate_dataset(
test_original_data,
num_timesteps_input=num_timesteps_input,
num_timesteps_output=num_timesteps_output)
return A, means, stds, training_input, training_target, val_input, val_target, test_input, test_target, nodes
def Data_load(num_timesteps_input, num_timesteps_output):
A, X, means, stds, X_val = load_metr_la_data()
# split_line1 = int(X.shape[0] * 0.6)
# split_line2 = int(X.shape[0] * 0.8)
# train_original_data = X[:, :, :split_line1]
train_original_data = X
# val_original_data = X[:, :, split_line1:split_line2]
val_original_data = X_val
test_original_data = X #X[split_line1:, :, :]
training_input, training_target = generate_dataset(
train_original_data,
num_timesteps_input=num_timesteps_input,
num_timesteps_output=num_timesteps_output)
val_input, val_target = generate_dataset(
val_original_data,
num_timesteps_input=num_timesteps_input,
num_timesteps_output=num_timesteps_output)
test_input, test_target = generate_dataset(
test_original_data,
num_timesteps_input=num_timesteps_input,
num_timesteps_output=num_timesteps_output)
return A, means, stds, training_input, training_target, val_input, val_target, test_input, test_target
def main():
print('Welcome aboard. Fasten your seatbelts')
#read parameters
arguments = read_args()
#download pre-trained model and initialize
model, criterion, optimizer = utils.create_model(arguments.arch,
arguments.hidden_units,
n_classes,
arguments.learning_rate,
dropout_per)
#gpu or cpu
processor = utils.get_processor(arguments.gpu)
#generate and normalize images for training and validation
train_dataset, train_classes_ids = utils.generate_dataset('train', arguments.data_dir)
valid_dataset, test_classes_ids = utils.generate_dataset('valid', arguments.data_dir)
#train network
model, optimizer = do_deep_learning(processor,
model,
criterion,
optimizer,
arguments.epochs,
train_dataset,
valid_dataset)
#test network with test dataset
test_dataset, test_classes_ids = utils.generate_dataset('test', arguments.data_dir)
check_accuracy_on_test(test_dataset,
model,
processor)
#save checkpoint
save_params_dict = {'hidden_size': arguments.hidden_units,
'output_size': n_classes,
'dropout_per': dropout_per,
'learning_rate': arguments.learning_rate,
'epochs_trained': arguments.epochs,
'img_mapping': train_classes_ids,
'optimizer_state': optimizer.state_dict(),
'model_state': model.state_dict(),
'classifier': model.classifier,
'arch': arguments.arch,
'input_size': model.classifier[0].in_features}
save_checkpoint(utils.basepath+arguments.save_dir+checkpoint_name,save_params_dict)
print("**INFO: Training Ended!")
def main():
X = generate_dataset(shape="blobs")
D = pairwise_distances(X) # euclidean distance as distance metric
A = gaussian_kernel(D, is_sym=True) # Gaussian distance as affinity metric
# K-MEANS
clusters, _ = apply_kmeans(X)
plot_clustering_result(X,
A,
clusters,
clustering_name="K means clustering")
# DBSCAN
clusters, noise = apply_dbscan(X, D)
plot_clustering_result(X,
A,
clusters,
noise,
clustering_name="DBSCAN clustering")
# EIGENVECTOR BASED CLUSTERING
A_eigen = gaussian_kernel(
D, mult=0.05, is_sym=True) # Gaussian distance as affinity metric
clusters, noise = apply_eigenvector_based(X, A_eigen)
plot_clustering_result(X,
A_eigen,
clusters,
noise,
clustering_name="Eigenvector based clustering")
def __init__(self, X, train_fraction=0.1, consider_explicit=False):
self.X_train, self.X_validation, self.X_test, self.test_list = utils.generate_dataset(
X, train_fraction)
self.n = np.shape(X)[0]
self.consider_explicit = consider_explicit
self.U = np.random.rand((self.r, self.n))
self.V = np.random.rand((self.r, self.n))
self.update_estimation()
def generate_datafile_old(number_items=1000):
"""
Create the samples.py file
"""
from utils import get_names, generate_dataset
from pprint import pprint
filename = "samples.py"
dataset = generate_dataset(number_items)
fo = open(filename, "wb")
fo.write("#!/usr/bin/env python\n")
fo.write("# -*- coding: utf-8 -*-\n")
fo.write("#Brainaetic: http://www.thenetplanet.com\n\n")
fo.write("SAMPLES = ")
pprint(dataset, fo)
fo.close()
print "%s generated with %d samples"%(filename, number_items)
def generate_datafile_old(number_items=1000):
"""
Create the samples.py file
"""
from utils import get_names, generate_dataset
from pprint import pprint
filename = "samples.py"
dataset = generate_dataset(number_items)
fo = open(filename, "wb")
fo.write("#!/usr/bin/env python\n")
fo.write("# -*- coding: utf-8 -*-\n")
fo.write("#Brainaetic: http://www.thenetplanet.com\n\n")
fo.write("SAMPLES = ")
pprint(dataset, fo)
fo.close()
print "%s generated with %d samples" % (filename, number_items)
def plot_noise_condition(k, l, outfile="noise.pdf", n=100000, a=DEF_A):
x, y = generate_dataset(n=n, a=a)
filt = np.logical_or(y == k, y == l)
vals = list()
for x0 in x[filt]:
etas = get_etas(x0, a=a)
eta_kl = etas[k] / (etas[k] + etas[l])
vals.append(np.abs(eta_kl - 1 / 2))
n_classes = len(etas)
plt.figure(figsize=(4, 2))
plt.title("class {} vs class {} (k={}, l={})".format(k, l, k, l))
colors = cm.rainbow(np.linspace(0, 1, n_classes))
plt.hist(vals, bins=100, density=True)
plt.ylabel(r"$P\{ \| \eta_{k,l}(x) - 1/2 \| > t \}$")
plt.xlabel("$t$")
plt.grid()
plt.savefig(outfile)
A, X, means, stds = load_metr_la_data()
print(A.shape)
split_line1 = int(len(X) * 0.6)
split_line2 = int(len(X) * 0.8)
#
train_original_data = X[:split_line1]
val_original_data = X[split_line1:split_line2]
# val_mean, val_std = means[split_line1:split_line2], stds[split_line1:split_line2]
test_original_data = X[split_line2:]
# test_mean, test_std = means[split_line2:], stds[split_line2:]
training_input, training_target, train_mean_t, train_std_t = generate_dataset(
train_original_data,
num_timesteps_input=num_timesteps_input,
num_timesteps_output=num_timesteps_output,
means=means,
stds=stds)
val_input, val_target, val_mean_t, val_std_t = generate_dataset(
val_original_data,
num_timesteps_input=num_timesteps_input,
num_timesteps_output=num_timesteps_output,
means=means,
stds=stds)
test_input, test_target, test_mean_t, test_std_t = generate_dataset(
test_original_data,
num_timesteps_input=num_timesteps_input,
num_timesteps_output=num_timesteps_output,
means=means,
stds=stds)
epochs=5,
batch_size=2,
validation_data=(FM, cRM),
shuffle=False,
callbacks=[TensorBoard(log_dir='./log'), checkpoint])
pred_cRM = model.predict(FM)
if MODEL_CHECK2:
# check data generative function
people_num = 2
sample_range = (0, 2)
repo_path = os.path.expanduser('../../data/audio/audio_train')
X_data, y_data = avp.generate_dataset(sample_range,
repo_path,
num_speaker=2,
verbose=1)
print('shape of the X data: ', X_data.shape)
print('shape of the y data: ', y_data.shape)
# split data to training and validation set
X_train, X_val, y_train, y_val = train_test_split(X_data,
y_data,
test_size=0.05,
random_state=30)
# check feeding tensor to model
input_dim = (298, 257, 2)
output_dim = (298, 257, 2, 2)
# train and load model
def load_data(args):
# Loading embeddings matrices
print(
"=======================\nLoading embedding files\n======================="
)
embedding_dims = []
embedding_matrices = []
if "glove" in args.embeddings:
emb_fn = "msrpc_{}_{}_{}_{}_{}.pickle".format(args.pp_name,
args.lower_opt, "glove",
args.emb_opt,
args.version)
print("loading ...", emb_fn)
[embeddings_matrix,
unknown_words] = pickle.load(open("./data/" + emb_fn, 'rb'))
print("Embeddings shape (pretrained GloVe): {} unknown tokens: {}".
format(embeddings_matrix.shape, len(unknown_words)))
embedding_dims.append(embeddings_matrix.shape[1])
embedding_matrices.append(embeddings_matrix)
if "POSword2vec" in args.embeddings:
emb_fn = "msrpc_{}_{}_{}_{}_{}.pickle".format(args.pp_name,
args.lower_opt,
"POSword2vec",
args.emb_opt,
args.version)
print("loading ...", emb_fn)
[embeddings_matrix,
unknown_words] = pickle.load(open("./data/" + emb_fn, 'rb'))
print(
"Embeddings shape (pretrained POS word2vec): {} unknown tokens: {}"
.format(embeddings_matrix.shape, len(unknown_words)))
embedding_dims.append(embeddings_matrix.shape[1])
embedding_matrices.append(embeddings_matrix)
if "paragram25" in args.embeddings:
emb_fn = "msrpc_{}_{}_{}_{}_{}.pickle".format(args.pp_name,
args.lower_opt,
"paragram25",
args.emb_opt,
args.version)
print("loading ...", emb_fn)
[embeddings_matrix,
unknown_words] = pickle.load(open("./data/" + emb_fn, 'rb'))
print("Embeddings shape (pretrained PARAGRAM): {} unknown tokens: {}".
format(embeddings_matrix.shape, len(unknown_words)))
embedding_dims.append(embeddings_matrix.shape[1])
embedding_matrices.append(embeddings_matrix)
print("Final embeddings dim:", sum(embedding_dims))
print("\n")
# Generating datasets from parsed MSRPC
print("===================\nGenerating datasets\n===================")
(index_to_word, word_to_index, X_train1, X_train2, Y_train, X_test1,
X_test2,
Y_test) = utils.generate_dataset(args.pp_name,
args.lower_opt,
args.version,
max_seq_length=-1,
reverse_train_pairs=args.reversed_train,
padding=True,
autoneg=args.autoneg)
max_seq_length = X_train1.shape[1]
print("Max seq length:", max_seq_length)
print("X_train:", X_train1.shape)
print("Y_train:", Y_train.shape)
print("X_test:", X_test1.shape)
print("Y_test:", Y_test.shape)
print("\n")
return (word_to_index, max_seq_length, X_train1, X_train2, Y_train,
X_test1, X_test2, Y_test, embedding_dims, embedding_matrices)
# Path to vocabs for indexing
params["word_emb_vocab"] = os.path.join(params["embedding_path"],
"vocab.txt")
params["char_vocab"] = os.path.join(params["embedding_path"], "chars.txt")
cond = os.path.isfile(params["word_emb_vocab"]) and os.path.isfile(
params["char_vocab"])
if not cond:
train_path = glob(os.path.join(params['filepath'], '*train*'))[0]
create_full_vocab(train_path, params['embedding_path'])
# Create dataset files if not already done:
params['data_path'] = os.path.join(params['log_dir'], 'dataset_lm')
if not os.path.isdir(params['data_path']):
generate_dataset(filedir=params['filepath'],
mode='lm',
logdir=params['data_path'],
n_grams=params['add_n_grams_deps'])
# Get nb_chars, nb_labels, & nb_words for params (used in model):
with open(params["word_emb_vocab"], "r", encoding="utf-8") as f:
lines = f.readlines()
params["vocab_size"] = len(lines)
params["max_len_sent"] = max(map(len, lines))
with open(params["char_vocab"], "r", encoding="utf-8") as f:
params["nb_chars"] = len(f.readlines())
params['frequencies'] = genfromtxt(os.path.join(params['embedding_path'],
'freq.txt'),
delimiter="\t")
params['activation_finish'] = tf.nn.leaky_relu
# Create input functions
train_pairs = readdata.read_snli(args.train)
logger.info('Reading validation data')
valid_pairs = readdata.read_snli(args.validation)
logger.info('Reading word embeddings')
word_dict, embeddings = readdata.load_embeddings(args.embeddings,
args.vocab)
readdata.write_extra_embeddings(embeddings, args.save)
embeddings = utils.normalize_embeddings(embeddings)
logger.debug(
'Embeddings have shape {} (including unknown, padding and null)'.
format(embeddings.shape))
logger.info('Converting words to indices')
max_size1, max_size2 = utils.get_max_sentence_sizes(
train_pairs, valid_pairs)
train_data = utils.generate_dataset(train_pairs, word_dict, max_size1,
max_size2)
valid_data = utils.generate_dataset(valid_pairs, word_dict, max_size1,
max_size2)
# count the NULL token (it is important when there's no alignment for a given word)
max_size1 += 1
max_size2 += 1
msg = '{} sentences have shape {} (firsts) and {} (seconds)'
logger.debug(
msg.format('Training', train_data.sentences1.shape,
train_data.sentences2.shape))
logger.debug(
msg.format('Validation', valid_data.sentences1.shape,
valid_data.sentences2.shape))
A_wave = get_normalized_adj(A)
A_wave = torch.from_numpy(A_wave)
# A_wave = torch.memory_format(padding)
A_wave = A_wave.to(device=args.device)
split_line1 = int(len(X) * 0.6)
split_line2 = int(len(X) * 0.8)
val_original_data = X[split_line1:split_line2]
val_mean, val_std = means[split_line1:split_line2], stds[
split_line1:split_line2]
val_input, val_target, val_mean_t, val_std_t = generate_dataset(
val_original_data,
num_timesteps_input=num_timesteps_input,
num_timesteps_output=num_timesteps_output,
means=val_mean,
stds=val_std)
test_original_data = X[split_line2:]
test_mean, test_std = means[split_line2:], stds[split_line2:]
test_input, test_target, test_mean_t, test_std_t = generate_dataset(
test_original_data,
num_timesteps_input=num_timesteps_input,
num_timesteps_output=num_timesteps_output,
means=test_mean,
stds=test_std)
batch_size = 100
num_batches = val_target.shape[0] // batch_size
for batch_idx in range(0, num_batches + 1):
# word2vec, glove, paragram
emb_name = "word2vec"
# Duplicate training by switching pairs
reverse_train = False
# Randomly generate negative samples
autoneg = 0
#######################
# END DATA PARAMETERS #
#######################
# Generating dataset from parsed MSRPC
(index_to_word, word_to_index, X_train1, X_train2, Y_train, X_test1, X_test2,
Y_test) = utils.generate_dataset(pp_name,
lower_opt,
version,
max_seq_length=30,
reverse_train_pairs=reverse_train,
padding=True,
autoneg=autoneg)
max_seq_length = X_train1.shape[1]
print("Max seq length:", max_seq_length)
print("X_train:", X_train1.shape)
print("Y_train:", Y_train.shape)
print("X_test:", X_test1.shape)
print("Y_test:", Y_test.shape)
# Loading embeddings matrix
emb_fn = "msrpc_{}_{}_{}_{}_{}.pickle".format(pp_name, lower_opt, emb_name,
emb_opt, version)
[embedding_matrix, unknown_words] = pickle.load(open("./data/" + emb_fn, 'rb'))
vocab_size = embedding_matrix.shape[0]
print(f"Vocab Size with >= {threshold} is {len(vocab)}")
#Loading Image Encodings
read_file = open("img_encoded/img_inceptionv3_encoding.pkl", "rb")
img_encodings = pickle.load(read_file)
read_file.close()
print("Total Images : ", len(img_encodings))
# print(img_encodings[list(img_encodings.keys())[0]]) : Prints first img numpy array
#Caption Dataset
train_df,test_df = generate_dataset()
############# Data Specifications ########################
print("Unique Train Images :", len(set(train_df['image'])))
print("Unqiue Test Images : ", len(set(test_df['image'])))
print("Total Train Instances :", train_df.shape[0])
print("Total Test Instances : ", test_df.shape[0])
###########################################################
train_df['caption'] = train_df['caption'].apply(lambda text: "<SOS> " + text + " <EOS>")
test_df['caption'] = test_df['caption'].apply(lambda text: "<SOS> " + text + " <EOS>")
train_descriptions = train_df.groupby(['image']).apply(lambda sub_df : sub_df["caption"].to_list()).to_dict()
test_descriptions = test_df.groupby(['image']).apply(lambda sub_df : sub_df["caption"].to_list()).to_dict()
import numpy as np
import utils
from numba import int64
from numba.typed import List
import numba_test
X = np.load("data/wiki/wiki.npy")
X_train, X_validation, X_test, test_list = utils.generate_dataset(X, 0.6)
numba_test.SGD_algorithm(X_train, 1e-4, 1e-4, 16, 1e-4, 100)
netG.load_state_dict(torch.load(args.netG))
print(netG)
if args.cuda:
netG.cuda()
###
# Create Generative validation set and Generative training set if necessary.
###
# If we need one day a validation set on generated data.
# print('Create Validation set with generated data ...')
# validloader_gen = utils.generate_dataset(netG, 10000, args.batchSize, args.workers, args.nz, n_class)
if (args.training_size != -1) and (args.train_real == False):
trainloader_gen = utils.generate_dataset(netG, args.training_size,
args.batchSize, args.workers,
args.nz, n_class)
if args.cuda:
model.cuda()
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
betas=(args.beta1, 0.999))
criterion = nn.CrossEntropyLoss()
def train(epoch, train_loader=None):
model.train()
if train_loader != None:
for batch_idx, (data, target) in enumerate(train_loader):
def create_score_map(test_target,
data_name,
config,
ratio,
model_path=None,
save_score_map=False,
save_image=False):
MODEL_DIR = '.\\logs'
# check config
utils.check_config(config)
# create model
model_inference = modellib.MaskRCNN(mode="inference",
config=config,
model_dir=MODEL_DIR)
if model_path is None:
model_path = model_inference.find_last()
print("Loading weights from ", model_path)
model_inference.load_weights(model_path, by_name=True)
# generate dataset
# 检查data是否包含small
if 'pad' in data_name:
_start, _end = re.search('pad', data_name).span()
data_pad = int(data_name[:_start].split('_')[-1])
else:
data_pad = 0
print('test_target: %s, data_pad: %s' % (test_target, data_pad))
dataset_test = utils.generate_dataset(test_target, data_name)
image_shape = utils.get_image_shape(test_target)
idsNum = len(dataset_test.image_ids)
# generate score_map_total
score_map_total = np.zeros(
(image_shape[1] // ratio, image_shape[0] // ratio,
config.NUM_CLASSES - 1),
dtype=np.float32)
for image_id in dataset_test.image_ids[:]:
if image_id % 50 == 0:
print('%s/%s' % (image_id, idsNum))
# generate image
image_path = dataset_test.image_info[image_id]['path']
image = skimage.io.imread(image_path)
# generate LMB
LMB_path = dataset_test.image_info[image_id]['LMB_path']
LMB = skimage.io.imread(LMB_path)
# generate x1,y1,interval
x1, y1, interval = image_path.split('\\')[-1].split('.')[-2].split(
'_')[2:5]
def _ratio(x):
return int(x) // ratio
x1_ratio, y1_ratio, interval_ratio = list(
map(_ratio, [x1, y1, interval]))
x1_ratio -= data_pad // ratio
y1_ratio -= data_pad // ratio
score_map_total_x1, score_map_total_y1 = x1_ratio, y1_ratio
score_map_total_x2 = score_map_total_x1 + interval_ratio
score_map_total_y2 = score_map_total_y1 + interval_ratio
score_map_x1, score_map_y1 = 0, 0
if x1_ratio < 0:
score_map_total_x1 = 0
score_map_x1 = -x1_ratio
if y1_ratio < 0:
score_map_total_y1 = 0
score_map_y1 = -y1_ratio
# predict results
results = model_inference.detect([image], verbose=0)
r = results[0]
for n, class_id in enumerate(r['class_ids']):
# generate mask_n
mask_n = r['masks'][:, :, n].astype(np.float32)
mask_n = mask_n * LMB
if ratio != 1:
mask_n = cv2.resize(mask_n, (interval_ratio, interval_ratio),
interpolation=cv2.INTER_NEAREST)
# generate score_map
score_n = r['scores'][n]
score_map = mask_n * score_n
# 取score_map_total和score_map对应pixel的较大值赋值到score_map_total
score_map_total[score_map_total_y1:score_map_total_y2,
score_map_total_x1:score_map_total_x2,
class_id - 1] = np.maximum(
score_map[score_map_y1:, score_map_x1:],
score_map_total[
score_map_total_y1:score_map_total_y2,
score_map_total_x1:score_map_total_x2,
class_id - 1])
# save phase
save_dir = '.\\merge_submit'
if not os.path.exists(save_dir):
os.mkdir(save_dir)
model_stamp = utils.create_model_stamp(model_path)
image_name = '%s_%ssmall_%spad' % (test_target, ratio, data_pad)
# save score_map_total as npy
if save_score_map:
npy_save_path = os.path.join(
save_dir, '%s_scoremap_%s.npy' % (image_name, model_stamp))
np.save(npy_save_path, score_map_total)
print(npy_save_path, 'has been saved succefully.')
# save and show score_map as png by class
if save_image:
rows = config.NUM_CLASSES - 1
fig, axs = plt.subplots(rows, 1, figsize=(16, 16 * rows))
if ratio == 1:
image_path = ".\\offical_data\\jingwei_round1_test_a_20190619\\%s.png" % test_target
else:
image_path = ".\\overview\\%s_%ssmall.png" % (test_target, ratio)
Image.MAX_IMAGE_PIXELS = None
image_frame_oringin = skimage.io.imread(image_path)
for class_id in range(1, config.NUM_CLASSES):
score_map = score_map_total[:, :, class_id - 1]
image_frame = image_frame_oringin.copy()
color = utils.COLOR_MAP[class_id]
for c in range(3):
image_frame[:, :, c] = image_frame[:, :, c] * (
1 - score_map) + score_map * color[c]
axs[class_id - 1].imshow(image_frame)
image_save_path = os.path.join(
save_dir,
'%s_class%d_%s.png' % (image_name, class_id, model_stamp))
skimage.io.imsave(image_save_path, image_frame)
print(image_save_path, 'has been saved succefully.')
return score_map_total, model_path
if __name__ == '__main__':
torch.manual_seed(7)
A, X, means, stds = load_metr_la_data()
split_line1 = int(X.shape[2] * 0.6)
split_line2 = int(X.shape[2] * 0.8)
train_original_data = X[:, :, :split_line1]
val_original_data = X[:, :, split_line1:split_line2]
test_original_data = X[:, :, split_line2:]
training_input, training_target = generate_dataset(
train_original_data,
num_timesteps_input=num_timesteps_input,
num_timesteps_output=num_timesteps_output)
val_input, val_target = generate_dataset(
val_original_data,
num_timesteps_input=num_timesteps_input,
num_timesteps_output=num_timesteps_output)
test_input, test_target = generate_dataset(
test_original_data,
num_timesteps_input=num_timesteps_input,
num_timesteps_output=num_timesteps_output)
A_wave = get_normalized_adj(A)
A_wave = torch.from_numpy(A_wave)
A_wave = A_wave.to(device=args.device)
args = parser.parse_args()
args.device = None
if args.enable_cuda and torch.cuda.is_available():
args.device = torch.device('cuda')
print("Use GPU.")
else:
args.device = torch.device('cpu')
print("Use CPU.")
if __name__ == '__main__':
torch.manual_seed(7)
A, X, means, stds = load_metr_la_data()
split_line2 = int(X.shape[2] * 0.8)
test_original_data = X[:, :, split_line2:]
test_input, test_target = generate_dataset(
test_original_data,
num_timesteps_input=num_timesteps_input,
num_timesteps_output=num_timesteps_output)
print("INFO: Test data load finish!")
A_wave = get_normalized_adj(A)
A_wave = torch.from_numpy(A_wave)
A_wave = A_wave.to(device=args.device)
net = STGCN(A_wave.shape[0], test_input.shape[3], num_timesteps_input,
num_timesteps_output).to(device=args.device)
net.load_state_dict(torch.load('parameter.pkl'))
print("INFO: Load model finish!")
loss_criterion = nn.MSELoss()
import utils
model_file = 'DeepPoemModel.h5'
checkpoint_dir = './training_checkpoints'
batch_size = 64
epochs = 50
learning_rate = 0.001
drop_rate = 0.1
word_vec_size = 256
rnn_size = 1024
rnn_layers = 3
dense_layers = 1
idx2word, word2idx, x_train, y_train = utils.generate_dataset(True)
vob_size = len(idx2word)
def create_model_cell():
input_data = layers.Input(shape=(None, ))
initial_state_1 = [
layers.Input(shape=(128, )),
layers.Input(shape=(128, ))
]
initial_state_2 = [
layers.Input(shape=(128, )),
layers.Input(shape=(128, ))
]
x = layers.Embedding(input_dim=vob_size,
output_dim=word_vec_size)(input_data)
np.save("val_cd.npy", X[:, :, split_line1:split_line2])
np.save("test_cd.npy", X[:, :, split_line2:])
means = np.mean(X[:, :, :split_line1], axis=(0, 2))
stds = np.std(X[:, :, :split_line1], axis=(0, 2))
X = X - means[0]
X = X / stds[0]
print(means)
print(stds)
print(X.shape)
train_original_data = X[:, :, :split_line1]
val_original_data = X[:, :, split_line1 - 1008:split_line2]
test_original_data = X[:, :, split_line2 - 1008:]
training_input, training_daily_input, training_weekly_input, training_coarse_input, training_target = generate_dataset(
train_original_data,
num_timesteps_input=num_timesteps_input,
num_timesteps_output=num_timesteps_output)
val_input, val_daily_input, val_weekly_input, val_coarse_input, val_target = generate_dataset(
val_original_data,
num_timesteps_input=num_timesteps_input,
num_timesteps_output=num_timesteps_output)
test_input, test_daily_input, test_weekly_input, test_coarse_input, test_target = generate_dataset(
test_original_data,
num_timesteps_input=num_timesteps_input,
num_timesteps_output=num_timesteps_output)
print(training_input.shape)
print(training_target.shape)
if args.model == 'Traj-net':
net = PCN(X.shape[0], training_input.shape[3], num_timesteps_input,
num_timesteps_output).to(device=args.device)
