def main():
# Reads in MNIST data, Initializes the Model, and Trains and Tests the Model for one Epoch
num_train = 60000
num_test = 10000
random_num = randint(0, num_test - 10)
train_inputs, train_labels = get_data(
'MNIST_data/train-images-idx3-ubyte.gz',
'MNIST_data/train-labels-idx1-ubyte.gz', num_train)
test_inputs, test_labels = get_data(
'MNIST_data/t10k-images-idx3-ubyte.gz',
'MNIST_data/t10k-labels-idx1-ubyte.gz', num_test)
# Creates and Trains Model
model = Model()
train(model, train_inputs, train_labels)
# Tests Accuracy
accuracy = test(model, test_inputs, test_labels)
print('accuracy =', accuracy)
# Randomly Take 10 Images for Visualization
input = test_inputs[random_num:random_num + 10, :]
label = test_labels[random_num:random_num + 10]
visualize_results(input, model.call(input), label)
def main():
# Create model
model = YOLO()
# For saving/loading models
checkpoint_dir = './checkpoints'
checkpoint = tf.train.Checkpoint(model=model)
manager = tf.train.CheckpointManager(checkpoint,
checkpoint_dir,
max_to_keep=3)
if args.restore_checkpoint or args.mode == 'test':
# restores the latest checkpoint using from the manager
checkpoint.restore(manager.latest_checkpoint).expect_partial()
if args.mode == 'train':
train_boxes, train_size = get_data('train')
# Train model
for epoch in range(model.num_epochs):
print(
'========================== EPOCH %d =========================='
% epoch)
train(model, train_boxes, train_size)
print("**** SAVING CHECKPOINT AT END OF EPOCH ****")
manager.save()
test_boxes, test_size = get_data('test')
test(model, test_boxes, test_size)
elif args.mode == 'test':
# Test model
test_boxes, test_size = get_data('test')
test(model, test_boxes, test_size)
return
def main():
'''
Read in CIFAR10 data (limited to 2 classes), initialize model, and train and
test model for a number of epochs.
:return: None
'''
train_inputs, train_labels = get_data('CIFAR_data_compressed/train', 3, 5)
test_inputs, test_labels = get_data('CIFAR_data_compressed/test', 3, 5)
model = Model()
t1 = time.time()
for epoch in range(10):
print("epoch", epoch)
train(model, train_inputs, train_labels)
t2 = time.time()
print("training took", t2 - t1)
acc = test(model, test_inputs, test_labels)
t3 = time.time()
print("testing took", t3 - t2)
print("accuracy", acc)
#visualize_results(test_inputs[0:10], model.call(test_inputs[0:10]), test_labels[0:10], "cat", "dog")
return
def main():
# Preprocess data
train_labels, train_images = get_data(args.train_csv_path, args.warp_size)
test_labels, test_images = get_data(args.test_csv_path, args.warp_size)
# Convert to tensors
train_labels = tf.convert_to_tensor(train_labels)
train_images = tf.convert_to_tensor(train_images)
test_labels = tf.convert_to_tensor(test_labels)
test_images = tf.convert_to_tensor(test_images)
# Instantiate, compile, and train
# One channel for our black and white images
input_shape = args.warp_size + (1, )
# TODO: How do we make training=False false later?
model = DenseNet(input_shape, args.growth_k, args.drop_rate, True)
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
model.fit(train_images,
train_labels,
batch_size=args.batch_size,
epochs=args.num_epochs,
validation_data=(test_images, test_labels))
def main():
'''
Read in MNIST data, initialize model, and train and test model for one
epoch. The number of training steps should be the number of batches there
are in a single epoch.
:return: None
'''
# TODO: load MNIST train and test examples
train_images, train_labels = get_data('MNIST_data/train-images-idx3-ubyte.gz',
'MNIST_data/train-labels-idx1-ubyte.gz',60000)
test_images, test_labels = get_data('MNIST_data/t10k-images-idx3-ubyte.gz',
'MNIST_data/t10k-labels-idx1-ubyte.gz', 10000)
# Create Model
model = Model()
# Train model by calling train() ONCE on all data
train(model, train_images, train_labels)
# Test the accuracy by calling test() after running train()
accuracy = test(model, test_images, test_labels)
print (accuracy)
# Visualize the data by using visualize_results()
# generate 10 random numbers between 0 and 10000
selection = np.random.randint(low=0, high=9999, size=10)
vis_images = test_images[selection,:]
vis_labels = test_labels[selection]
prob = model.call(vis_images)
visualize_results(vis_images, prob, vis_labels)
def main():
'''
Read in MNIST data, initialize your model, and train and test your model for one epoch.
The number of training steps should be the number of batches you run through in a single epoch.
You should receive a final accuracy on the testing examples of > 80%.
:return: None
'''
# TODO: load MNIST train and test examples into train_inputs, train_labels, test_inputs, test_labels
num_train = 60000
train_inputs, train_labels = \
get_data('MNIST_data/train-images-idx3-ubyte.gz', 'MNIST_data/train-labels-idx1-ubyte.gz', num_train)
num_test = 10000
test_inputs, test_labels = \
get_data('MNIST_data/t10k-images-idx3-ubyte.gz', 'MNIST_data/t10k-labels-idx1-ubyte.gz', num_test)
# TODO: Create Model
model = Model()
# TODO: Train model by calling train() ONCE on all data
train(model, train_inputs, train_labels)
# TODO: Test the accuracy by calling test() after running train()
accuracy = test(model, test_inputs, test_labels)
print('accuracy =', accuracy)
# TODO: Visualize the data by using visualize_results()
# take the first 10 images for visualization
input = test_inputs[0:10, :]
label = test_labels[0:10]
visualize_results(input, model.call(input), label)
def main():
#preprocess cifar dataset
train_inputs, train_labels = get_data("CIFAR_data_compressed/train", 3, 5)
test_inputs, test_labels = get_data("CIFAR_data_compressed/test", 3, 5)
#run model
model = Model()
for i in range(25):
holder = time.time()
train(model, train_inputs, train_labels)
def main():
# store data in folder called data
train_inp, train_lab = get_data("data/train", 3, 5)
test_inp, test_lab = get_data("data/test", 3, 5)
mod = Model()
for i in range(0, mod.epochs):
train(mod, train_inp, train_lab)
acc = test(mod, test_inp, test_lab)
print(acc)
return
def main():
# TO-DO: Pre-process and vectorize the data
# HINT: Please note that you are predicting the next word at each timestep, so you want to remove the last element
# from train_x and test_x. You also need to drop the first element from train_y and test_y.
# If you don't do this, you will see impossibly small perplexities.
# TO-DO: Separate your train and test data into inputs and labels
print("Main starts")
trainOut, test_data, notes_dict = get_data()
trainInputs = np.array(trainOut[:-1])
trainLabels = np.array(trainOut[1:])
# TODO: initialize model and tensorflow variables
# vocab_size, window_size, embedding_size, batch_size, rnn_size, hidden_layer
window_size = 20
embedding_size = 256
batch_size = 128
rnn_size = 512
hidden_layer = 512
model = Model(len(notes_dict), window_size, embedding_size, batch_size,
rnn_size, hidden_layer)
# TODO: Set-up the training step
lossArray, EW, LSTMLayerW, dense1W, dense2W = train(
model, trainInputs, trainLabels)
def main():
args = get_train_args()
X_train, X_test, X_val, y_train, y_test, y_val, group_vali, group_train = get_data(
args["data_path"])
# Now that we found the best hyperparameters, let's use them to train our model for a longer time.
gbm = lgb.LGBMRanker(
n_estimators=10000,
num_leaves=args["num_leaves"],
learning_rate=args["learning_rate"],
reg_lambda=args["reg_lambda"],
)
gbm.fit(
X_train,
y_train,
group=group_train,
eval_group=[group_vali],
eval_set=[(X_val, y_val)],
early_stopping_rounds=150,
)
gbm.booster_.save_model(args["output_file_name"],
num_iteration=gbm.best_iteration)
def main():
# TODO: Pre-process and vectorize the data
train_token, test_token, vocab_dict = get_data('data/train.txt', 'data/test.txt')
num_train = train_token.shape[0]
num_test = test_token.shape[0]
# HINT: Please note that you are predicting the next word at each timestep, so you want to remove the last element
# from train_x and test_x. You also need to drop the first element from train_y and test_y.
# If you don't do this, you will see very, very small perplexities.
# TODO: initialize model and tensorflow variables
vocab_size = 7342
model = Model(vocab_size)
# TODO: Separate your train and test data into inputs and labels
num_train = (num_train - 1) // model.window_size
num_test = (num_test - 1) // model.window_size
train_inputs = np.zeros((num_train, model.window_size), dtype=np.int32)
train_labels = np.zeros((num_train, model.window_size), dtype=np.int32)
for i in range(num_train):
train_inputs[i] = train_token[i * model.window_size: (i + 1) * model.window_size]
train_labels[i] = train_token[i * model.window_size + 1: (i + 1) * model.window_size + 1]
test_inputs = np.zeros((num_test, model.window_size), dtype=np.int32)
test_labels = np.zeros((num_test, model.window_size), dtype=np.int32)
for i in range(num_test):
test_inputs[i] = test_token[i * model.window_size: (i + 1) * model.window_size]
test_labels[i] = test_token[i * model.window_size + 1: (i + 1) * model.window_size + 1]
# TODO: Set-up the training step
train(model, train_inputs, train_labels)
# TODO: Set up the testing steps
Perplexity = test(model, test_inputs, test_labels)
print('Perplexity = {}'.format(Perplexity))
generate_sentence('I', 10, vocab_dict, model)
def train(model, img_dir, train_img_names, img_to_encodings, manager):
num_inputs = len(train_img_names)
steps = int(num_inputs / model.batch_size)
random.shuffle(train_img_names)
for i in range(0, steps):
start = i * model.batch_size
end = (i + 1) * model.batch_size
# now we load the actual content of the images, which is a huge amount of data
inputs, labels = get_data(img_dir, train_img_names[start:end],
img_to_encodings)
with tf.GradientTape() as tape:
probs = model(inputs)
loss = model.dice_loss(probs, labels)
gradients = tape.gradient(loss, model.trainable_variables)
model.optimizer.apply_gradients(
zip(gradients, model.trainable_variables))
if i % args.log_every == 0:
train_acc, train_iou = model.accuracy(probs, labels)
r = tf.reduce_mean(recall(probs, labels)).numpy()
p = tf.reduce_mean(precision(probs, labels)).numpy()
print(
"========>Step %2d, accuracy = %3.4f, loss = %3.4f, IoU = %3.4f, recall = %3.4f, precision = %3.4f"
% (i, train_acc, loss, train_iou, r, p))
if i % args.save_every == 0:
manager.save()
def test_preprocess_1000(self):
location = os.path.join('data', 'unit_test')
sample_images.sample_images(location, seed=123, sample_num=1000)
labelled, unlabelled = preprocess.get_data(
'data/unit_test/hirise-map-proj-v3_2')
self.assertTrue(198 == len(list(labelled)))
self.assertTrue(802 == len(list(unlabelled)))
def initialize():
print("initialize")
X, T = get_data()
T = T.astype(float)
W = np.random.randn(X.shape[1], 1)
b = 0
return X, T, W, b
def main():
train_ids, test_ids, vocab = get_data("../../data/train.txt",
"../../data/test.txt")
model = Model(len(vocab))
print("model initialized")
train_inputs = train_ids[:-1]
train_labels = train_ids[1:]
test_inputs = test_ids[:-1]
test_labels = test_ids[1:]
for epoch in range(1):
train(model, train_inputs, train_labels)
print("training complete")
checkpoint = tf.train.Checkpoint(optimizer=model.optimizer,
embedding=model.embedding,
lstm=model.lstm,
dense_1=model.dense_1,
dense_2=model.dense_2)
manager = tf.train.CheckpointManager(checkpoint, './ckpts', max_to_keep=10)
manager.save()
print("Train perplexity: " + str(test(model, train_inputs, train_labels)))
print("Test perplexity: " + str(test(model, test_inputs, test_labels)))
return
def main():
# preprocess inputs into (num inputs-2,2)
train_corpus, test_corpus, dictionary = get_data('data/train.txt',
'data/test.txt')
train_input = np.zeros((len(train_corpus) - 2, 2), dtype=np.int)
train_input[:, 0] = train_corpus[0:len(train_corpus) - 2]
train_input[:, 1] = train_corpus[1:len(train_corpus) - 1]
train_labels = train_corpus[2:len(train_corpus)]
# Separate train and test data into inputs and labels
test_input = np.zeros((len(test_corpus) - 2, 2), dtype=np.int)
test_input[:, 0] = test_corpus[0:len(test_corpus) - 2]
test_input[:, 1] = test_corpus[1:len(test_corpus) - 1]
test_labels = test_corpus[2:len(test_corpus)]
# initialize model and tensorflow variables
model = Model(len(dictionary))
# Set-up the training step
train(model, train_input, train_labels)
# Set up the testing steps
perp = test(model, test_input, test_labels)
# Print out perplexity
print('perplexity', perp)
def main():
# Evaluate model performance
# Get the "ideal" order of y_test by sorting in descending order.
args = parse.get_test_args()
X_train, X_test, X_val, y_train, y_test, y_val, group_vali, group_train = get_data(
args["data_path"])
gbm = lgb.Booster(model_file=args["model_path"])
true_relevance = y_test.sort_values(ascending=False)
# Get the actual order of y_test by sorting it according to our model's predictions.
test_pred = gbm.predict(X_test)
y_test = pd.DataFrame({
"relevance_score": y_test,
"predicted_ranking": test_pred
})
relevance_score = y_test.sort_values("predicted_ranking", ascending=False)
# Use computed variables to calculate the nDCG score
print(
"nDCG score: ",
ndcg_score([true_relevance.to_numpy()],
[relevance_score["relevance_score"].to_numpy()]),
)
def __init__(self, data_num=None):
self.model = tk.Sequential()
self.data = {'train': {}, 'test': {}}
self.tokens = None
self.history = None
fname = self.data_name + '.csv'
self.raw_data = pp.get_data(fname, lib.data[self.data_name], data_num)
def bad_visualization():
train_data, test_data, daily_data = get_data()
x = daily_data[0, 1, :]
print(x.shape)
for i in range(4):
plt.plot(daily_data[i, 1, :])
plt.show()
def main():
print("Preprocessing...")
train_inputs, train_labels, test_inputs, test_labels = preprocess.get_data(
"../data/fma_metadata/tracks.csv")
char2id, char_inputs = preprocess.make_char_dict(train_inputs)
_, test_char_inputs = preprocess.make_char_dict(test_inputs)
actor_char2id, actor_char_inputs = preprocess.make_char_dict(train_inputs)
_, actor_test_char_inputs = preprocess.make_char_dict(test_inputs)
model = Model(len(char2id), len(actor_char2id))
numerical_train, numerical_test = preprocess.make_numerical_lists(
train_inputs, test_inputs)
feature_train, feature_test = preprocess.make_feature_lists(
train_inputs, test_inputs)
print("Training...")
losses = []
for epoch in range(2):
losses.extend(
train(model, numerical_train, feature_train, char_inputs,
actor_char_inputs, train_labels))
print("Testing...")
test(model, numerical_test, feature_test, test_char_inputs,
actor_test_char_inputs, test_labels)
visualize_loss(losses)
def main():
# Parse command line arguments
host, port = parse_args()
# Create a grpc channel using the IP adress and the port
channel = implementations.insecure_channel(host, int(port))
# Create a stub
stub = prediction_service_pb2.beta_create_PredictionService_stub(channel)
# Name of the data that should be send to the server
file = '../data/raw/creditcard_dataset_fraud.csv'
data = get_data(data_type="test", data_dir=file)
sample_df = data.sample(20)
for _, row in sample_df.iterrows():
input_data = [[s for s in row]]
# print(input_data)
# Create a request object
request = predict_pb2.PredictRequest()
# Name of the model running on the tensorflow_model_server (either locally or in Docker container)
request.model_spec.name = 'anamoly_detection'
# Name of the defined prediction signature in the SavedModelInstance on the server (either locally or in Docker container)
request.model_spec.signature_name = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
# Make a request (time-out after 20 seconds)
request.inputs['inputs'].CopyFrom(
make_tensor_proto(input_data, shape=[1, 29]))
result = stub.Predict(request, 20.0) # 60 secs timeout
print(result)
def main():
# TODO: Pre-process and vectorize the data using get_data from preprocess
train_token, test_token, vocab_dict = get_data('data/train.txt',
'data/test.txt')
num_train = train_token.shape[0]
num_test = test_token.shape[0]
# TODO: Separate your train and test data into inputs and labels
train_inputs = np.zeros((num_train, 2), dtype=np.int32)
train_labels = np.zeros((num_train, 1), dtype=np.int32)
for i in range(num_train - 2):
train_inputs[i, :] = train_token[i:i + 2]
train_labels[i] = train_token[i + 2]
test_inputs = np.zeros((num_test, 2), dtype=np.int32)
test_labels = np.zeros((num_test, 1), dtype=np.int32)
for i in range(num_test - 2):
test_inputs[i, :] = test_token[i:i + 2]
test_labels[i] = test_token[i + 2]
# TODO: initialize model and tensorflow variables
vocab_size = 7342
model = Model(vocab_size)
# TODO: Set-up the training step
train(model, train_inputs, train_labels)
# TODO: Set up the testing steps
perplexity = test(model, test_inputs, test_labels)
print('Perplexity = {}'.format(perplexity))
# Print out perplexity
word1 = 'I'
word2 = 'like'
length = 10
generate_sentence(word1, word2, length, vocab_dict, model)
def main():
model = ResNet16()
train_images, train_labels, test_images, test_labels = get_data()
accuracy_list = []
matplotlib.use('Agg')
label_names = {
"0": "airplane",
"1": "automobile",
"2": "bird",
"3": "cat",
"4": "deer",
"5": "dog",
"6": "frog",
"7": "horse",
"8": "ship",
"9": "truck"
}
for i in range(model.epochs):
print(i)
train(model, train_images, train_labels)
test_accuracy = test(model, test_images, test_labels)
accuracy_list.append(test_accuracy)
print(test_accuracy)
visualize_loss_accuracy(model.loss_list, accuracy_list)
visualize_results(test_images[0:10], model(test_images[0:10]),
test_labels[0:10], label_names)
def main():
train_inputs, train_labels, test_inputs, test_labels, vocab_dict = get_data(
'data_set/train.csv', 'data_set/test.csv', 'data_set/test_labels.csv')
model = Model(len(vocab_dict), train_inputs.shape[1])
train(model, train_inputs, train_labels)
accuracy, roc_score = test(model, test_inputs, test_labels)
print("Accuracy: ", accuracy)
print("roc_score: ", roc_score)
visualize_embeddings(model, vocab_dict)
def train():
logging.info("Start training.")
# get data
trainloader = get_data(config.data_path)
# get HRNet
net = cls_net()
# define a optimizer
optimizer = optim.Adam(net.parameters(), lr=config.lr)
# load model if exists
if os.path.exists(config.model_load_path) is not True:
criterion = nn.CrossEntropyLoss()
else:
checkpoint = torch.load(path)
net.load_state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
initepoch = checkpoint['epoch']
criterion = checkpoint['loss']
# train
total = correct = 0
for epoch in range(config.epoch):
timestart = time.time()
running_loss = 0.0
for i, data in enumerate(trainloader, 0):
inputs, labels = data
optimizer.zero_grad()
# forward, backward and step
outputs = net(inputs)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
running_loss += loss.item()
if i % 200 == 199:
logging.info('[%d, %5d] loss: %.3f' %
(epoch + 1, i + 1, running_loss / 200))
running_loss = 0.0
# get accuaracy of train dataset
_, predicted = torch.max(outputs.data, 1)
total += labels.size(0)
correct += (predicted == labels).sum().item()
logging.info('Accuracy of the network on the %d train images: %.3f %%' % (total,
100.0 * correct / total))
# save the model
torch.save({'epoch':epoch,
'model_state_dict':net.state_dict(),
'optimizer_state_dict':optimizer.state_dict(),
'loss':criterion
}, config.model_save_path)
logging.info('epoch %d cost %3f sec' %(epoch,time.time()-timestart))
logging.info('Finished Training')
def __init__(self, fold=5, rotate=False, crop=False):
self.fold = fold
(self.X, self.Y) = get_data(reverse=True,
crop=crop,
rotate_extend=rotate)
self.sample_num = self.X.shape[0]
self.blockSize = round(self.sample_num / self.fold)
self.I = np.arange(self.sample_num)
for i in range(3):
random.shuffle(self.I)
def main():
"""
the main function for creating, training, saving, and testing the RNN model
return: nothing
"""
SAVE = True # whether to save the transformer model parameters
RESUME = False # resume from a saved model
SAVED_DATA = True # whether the preprocessed data is saved
PATH_TO_MODEL = "rnn_model"
# tickers = ["AAPL", "AMZN", "GOOGL", "MSFT", "CVS", "DIS", "FDX", "JPM", "REGN", "WMT", "JNJ", "HON"] # all stocks
tickers = ["AAPL", "AMZN"]
train_data, test_data, _ = get_data(
tickers) # data of shape (num_stocks, num_days, datum_size)
past_num = 50 # the number of days the model looks at to predict the next stock price
# create model
if RESUME:
model = load_model(PATH_TO_MODEL)
else:
model = rnn_create()
# pre-process data
print("========== beginning to pre-process data ==========")
train_x_path = "data/new_rnn_train_x.npy"
train_y_path = "data/new_rnn_train_y.npy"
test_x_path = "data/new_rnn_test_x.npy"
test_y_path = "data/new_rnn_test_y.npy"
if SAVED_DATA:
print("loading saved data")
train_x, train_y = np.load(train_x_path), np.load(train_y_path)
test_x, test_y = np.load(test_x_path), np.load(test_y_path)
else:
train_x, train_y = process_data(train_data, past_num, train_x_path,
train_y_path)
test_x, test_y = process_data(test_data, past_num, test_x_path,
test_y_path)
print("========== finished pre-process data ==========")
# train the model
NUM_EPOCH = 300
print("========== begin training ==========")
train_history = rnn_train(model, train_x, train_y, NUM_EPOCH)
print("========== finish training ==========")
if SAVE:
save_model(model, PATH_TO_MODEL)
print("====== saved model =======")
# evaluate the model
print("========== begin testing ==========")
result = rnn_test(model, test_x, test_y)
print("printing results")
print(model.metrics_names)
print(result)
print("========== finished testing ==========")
def main_predict():
lst = [
1768, 1555, 894, 674, 65, 54, 137, 327, 354, 492, 553, 634, 844, 928,
1054, 1118, 1228, 1449, 1474, 1483, 1504, 1529, 1559, 1733, 1881, 1917
]
data = get_data("products_sentiment_train.tsv",
"products_sentiment_test_copy.tsv",
balance=True,
drop_lst=lst)
train_size = (np.array(data[0]["text"])).shape[0]
X_full, Y_full = vectorizer(np.append(np.array(data[0]["text"]), np.array(data[1]["text"])),
tokenizer=tokenize,
ngram_range=(1, 4),
max_df=0.85,
min_df=1,
max_features=None), \
np.array(data[0]["label"])
network_class = KerasNeuralNetwork(hidden_layer_sizes=(400, 200, 10),
nonlin_functions=("tanh", "tanh",
"tanh"),
dropout_coef=(0.7, 0.7, 0.7))
X_train, Y_train, Y_flat_train = network_class.text_process_data(
X_full[:train_size, :], Y=Y_full)
neural_network = network_class.init_model(D=X_train.shape[1],
K=Y_train.shape[1],
loss='categorical_crossentropy',
optimizer=keras.optimizers.Adam(
lr=0.0001,
beta_1=0.9,
beta_2=0.99,
epsilon=1e-08,
decay=0.0),
metrics=['accuracy'])
r = network_class.fit_network(X_train,
Y_train,
neural_network,
epochs=30,
batch=130,
show=False)
X_predict = network_class.text_process_data(X_full[train_size:, :])
result = network_class.make_prediction(X_predict,
neural_network,
batch=130)
with open("keras_adam.csv", 'w') as f_out:
f_out.write(
pd.DataFrame(pd.Series(map(str, range(0, 500))).str.cat(list(
map(str, result)),
sep=','),
columns=["Id,y"]).to_csv(sep=" ", index=False))
def main():
parser = ArgumentParser()
parser.add_argument("--epochs", type=int, default=10000, help="number of epochs to run the model")
parser.add_argument("--checkpoint_folder", type=str, default="./checkpoints",
help="folder to save the model checkpoints to")
parser.add_argument("--print_every", type=int, default=1000, help="how many batches between each report")
opt = parser.parse_args()
iter_train, iter_validate, iter_test = get_data(BATCH_SIZE)
input_dim = len(DE_FIELD.vocab)
output_dim = len(EN_FIELD.vocab)
device = "cuda" if torch.cuda.is_available() else "cpu"
sos_token = DE_FIELD.vocab.stoi["<SOS>"]
translator = Translator(ENCODER_HIDDEN, DECODER_HIDDEN, input_dim, output_dim, EMBEDDING_DIM_ENCODER,
EMBEDDING_DIM_DECODER, sos_token=sos_token, device=device).to(device)
cross_entropy = torch.nn.CrossEntropyLoss(ignore_index=EN_FIELD.vocab.stoi["<PAD>"])
optimizer = torch.optim.Adam(translator.parameters(), lr=0.001)
epoch = load_model(translator, opt.checkpoint_folder)
total_iter = 0
total_loss = 0
print("running...")
for epoch_ in range(epoch, epoch + opt.epochs):
for i, batch in enumerate(iter_train):
if i * BATCH_SIZE < NUMBER_OF_EXAMPLES:
try:
optimizer.zero_grad()
source = batch.src.to(device)
target = batch.trg.to(device)[1:, :] # without <SOS> token
output = translator(source, target.shape[0])
try:
loss = cross_entropy(output.reshape(-1, output_dim), target.view(-1))
loss.backward()
optimizer.step()
total_loss += loss.item()
total_iter += 1
if total_iter % opt.print_every == 0:
print(f"iter #{total_iter} loss: {total_loss / opt.print_every}")
total_loss = 0
test_batch_ = None
for j, batch_validate in enumerate(iter_validate):
test_batch_ = batch_validate
if j == (total_iter / opt.print_every) % 50:
break
test_batch(DE_FIELD, EN_FIELD, translator, test_batch_, device=device, max_examples=5)
del test_batch_
save_model(translator, opt.checkpoint_folder, epoch_)
finally:
del source, target, output, loss
except Exception as e:
print(e)
sleep(120)
else:
break
def unit_test_visualization():
train, test, _ = get_data()
print(f'shape: {train.shape}')
print(train[:, :10, :])
visualize_linegraph(train[0, :, 0])
visualize_linegraph(train[1, :, 0])
tickers = ['AAPL', 'AMZN', 'MSFT', 'INTC', 'REGN', 'CASH']
portfolio = [[1, 2, 3, 2, 1], [2, 3, 4, 3, 1], [3, 2, 1, 4, 2],
[5, 9, 2, 1, 8], [1, 3, 2, 2, 3], [4, 3, 1, 1, 4]]
visualize_portfolio(portfolio, tickers)
def main():
# get dataset
data, label, ip = preprocess.get_data("../data/")
data = data[:, :3]
datasize, numfeat = data.shape
# set variables
x = tf.placeholder("float", shape=[None, numfeat])
y = tf.placeholder("float", shape=[None, 1])
wgt = tf.Variable(tf.zeros([numfeat, 1]))
bias = tf.Variable(tf.zeros([1]))
yraw = tf.matmul(x, wgt) + bias
# initialize variables
init = tf.initialize_all_variables()
# set optimization function
reg_loss = 0.5 * tf.reduce_sum(tf.square(wgt))
hng_loss = svmC * tf.reduce_sum(tf.maximum(tf.zeros([tf.minimum(BATCH_SIZE, REAL_SIZE), 1]), 1 - y*yraw))
svm_loss = reg_loss + hng_loss
min_loss = tf.train.GradientDescentOptimizer(0.01).minimize(svm_loss)
# set prediction function
pred = tf.sign(yraw)
corr = tf.equal(y, pred)
pred_type = tf.reduce_mean(tf.cast(corr, "float"))
# run everything
with tf.Session() as sess:
sess.run(init)
# before training
print sess.run(pred_type, feed_dict={x:data, y:label})
# training
for step in xrange(EPOCHS * datasize / BATCH_SIZE):
offset = (step * BATCH_SIZE) % datasize
batch_data = data[offset:offset+BATCH_SIZE]
batch_label = label[offset:offset+BATCH_SIZE]
realsize = batch_label.shape[0]
sess.run(min_loss, feed_dict={x:batch_data, y:batch_label, REAL_SIZE:realsize})
# after training
print sess.run(pred_type, feed_dict={x:data, y:label})
def prep(self,
docs_path='../data/TAC_2014_BiomedSumm_Training_Data',
json_data_path='../data/v1-2a.json'):
data = get_data(docs_path, json_data_path)
train_set = {}
for tid in data:
train_set[tid] = []
# citation number
for cit in data[tid]:
offsets = []
ref_art = ''
for ann in data[tid][cit].values():
for off in ann['ref_offset']:
offsets.append(off)
query = ann['cit_text']
ref_art = ann['ref_art']
# union of all annotators reference offsets
offsets = union(offsets)
doc_type = tid.lower() + '_' + ref_art.lower()[:-4]
d = self._prep_data(clean(query), doc_type, offsets)
train_set[tid].append(
d)
return train_set
def prep(self,
docs_path='../data/TAC_2014_BiomedSumm_Training_Data',
json_data_path='../data/v1-2a.json'):
'''
Converts the raw data into a list of sentences for each
Topic.
Args:
docs_path (str), json_data_path (str)
Returns:
dict
kes: topic_ids - UPPER CASE: e.g. D1410_TRAIN
value: (list of tuples) - a list of training tuples
for format see _prep_data
'''
data = get_data(docs_path, json_data_path)
train_set = {}
for tid in data:
train_set[tid] = []
# citation number
for cit in data[tid]:
offsets = []
ref_art = ''
for ann in data[tid][cit].values():
for off in ann['ref_offset']:
offsets.append(off)
query = ann['cit_text']
ref_art = ann['ref_art']
# union of all annotators reference offsets
offsets = union(offsets)
doc_type = tid.lower() + '_' + ref_art.lower()[:-4]
d = self._prep_data(clean(query), doc_type, offsets)
train_set[tid].extend(
d)
return train_set
rest = res[1:]
for max_sequence_length, loss, acc in rest:
print('%d\t%f\t%f' % (max_sequence_length, loss, acc), end='\t')
print()
print()
if __name__ == '__main__':
print('Indexing word vectors.')
embeddings_index = get_embeddings_index()
print('Collected %s word vectors.' % len(embeddings_index))
print()
print('Loading data')
categories, data_train, data_test = get_data(True, False, True)
all_unproc_X_train, all_unproc_X_test = data_train.data, data_test.data
all_y_train, y_test = data_train.target, data_test.target
if args.frac:
unproc_X_train, y_train = get_frac(args.frac, all_unproc_X_train, all_y_train)
all_unproc_X_test, y_test = get_frac(args.frac, all_unproc_X_test, y_test)
else:
unproc_X_train = all_unproc_X_train
y_train = all_y_train
#texts, labels, labels_index = get_data_locally()
print('Preprocessing training text')
texts_train, avg_length, max_length = preproc(unproc_X_train)
def main():
# get dataset
data, label, ip = preprocess.get_data("../data/")
data = data[:, :3]
datasize, numfeat = data.shape
import som
import numpy as np
import preprocess
from matplotlib import pyplot as plt
#Training inputs for RGBcolors
data, label, ip = preprocess.get_data("../data/")
color_names = ['black', 'blue', 'darkblue', 'skyblue',
'greyblue', 'lilac', 'green', 'red', 'cyan', 'violet', 'yellow', 'white', 'darkgrey', 'mediumgrey', 'lightgrey'
]
datasize, numfeat = data.shape
#Train a 20x30 SOM with 400 iterations
som = som.SOM(30, 30, numfeat, 400)
som.train(data)
exit()
#Get output grid
image_grid = som.get_centroids()
#Map colours to their closest neurons
mapped = som.map_vects(colors)
#Plot
plt.imshow(image_grid)
plt.title('Color SOM')
for i, m in enumerate(mapped):
plt.text(m[1], m[0], color_names[i], ha='center', va='center',
