def main():
logger_initializing(CONFIG["path_to_logger"])
logger = logging.getLogger(__name__)
try:
log_processing = LogProcessing(BASH_ARGUMENTS["path_to_zip"])
logs = log_processing.read_zip_file()
result_logs = log_processing.parse_logs(logs)
save_result(result_logs, PATH_TO_RESULT)
if BASH_ARGUMENTS["print"]:
print(result_logs)
except Exception as e:
logger.debug(str(e))
raise e
def main():
"""
(1) load training and test data
(2) create model based on the settings in hyperparameter
(3) train the model
(4) save the result from training
(5) test the model
"""
# result path
if not os.path.exists(hp.RESULT_PATH):
os.makedirs(hp.RESULT_PATH)
# data path
enc_train = hp.ENC_TRAIN_PATH
dec_train = hp.DEC_TRAIN_PATH
enc_test = hp.ENC_TEST_PATH
dec_test = hp.DEC_TEST_PATH
# preprocess data
train_dec, test_dec, train_enc, test_enc, dec_vocab, enc_vocab, dec_padding_index = get_data(
enc_train, dec_train, enc_test, dec_test)
# initialize the linear transformer model
model = LinformerEncDec(
enc_num_tokens=len(enc_vocab), # encoder setting
enc_input_size=hp.INPUT_SIZE,
enc_channels=hp.CHANNEL,
enc_full_attention=hp.FULL_ATTENTION,
enc_dim_k=hp.DIM_K,
enc_dim_d=hp.DIM_D,
enc_dim_ff=hp.DIM_FF,
enc_depth=hp.DEPTH,
enc_parameter_sharing=hp.PARAMETER_SHARING,
dec_num_tokens=len(dec_vocab), # decoder setting
dec_input_size=hp.INPUT_SIZE,
dec_channels=hp.CHANNEL,
dec_full_attention=hp.FULL_ATTENTION,
dec_dim_k=hp.DIM_K,
dec_dim_d=hp.DIM_D,
dec_dim_ff=hp.DIM_FF,
dec_depth=hp.DEPTH,
dec_parameter_sharing=hp.PARAMETER_SHARING,
learning_rate=hp.LEARNING_RATE,
activation="relu")
# train the model
result_list = train(model, train_enc, train_dec, dec_padding_index)
# save result tuple {update_id, perplexity, accuracy}
save_result(result_list)
def main():
test = pd.read_csv("../data/test.csv")
train = pd.read_csv("../data/train.csv")
test = preprocess_time(test)
train = preprocess_time(train)
# KNN regressor start_end_Lon_lat grid_search
params = {
"n_neighbors": [10, 20, 30],
"leaf_size": [30, 50, 70],
"p": [1, 2, 3],
"weights": ["uniform", "distance"]
}
model = GridSearchCV(KNeighborsRegressor(),
params,
cv=5,
verbose=2,
scoring="neg_mean_squared_log_error",
n_jobs=-1)
print model
min_max_scaler = MinMaxScaler()
train_x = train[[
"pickup_longitude", "pickup_latitude", "dropoff_longitude",
"dropoff_latitude"
]].values
train_y = train["trip_duration"].values
test_x = test[[
"pickup_longitude", "pickup_latitude", "dropoff_longitude",
"dropoff_latitude"
]].values
min_max_scaler.fit(np.concatenate([train_x, test_x]))
transform_train_x = min_max_scaler.transform(train_x)
transform_test_x = min_max_scaler.transform(test_x)
# model = KNeighborsRegressor()
model.fit(transform_train_x, train_y)
print "best_params:", model.best_params_
print "best_score", model.best_score_
predicts = model.predict(transform_test_x)
test["trip_duration"] = predicts
save_result(test[["id", "trip_duration"]],
"../result/start_end_Lon_Lat_KNN_gridsearch_avg.csv")
def queue_listener(queue, c, timestamp, num_iterations_total):
'''listens for messages on the queue, writes to file. '''
count_iterations_total = 0
while True:
msg = queue.get()
if msg == 'kill':
break
# if not, msg is latest result
count_iterations_total += 1
results_file = save_result(msg, c['RESULTS_DIR'], c['RESULT_FORMAT'],
c['NAME'], timestamp)
logger.info(f"Total progress: [ {count_iterations_total} / {num_iterations_total} ] " \
f"--> Results saved to [ {results_file} ]")
from util import load_train_data, load_test_data, save_result
from keras.utils import np_utils
from keras.layers import Dense, Input
from keras.models import Model
train_file = os.path.join('data', 'train.csv')
test_file = os.path.join('data', 'test.csv')
x_train, y_train = load_train_data(train_file)
x_test = load_test_data(test_file)
batch_size = 100
nb_epoch = 20
hidden_units_1 = 256
hidden_units_2 = 100
y_train = np_utils.to_categorical(y_train)
input_layer = Input(shape=(784, ))
hidden_layer_1 = Dense(hidden_units_1, activation='sigmoid')(input_layer)
hidden_layer_2 = Dense(hidden_units_2, activation='sigmoid')(hidden_layer_1)
output_layer = Dense(10, activation='softmax')(hidden_layer_2)
model = Model(input_layer, output_layer)
model.compile(optimizer='sgd', loss='categorical_crossentropy')
model.fit(x_train, y_train, epochs=nb_epoch, batch_size=batch_size)
y_pred = np.argmax(model.predict(x_test), axis=-1)
print(y_pred)
save_file = os.path.join('result', 'mlp_keras.csv')
save_result(y_pred, file_name=save_file)
dev_gold_graphs.extend(batch.graphs)
dev_pred_graphs.extend(graphs)
dev_sent_ids.extend(batch.sent_ids)
dev_tokens.extend(batch.tokens)
progress.close()
dev_scores = score_graphs(dev_gold_graphs, dev_pred_graphs,
relation_directional=config.relation_directional)
for task in tasks:
if dev_scores[task]['f'] > best_dev[task]:
best_dev[task] = dev_scores[task]['f']
if task == 'role':
print('Saving best role model')
torch.save(state, best_role_model)
best_dev_role_model = True
save_result(dev_role_result_file,
dev_gold_graphs, dev_pred_graphs, dev_sent_ids,
dev_tokens)
if task == 'trigger':
print('Saving best event model')
torch.save(state, best_event_model)
best_dev_event_model = True
save_result(dev_event_result_file,
dev_gold_graphs, dev_pred_graphs, dev_sent_ids,
dev_tokens)
if task == 'relation':
print('Saving best relation model')
torch.save(state, best_relation_model)
best_dev_relation_model = True
save_result(dev_relation_result_file,
dev_gold_graphs, dev_pred_graphs, dev_sent_ids,
dev_tokens)
import os
import sys
import logging
import pandas as pd
import numpy as np
from util import load_train_data, load_test_data, save_result
train_file = os.path.join('data', 'train.csv')
test_file = os.path.join('data', 'test.csv')
x_train, y_train = load_train_data(train_file)
x_test = load_test_data(test_file)
from sklearn.svm import SVC
# C Penalty parameter C of the error term
# kernel Specifies the kernel type to be used in the algorithm. It must be one of ‘linear’, ‘poly’, ‘rbf’, ‘sigmoid’, ‘precomputed’ (default=rbf)
clf = SVC()
save_file = os.path.join('result', 'svm.csv')
clf.fit(x_train, y_train)
y_pred = clf.predict(x_test)
save_result(y_pred, save_file)
metrics.macro_rec,
metrics.macro_fscore))
print('Micro: P: {:.2f}, R: {:.2f}, F:{:.2f}'.format(
metrics.micro_prec,
metrics.micro_rec,
metrics.micro_fscore))
# Save model
if metrics.accuracy > best_scores['best_acc_dev']:
best_acc = True
best_scores['best_acc_dev'] = metrics.accuracy
if metrics.macro_fscore > best_scores['best_mac_dev']:
best_mac = True
best_scores['best_mac_dev'] = metrics.macro_fscore
print('Saving new best macro F1 model')
torch.save(state, model_mac)
save_result(results, label_itos, result_dev)
if metrics.micro_fscore > best_scores['best_mic_dev']:
best_mic = True
best_scores['best_mic_dev'] = metrics.micro_fscore
print('Saving new best micro F1 model')
torch.save(state, model_mic)
# Test set
results = defaultdict(list)
for batch in test_set.all_batches(label_size, batch_size=10, gpu=gpu, dev=True):
elmo_ids, labels, men_masks, ctx_masks, dists, gathers, men_ids = batch
preds = model.predict(elmo_ids, men_masks, ctx_masks, dists,
gathers)
results['gold'].extend(labels.int().data.tolist())
results['pred'].extend(preds.int().data.tolist())
results['ids'].extend(men_ids)
rules_data[endpoint.find('label')['for']] = 'on'
client.post(url_rules, data=rules_data, headers=dict(Referer=url_rules))
def measure_execution_time(host, page, n=100):
""" Call a certain page n times and returns the execution time (in ms) """
data = []
for _ in range(n):
now = time.time()
try:
requests.get(host + page)
except Exception:
print('Can\'t open url {}{}'.format(host, page))
data.append((time.time() - now) * 1000)
return data
if __name__ == '__main__':
host, name = parse_args()
sleep_until_ready(host)
print('\nHost is up.')
if name == 'with_dashboard':
print('Enabling monitoring of all endpoints...')
monitor_all_endpoints(host)
print('All endpoints are now monitored.')
print('Testing the overhead now...')
data = measure_execution_time(host, page='available_languages')
save_result(data, name + '.txt')
print('Results saved.')
def run_model_on_beatles(train,
model_name,
chords=None,
files=None,
data_independent=False):
"""
:param train: function to train the model. Must accept a parameter called chromagram_data
:param model_name: Name of the model
:param chords: Set to not None to display the chords mean/cov images
:param files: files to train on
:param data_independent: Set to True if training does not require data
:return: None
"""
if files is None:
files = [
'cqt_512', 'cqt_1024_complete', 'cqt_256_complete'
'stft', 'stft_0.5_pow', 'stft_1.5_pow', 'stft_2_pow',
'cqt_512_hop_2_tol', 'cqt_1024'
]
for f in files:
type_ = ''
if 'cqt' in f:
type_ = 'CQT'
else:
type_ = 'STFT'
hop_length = 512
if '1024' in f:
hop_length = 1024
elif '8192' in f:
hop_length = 8192
base_name = '{}_{}_{}'.format(model_name, type_, hop_length)
title = '{} w/ {} Chromagram \nHop Length = {}'.format(
model_name, type_, hop_length)
if '_tol' in f:
idx = f.index('_tol')
base_name = base_name + '_' + str(f[idx - 1]) + '_tol'
title = title + ' 0.{} Tolerance'.format(f[idx - 1])
if 'pow' in f:
idx = f.index('_pow')
try:
num = float(f[idx - 3:idx])
except Exception:
num = float(f[idx - 1])
base_name = base_name + '_' + str(num) + '_pow'
title = title + ' {} power'.format(num)
chromagram_data = load_data(f)
chromagram_data = util.remove_song([
'10CD1_-_The_Beatles/06 - The Continuing Story of Bungalow Bill.flac',
'10CD1_-_The_Beatles/05 - Wild Honey Pie.flac'
], chromagram_data)
del chromagram_data['err']
if not data_independent:
test_data, train_data = util.split_data(chromagram_data, 0.15)
else:
test_data = chromagram_data
train_data = chromagram_data
model = train(chromagram_data=train_data)
evaluation = util.evaluate(model, test_data)
util.save_result('{}.json'.format(base_name), evaluation)
print(evaluation)
util.display_err_matrix(matrix=evaluation['err_matrix'],
title=title,
file_name='{}.png'.format(base_name))
if chords is not None:
s = util.bucket_sort(train_data['labels'],
train_data['annotated_chromas'])
mean = util.mean_matrix(s)
cov = util.cov_matrix(s)
for c in util.CHORDS:
util.display_mean_cov_for_chord(c,
mean[util.CHORD_IDX[c]],
cov[util.CHORD_IDX[c]],
file_name=base_name + '_' + c)
