def main(): logger_initializing(CONFIG["path_to_logger"]) logger = logging.getLogger(__name__) try: log_processing = LogProcessing(BASH_ARGUMENTS["path_to_zip"]) logs = log_processing.read_zip_file() result_logs = log_processing.parse_logs(logs) save_result(result_logs, PATH_TO_RESULT) if BASH_ARGUMENTS["print"]: print(result_logs) except Exception as e: logger.debug(str(e)) raise e
def main(): """ (1) load training and test data (2) create model based on the settings in hyperparameter (3) train the model (4) save the result from training (5) test the model """ # result path if not os.path.exists(hp.RESULT_PATH): os.makedirs(hp.RESULT_PATH) # data path enc_train = hp.ENC_TRAIN_PATH dec_train = hp.DEC_TRAIN_PATH enc_test = hp.ENC_TEST_PATH dec_test = hp.DEC_TEST_PATH # preprocess data train_dec, test_dec, train_enc, test_enc, dec_vocab, enc_vocab, dec_padding_index = get_data( enc_train, dec_train, enc_test, dec_test) # initialize the linear transformer model model = LinformerEncDec( enc_num_tokens=len(enc_vocab), # encoder setting enc_input_size=hp.INPUT_SIZE, enc_channels=hp.CHANNEL, enc_full_attention=hp.FULL_ATTENTION, enc_dim_k=hp.DIM_K, enc_dim_d=hp.DIM_D, enc_dim_ff=hp.DIM_FF, enc_depth=hp.DEPTH, enc_parameter_sharing=hp.PARAMETER_SHARING, dec_num_tokens=len(dec_vocab), # decoder setting dec_input_size=hp.INPUT_SIZE, dec_channels=hp.CHANNEL, dec_full_attention=hp.FULL_ATTENTION, dec_dim_k=hp.DIM_K, dec_dim_d=hp.DIM_D, dec_dim_ff=hp.DIM_FF, dec_depth=hp.DEPTH, dec_parameter_sharing=hp.PARAMETER_SHARING, learning_rate=hp.LEARNING_RATE, activation="relu") # train the model result_list = train(model, train_enc, train_dec, dec_padding_index) # save result tuple {update_id, perplexity, accuracy} save_result(result_list)
def main(): test = pd.read_csv("../data/test.csv") train = pd.read_csv("../data/train.csv") test = preprocess_time(test) train = preprocess_time(train) # KNN regressor start_end_Lon_lat grid_search params = { "n_neighbors": [10, 20, 30], "leaf_size": [30, 50, 70], "p": [1, 2, 3], "weights": ["uniform", "distance"] } model = GridSearchCV(KNeighborsRegressor(), params, cv=5, verbose=2, scoring="neg_mean_squared_log_error", n_jobs=-1) print model min_max_scaler = MinMaxScaler() train_x = train[[ "pickup_longitude", "pickup_latitude", "dropoff_longitude", "dropoff_latitude" ]].values train_y = train["trip_duration"].values test_x = test[[ "pickup_longitude", "pickup_latitude", "dropoff_longitude", "dropoff_latitude" ]].values min_max_scaler.fit(np.concatenate([train_x, test_x])) transform_train_x = min_max_scaler.transform(train_x) transform_test_x = min_max_scaler.transform(test_x) # model = KNeighborsRegressor() model.fit(transform_train_x, train_y) print "best_params:", model.best_params_ print "best_score", model.best_score_ predicts = model.predict(transform_test_x) test["trip_duration"] = predicts save_result(test[["id", "trip_duration"]], "../result/start_end_Lon_Lat_KNN_gridsearch_avg.csv")
def queue_listener(queue, c, timestamp, num_iterations_total): '''listens for messages on the queue, writes to file. ''' count_iterations_total = 0 while True: msg = queue.get() if msg == 'kill': break # if not, msg is latest result count_iterations_total += 1 results_file = save_result(msg, c['RESULTS_DIR'], c['RESULT_FORMAT'], c['NAME'], timestamp) logger.info(f"Total progress: [ {count_iterations_total} / {num_iterations_total} ] " \ f"--> Results saved to [ {results_file} ]")
from util import load_train_data, load_test_data, save_result from keras.utils import np_utils from keras.layers import Dense, Input from keras.models import Model train_file = os.path.join('data', 'train.csv') test_file = os.path.join('data', 'test.csv') x_train, y_train = load_train_data(train_file) x_test = load_test_data(test_file) batch_size = 100 nb_epoch = 20 hidden_units_1 = 256 hidden_units_2 = 100 y_train = np_utils.to_categorical(y_train) input_layer = Input(shape=(784, )) hidden_layer_1 = Dense(hidden_units_1, activation='sigmoid')(input_layer) hidden_layer_2 = Dense(hidden_units_2, activation='sigmoid')(hidden_layer_1) output_layer = Dense(10, activation='softmax')(hidden_layer_2) model = Model(input_layer, output_layer) model.compile(optimizer='sgd', loss='categorical_crossentropy') model.fit(x_train, y_train, epochs=nb_epoch, batch_size=batch_size) y_pred = np.argmax(model.predict(x_test), axis=-1) print(y_pred) save_file = os.path.join('result', 'mlp_keras.csv') save_result(y_pred, file_name=save_file)
dev_gold_graphs.extend(batch.graphs) dev_pred_graphs.extend(graphs) dev_sent_ids.extend(batch.sent_ids) dev_tokens.extend(batch.tokens) progress.close() dev_scores = score_graphs(dev_gold_graphs, dev_pred_graphs, relation_directional=config.relation_directional) for task in tasks: if dev_scores[task]['f'] > best_dev[task]: best_dev[task] = dev_scores[task]['f'] if task == 'role': print('Saving best role model') torch.save(state, best_role_model) best_dev_role_model = True save_result(dev_role_result_file, dev_gold_graphs, dev_pred_graphs, dev_sent_ids, dev_tokens) if task == 'trigger': print('Saving best event model') torch.save(state, best_event_model) best_dev_event_model = True save_result(dev_event_result_file, dev_gold_graphs, dev_pred_graphs, dev_sent_ids, dev_tokens) if task == 'relation': print('Saving best relation model') torch.save(state, best_relation_model) best_dev_relation_model = True save_result(dev_relation_result_file, dev_gold_graphs, dev_pred_graphs, dev_sent_ids, dev_tokens)
import os import sys import logging import pandas as pd import numpy as np from util import load_train_data, load_test_data, save_result train_file = os.path.join('data', 'train.csv') test_file = os.path.join('data', 'test.csv') x_train, y_train = load_train_data(train_file) x_test = load_test_data(test_file) from sklearn.svm import SVC # C Penalty parameter C of the error term # kernel Specifies the kernel type to be used in the algorithm. It must be one of ‘linear’, ‘poly’, ‘rbf’, ‘sigmoid’, ‘precomputed’ (default=rbf) clf = SVC() save_file = os.path.join('result', 'svm.csv') clf.fit(x_train, y_train) y_pred = clf.predict(x_test) save_result(y_pred, save_file)
metrics.macro_rec, metrics.macro_fscore)) print('Micro: P: {:.2f}, R: {:.2f}, F:{:.2f}'.format( metrics.micro_prec, metrics.micro_rec, metrics.micro_fscore)) # Save model if metrics.accuracy > best_scores['best_acc_dev']: best_acc = True best_scores['best_acc_dev'] = metrics.accuracy if metrics.macro_fscore > best_scores['best_mac_dev']: best_mac = True best_scores['best_mac_dev'] = metrics.macro_fscore print('Saving new best macro F1 model') torch.save(state, model_mac) save_result(results, label_itos, result_dev) if metrics.micro_fscore > best_scores['best_mic_dev']: best_mic = True best_scores['best_mic_dev'] = metrics.micro_fscore print('Saving new best micro F1 model') torch.save(state, model_mic) # Test set results = defaultdict(list) for batch in test_set.all_batches(label_size, batch_size=10, gpu=gpu, dev=True): elmo_ids, labels, men_masks, ctx_masks, dists, gathers, men_ids = batch preds = model.predict(elmo_ids, men_masks, ctx_masks, dists, gathers) results['gold'].extend(labels.int().data.tolist()) results['pred'].extend(preds.int().data.tolist()) results['ids'].extend(men_ids)
rules_data[endpoint.find('label')['for']] = 'on' client.post(url_rules, data=rules_data, headers=dict(Referer=url_rules)) def measure_execution_time(host, page, n=100): """ Call a certain page n times and returns the execution time (in ms) """ data = [] for _ in range(n): now = time.time() try: requests.get(host + page) except Exception: print('Can\'t open url {}{}'.format(host, page)) data.append((time.time() - now) * 1000) return data if __name__ == '__main__': host, name = parse_args() sleep_until_ready(host) print('\nHost is up.') if name == 'with_dashboard': print('Enabling monitoring of all endpoints...') monitor_all_endpoints(host) print('All endpoints are now monitored.') print('Testing the overhead now...') data = measure_execution_time(host, page='available_languages') save_result(data, name + '.txt') print('Results saved.')
def run_model_on_beatles(train, model_name, chords=None, files=None, data_independent=False): """ :param train: function to train the model. Must accept a parameter called chromagram_data :param model_name: Name of the model :param chords: Set to not None to display the chords mean/cov images :param files: files to train on :param data_independent: Set to True if training does not require data :return: None """ if files is None: files = [ 'cqt_512', 'cqt_1024_complete', 'cqt_256_complete' 'stft', 'stft_0.5_pow', 'stft_1.5_pow', 'stft_2_pow', 'cqt_512_hop_2_tol', 'cqt_1024' ] for f in files: type_ = '' if 'cqt' in f: type_ = 'CQT' else: type_ = 'STFT' hop_length = 512 if '1024' in f: hop_length = 1024 elif '8192' in f: hop_length = 8192 base_name = '{}_{}_{}'.format(model_name, type_, hop_length) title = '{} w/ {} Chromagram \nHop Length = {}'.format( model_name, type_, hop_length) if '_tol' in f: idx = f.index('_tol') base_name = base_name + '_' + str(f[idx - 1]) + '_tol' title = title + ' 0.{} Tolerance'.format(f[idx - 1]) if 'pow' in f: idx = f.index('_pow') try: num = float(f[idx - 3:idx]) except Exception: num = float(f[idx - 1]) base_name = base_name + '_' + str(num) + '_pow' title = title + ' {} power'.format(num) chromagram_data = load_data(f) chromagram_data = util.remove_song([ '10CD1_-_The_Beatles/06 - The Continuing Story of Bungalow Bill.flac', '10CD1_-_The_Beatles/05 - Wild Honey Pie.flac' ], chromagram_data) del chromagram_data['err'] if not data_independent: test_data, train_data = util.split_data(chromagram_data, 0.15) else: test_data = chromagram_data train_data = chromagram_data model = train(chromagram_data=train_data) evaluation = util.evaluate(model, test_data) util.save_result('{}.json'.format(base_name), evaluation) print(evaluation) util.display_err_matrix(matrix=evaluation['err_matrix'], title=title, file_name='{}.png'.format(base_name)) if chords is not None: s = util.bucket_sort(train_data['labels'], train_data['annotated_chromas']) mean = util.mean_matrix(s) cov = util.cov_matrix(s) for c in util.CHORDS: util.display_mean_cov_for_chord(c, mean[util.CHORD_IDX[c]], cov[util.CHORD_IDX[c]], file_name=base_name + '_' + c)