def main(): start_time = time.time() args = parse_args() logger.setLevel(getattr(logging, args.verbosity.upper())) logger.info("Started") build_constants() df = pd.read_json(path_or_buf=DATA_PATH, orient='records', encoding="UTF8") logger.debug("Loaded {} rows into df".format(len(df))) df = utils.get_data_subset.crop(df, None, None) df = utils.get_data_subset.filter_rows_by_string(df, [TARGET_COL], ['Rock', 'Hip Hop']) df = utils.clean_data.execute_cleaners(df) df = utils.normalize_data.normalize_genres(df, TARGET_COL) X, y = utils.get_data_subset.get_x_y(df, SAMPLE_COL, TARGET_COL) clf = model_pipeline.get_pipeline(SAMPLE_COL) utils.persistence.dump(DF_DUMP_NAME, df) utils.persistence.dump(CLF_DUMP_NAME, clf) if args.train: train_and_test.train_and_dump(X, y, clf) elif args.test: train_and_test.test_using_kfold(X, y, clf) logger.info("Finished in {0:.2f} seconds".format(time.time() - start_time))
def run(self): for server_name in config.sections(): # Changing the log level to the level specified in the config file logger.setLevel( logging.getLevelName(config[server_name]["log_level"])) address = config[server_name]["address"] user = config[server_name]["username"] password = config[server_name]["password"] game_password = config[server_name]["game_password"] motd_scoreboard = str_to_bool( config[server_name]["motd_scoreboard"]) scoreboard_type = config[server_name]["scoreboard_type"] level_threshhold = config[server_name]["level_threshhold"] enable_greeter = str_to_bool(config[server_name]["enable_greeter"]) max_players = config[server_name]["max_players"] server = Server(server_name, address, user, password, game_password, max_players, level_threshhold) self.servers.append(server) if motd_scoreboard: motd_updater = MotdUpdater(server, scoreboard_type) motd_updater.start() self.motd_updaters.append(motd_updater) cb = Chatbot(server, greeter_enabled=enable_greeter) server.chat.add_listener(cb) self.chatbots.append(cb) print("Initialisation complete")
def run(config): """ Runs Trainer. """ # for parallel workers training (distributed training) rank = MPI.COMM_WORLD.Get_rank( ) # Each process is assigned a rank that is unique within Communicator(group of processes) config.rank = rank config.is_chef = rank == 0 # Binary config.seed = config.seed + rank config.num_workers = MPI.COMM_WORLD.Get_size( ) # total no. of processes is a size of communicator if config.is_chef: logger.warn('Run a base worker.') make_log_files(config) else: logger.warn('Run worker %d and disable logger.', config.rank) import logging logger.setLevel(logging.CRITICAL) def shutdown(signal, frame): logger.warn('Received signal %s: exiting', signal) sys.exit(128 + signal) signal.signal(signal.SIGHUP, shutdown) signal.signal(signal.SIGINT, shutdown) signal.signal(signal.SIGTERM, shutdown) # set global seed np.random.seed(config.seed) torch.manual_seed(config.seed) torch.cuda.manual_seed_all(config.seed) # set the display no. configured with gpu os.environ["DISPLAY"] = ":0" # use gpu or cpu if config.gpu is not None: os.environ["CUDA_VISIBLE_DEVICES"] = "{}".format(config.gpu) assert torch.cuda.is_available() config.device = torch.device("cuda") else: config.device = torch.device("cpu") # build a trainer trainer = Trainer(config) if config.is_train: trainer.train() logger.info("Finish training") else: trainer.evaluate() logger.info("Finish evaluating")
def main(): logger.setLevel(logging.DEBUG) dynamic_loading.imports_from_modules(constants.DATA_CLEANERS_DIR) dynamic_loading.imports_from_modules(constants.FEAT_EXTS_DIR) clf = persistence.load(CLF_NAME) print("Guess Genre By Lyrics") if fetch_lyrics: artist = input(">Artist: ").lower() song = input(">Song: ").lower() lyric = fetch_lyrics.fetch("%s" % artist, "%s" % song) else: lyric = text_from_file.load(constants.SAMPLE_LYRIC_PATH) if lyric: prediction = predict_genre(lyric, clf) print('{}\n{}'.format('-' * 100, lyric)) print('{}\nPrediction: {}'.format('-' * 100, prediction)) else: print("No lyrics no prediction :(")
import sys import argparse from flask import Flask from flask import request from flask import jsonify cdir = os.path.dirname(os.path.abspath(__file__)) sys.path.append(os.path.join(cdir, '..')) from utils.logger import logger import model.preprocessing as pp from model.model import Model logger.setLevel('DEBUG') if __name__ == '__main__': mdl_prm_name = sys.argv[1] norm_prm_name = sys.argv[2] app = Flask(__name__) lr_model = Model() lr_model.load_model(mdl_prm_name) @app.route('/') def root(): ''' root '''
import unittest import os import sys import pandas as pd import numpy as np from sklearn.preprocessing import MinMaxScaler cdir = os.path.dirname(os.path.abspath(__file__)) lib_dir = os.path.join(cdir, '..', 'hec') sys.path.append(lib_dir) import model.preprocessing as pp from utils.logger import logger logger.setLevel('CRITICAL') class TestPreprocessing(unittest.TestCase): def test_sprit_train_test(self): df_src = dummy_data() drop_idx = df_src.index[0] df_src = df_src.drop(index=drop_idx) train, test = pp.split_df_train_test(df_src, 7, 0.7) self.assertEqual(train.shape, (6552, 1)) self.assertEqual(test.shape, (2976, 1)) self.assertEqual(train.index[0].hour, 0) self.assertEqual(train.index[-1].hour, 23) self.assertEqual(test.index[0].hour, 0) self.assertEqual(test.index[-1].hour, 23)
import cv2 import copy import numpy as np from utils.logger import logger logger.setLevel('debug') class DataAugmentor(object): ''' ''' def __init__(self): ''' ''' self._funcs = [] self._add_funcs() return def augmented(self, image): ''' ''' X = [] for func in self._funcs: x = copy.deepcopy(func(image)) X.append(x) return X def _add_funcs(self): ''' ''' self._funcs.append(self._fliplr) self._funcs.append(self._flipud)