def main():
start_time = time.time()
args = parse_args()
logger.setLevel(getattr(logging, args.verbosity.upper()))
logger.info("Started")
build_constants()
df = pd.read_json(path_or_buf=DATA_PATH, orient='records', encoding="UTF8")
logger.debug("Loaded {} rows into df".format(len(df)))
df = utils.get_data_subset.crop(df, None, None)
df = utils.get_data_subset.filter_rows_by_string(df, [TARGET_COL],
['Rock', 'Hip Hop'])
df = utils.clean_data.execute_cleaners(df)
df = utils.normalize_data.normalize_genres(df, TARGET_COL)
X, y = utils.get_data_subset.get_x_y(df, SAMPLE_COL, TARGET_COL)
clf = model_pipeline.get_pipeline(SAMPLE_COL)
utils.persistence.dump(DF_DUMP_NAME, df)
utils.persistence.dump(CLF_DUMP_NAME, clf)
if args.train:
train_and_test.train_and_dump(X, y, clf)
elif args.test:
train_and_test.test_using_kfold(X, y, clf)
logger.info("Finished in {0:.2f} seconds".format(time.time() - start_time))
def run(self):
for server_name in config.sections():
# Changing the log level to the level specified in the config file
logger.setLevel(
logging.getLevelName(config[server_name]["log_level"]))
address = config[server_name]["address"]
user = config[server_name]["username"]
password = config[server_name]["password"]
game_password = config[server_name]["game_password"]
motd_scoreboard = str_to_bool(
config[server_name]["motd_scoreboard"])
scoreboard_type = config[server_name]["scoreboard_type"]
level_threshhold = config[server_name]["level_threshhold"]
enable_greeter = str_to_bool(config[server_name]["enable_greeter"])
max_players = config[server_name]["max_players"]
server = Server(server_name, address, user, password,
game_password, max_players, level_threshhold)
self.servers.append(server)
if motd_scoreboard:
motd_updater = MotdUpdater(server, scoreboard_type)
motd_updater.start()
self.motd_updaters.append(motd_updater)
cb = Chatbot(server, greeter_enabled=enable_greeter)
server.chat.add_listener(cb)
self.chatbots.append(cb)
print("Initialisation complete")
def run(config):
"""
Runs Trainer.
"""
# for parallel workers training (distributed training)
rank = MPI.COMM_WORLD.Get_rank(
) # Each process is assigned a rank that is unique within Communicator(group of processes)
config.rank = rank
config.is_chef = rank == 0 # Binary
config.seed = config.seed + rank
config.num_workers = MPI.COMM_WORLD.Get_size(
) # total no. of processes is a size of communicator
if config.is_chef:
logger.warn('Run a base worker.')
make_log_files(config)
else:
logger.warn('Run worker %d and disable logger.', config.rank)
import logging
logger.setLevel(logging.CRITICAL)
def shutdown(signal, frame):
logger.warn('Received signal %s: exiting', signal)
sys.exit(128 + signal)
signal.signal(signal.SIGHUP, shutdown)
signal.signal(signal.SIGINT, shutdown)
signal.signal(signal.SIGTERM, shutdown)
# set global seed
np.random.seed(config.seed)
torch.manual_seed(config.seed)
torch.cuda.manual_seed_all(config.seed)
# set the display no. configured with gpu
os.environ["DISPLAY"] = ":0"
# use gpu or cpu
if config.gpu is not None:
os.environ["CUDA_VISIBLE_DEVICES"] = "{}".format(config.gpu)
assert torch.cuda.is_available()
config.device = torch.device("cuda")
else:
config.device = torch.device("cpu")
# build a trainer
trainer = Trainer(config)
if config.is_train:
trainer.train()
logger.info("Finish training")
else:
trainer.evaluate()
logger.info("Finish evaluating")
def main():
logger.setLevel(logging.DEBUG)
dynamic_loading.imports_from_modules(constants.DATA_CLEANERS_DIR)
dynamic_loading.imports_from_modules(constants.FEAT_EXTS_DIR)
clf = persistence.load(CLF_NAME)
print("Guess Genre By Lyrics")
if fetch_lyrics:
artist = input(">Artist: ").lower()
song = input(">Song: ").lower()
lyric = fetch_lyrics.fetch("%s" % artist, "%s" % song)
else:
lyric = text_from_file.load(constants.SAMPLE_LYRIC_PATH)
if lyric:
prediction = predict_genre(lyric, clf)
print('{}\n{}'.format('-' * 100, lyric))
print('{}\nPrediction: {}'.format('-' * 100, prediction))
else:
print("No lyrics no prediction :(")
import sys
import argparse
from flask import Flask
from flask import request
from flask import jsonify
cdir = os.path.dirname(os.path.abspath(__file__))
sys.path.append(os.path.join(cdir, '..'))
from utils.logger import logger
import model.preprocessing as pp
from model.model import Model
logger.setLevel('DEBUG')
if __name__ == '__main__':
mdl_prm_name = sys.argv[1]
norm_prm_name = sys.argv[2]
app = Flask(__name__)
lr_model = Model()
lr_model.load_model(mdl_prm_name)
@app.route('/')
def root():
'''
root
'''
import unittest
import os
import sys
import pandas as pd
import numpy as np
from sklearn.preprocessing import MinMaxScaler
cdir = os.path.dirname(os.path.abspath(__file__))
lib_dir = os.path.join(cdir, '..', 'hec')
sys.path.append(lib_dir)
import model.preprocessing as pp
from utils.logger import logger
logger.setLevel('CRITICAL')
class TestPreprocessing(unittest.TestCase):
def test_sprit_train_test(self):
df_src = dummy_data()
drop_idx = df_src.index[0]
df_src = df_src.drop(index=drop_idx)
train, test = pp.split_df_train_test(df_src, 7, 0.7)
self.assertEqual(train.shape, (6552, 1))
self.assertEqual(test.shape, (2976, 1))
self.assertEqual(train.index[0].hour, 0)
self.assertEqual(train.index[-1].hour, 23)
self.assertEqual(test.index[0].hour, 0)
self.assertEqual(test.index[-1].hour, 23)
import cv2
import copy
import numpy as np
from utils.logger import logger
logger.setLevel('debug')
class DataAugmentor(object):
'''
'''
def __init__(self):
'''
'''
self._funcs = []
self._add_funcs()
return
def augmented(self, image):
'''
'''
X = []
for func in self._funcs:
x = copy.deepcopy(func(image))
X.append(x)
return X
def _add_funcs(self):
'''
'''
self._funcs.append(self._fliplr)
self._funcs.append(self._flipud)
