def main(args): start = time.time() utils.fancy_print("Starting workflow", color="green", size=70) ml_params = utils.read_config(args.ml_config_path) utils.validate_ml_params(ml_params) if args.experiment_config_path: experiment_params = utils.read_config(args.experiment_config_path) else: assert args.experiment_id is not None, ( "If no --experiment_config_path provided, --experiment_id must" " be provided to fetch experiment config from bandit app." ) assert "bandit_app_credential_path" in ml_params, ( "If getting experiment config from banditml.com, must provide" " valid api key path in `bandit_app_credential_path` in ml config." ) logger.info("Getting experiment config from banditml.com...") experiment_params = utils.get_experiment_config_from_bandit_app( ml_params["bandit_app_credential_path"], args.experiment_id ) experiment_params["experiment_id"] = args.experiment_id logger.info("Using parameters: {}\n".format(ml_params)) train( ml_params=ml_params, experiment_params=experiment_params, predictor_save_dir=args.predictor_save_dir, s3_bucket_to_write_to=args.s3_bucket_to_write_to, ) logger.info("Workflow completed successfully.") logger.info(f"Took {time.time() - start} seconds to complete.")
def main(args): start = time.time() fancy_print("Starting workflow", color="green", size=70) ml_params = read_config(args.ml_config_path) if args.experiment_config_path: experiment_params = read_config(args.experiment_config_path) else: assert args.experiment_id is not None, ( "If no --experiment_config_path provided, --experiment_id must" " be provided to fetch experiment config from bandit app.") logger.info("Getting experiment config from banditml.com...") experiment_params = get_experiment_config_from_bandit_app( args.experiment_id) # in transit this gets dumped as a string so load it experiment_params["reward_function"] = json.loads( experiment_params["reward_function"]) experiment_params["experiment_id"] = args.experiment_id experiment_params["model_name"] = args.model_name logger.info("Using parameters: {}".format(ml_params)) train( ml_params=ml_params, experiment_params=experiment_params, predictor_save_dir=args.predictor_save_dir, s3_bucket_to_write_to=args.s3_bucket_to_write_to, ) logger.info("Workflow completed successfully.") logger.info(f"Took {time.time() - start} seconds to complete.")
def lambda_handler(ws_event, context): """ Lambda Handler that changes my slack status :param ws_event: body request and all that :param context: not sure :return: """ # read configs utils.read_config() ########### # PING # ########### if process_ping(ws_event): now = get_bst_datetime().isoformat() return { 'statusCode': 200, 'body': json.dumps('Ping received at ' + now) } ############################ # STATE (WORK, AWAY, OFF) # ############################ state = infer_state() # Check if slack status says 'in a meeting' and don't change if is_in_known_status_checker(state): return { 'statusCode': 200, 'body': json.dumps('In na event, waiting for event to expire') } ################################################ # detect HOME OFFICE, VACATION and MEETING # ################################################ try: found_relevant_event = check_calendar_events(state) except: logger.error("Expired token") found_relevant_event = False ############### # FALLBACK # ############### if not found_relevant_event: logger.info('NO relevant event. Processing fallback status') process_fallback(state) return { 'statusCode': 200, 'body': json.dumps('Everything OK!') }
def test_ddpg(self): CONFIG_PATH = 'tests/config_test.yml' config = read_config(CONFIG_PATH) config['env'] = 'BipedalWalker-v3' config['model'] = 'ddpg' config['state_dim'] = 24 config['action_dim'] = 4 engine = load_engine(config) engine.train()
def test_ddpg(self): CONFIG_PATH = 'tests/config_test.yml' config = read_config(CONFIG_PATH) config['env'] = 'Pendulum-v0' config['model'] = 'ddpg' config['state_dim'] = 3 config['action_dim'] = 1 engine = load_engine(config) engine.train()
def test_ddpg(self): CONFIG_PATH = 'tests/config_test.yml' config = read_config(CONFIG_PATH) config['env'] = 'LunarLanderContinuous-v2' config['model'] = 'ddpg' config['state_dim'] = 8 config['action_dim'] = 2 engine = load_engine(config) engine.train()
version='| Version | {:^8} |'.format(VERSION)) parser.add_argument( '--train', action='store_true', help='Train a new model or continue training an existing model.') parser.add_argument('--evaluate', action='store_true', help='Evaluate a trained model.') parser.add_argument('--config', type=str, default="configs/smn_last.config.json", help='Location of config file') args = parser.parse_args() # lower the configuration parameters dict trainerParams = utils.read_config(args.config) if args.train: logger = utils.create_logger(trainerParams["global"]["log_file"]) logger.info(json.dumps(trainerParams, indent=4)) train(trainerParams, evaluate=args.evaluate) elif args.evaluate: # modify the name of log file trainerParams["global"]["log_file"] = trainerParams["global"][ "log_file"].rsplit('.', maxsplit=1)[0] + ".evaluation.log" logger = utils.create_logger(trainerParams["global"]["log_file"]) logger.info(json.dumps(trainerParams, indent=4)) evaluate(trainerParams) else: parser.print_help() # logger.error("\n"+parser.format_help())
import argparse from models.engine import load_engine from utils.utils import read_config parser = argparse.ArgumentParser(description='Run training') parser.add_argument("--config", type=str, help="Path to the config file.") if __name__ == "__main__": args = vars(parser.parse_args()) config = read_config(args['config']) engine = load_engine(config) engine.train()
from models.engine import load_engine from utils.utils import read_config if __name__ == "__main__": CONFIG_PATH = "d4pg-pytorch/configs/follow_rl.yml" CONFIG_PATH = "configs/LunarLanderContinuous_d4pg.yml" config = read_config(CONFIG_PATH) engine = load_engine(config) engine.test()
def __init__(self, embedding_or_raw_data_file="", voc_limit=None, model_name="word2vec", intersecting_embedding="", config_file=""): """Initialize the embedding module. Args: embedding_or_raw_data_file: str Path to the word vectors. voc_limit: int A restriction on the embedding vocabulary. model_name: str The name of the embedding model that is used: 'word2vec' or 'bert'. embedding_combiner: numpy callable A function to combine two or more (sub-) word vectors. """ # get the configuration self.config = utils.read_config(config_file) self.embedding_model_name = model_name self.max_seq_length = 20 self.check_other_embedding = self.config["check_other_embedding"] if self.embedding_model_name == "word2vec": if word2vec_available and embedding_or_raw_data_file: self.word_vectors, self.embedding_dim = self.load_word2vec( embedding_or_raw_data_file, voc_limit) print(type(self.word_vectors)) self.vocab = self.word_vectors.vocab self.embedding_combiner = None elif word2vec_available and not embedding_or_raw_data_file: print("No file for embedding data given.") sys.exit(1) else: raise RuntimeError( "Please download the necessary word2vec embedding file.") # elif self.embedding_model_name == "bert": # if pretrained_bert_available: # used_bert_model = "bert-base-uncased" # self.word_vectors, self.embedding_dim = self.load_bert( # used_bert_model) # self.vocab = list(self.word_vectors.keys()) # self.embedding_combiner = getattr( # self, self.config["embedding_combiner"])() # print("\nNOTE: the chosen embedding combiner is {}\n".format( # self.embedding_combiner)) # else: # raise RuntimeError("Please install pytorch and the bert " # "model via 'pip install " # "pytorch-pretrained-bert'") elif self.embedding_model_name == "glove": self.word_vectors, self.embedding_dim = self.load_glove( embedding_or_raw_data_file, voc_limit) self.vocab = self.word_vectors.vocab self.embedding_combiner = None elif self.embedding_model_name == "fasttext": self.word_vectors, self.embedding_dim = self.load_fasttext( embedding_or_raw_data_file, voc_limit) self.vocab = self.word_vectors.vocab self.embedding_combiner = None else: assert False, "Embedding model name unknown: '{}'\n".format( self.embedding_model_name) if self.check_other_embedding == "word2vec": self.load_w2v_as_comparison(intersecting_embedding, voc_limit) if self.check_other_embedding == "bert": used_bert_model = "bert-base-uncased" self.load_bert_as_comparison(used_bert_model)
import scipy.signal import torch import torchaudio from flask import Flask, jsonify, request from typing import List, Dict, Text, Union from utils.utils import read_config from model.network import SpeechRecognitionModel from utils.text_transform import TextTransform app = Flask(__name__) config = read_config('config/config.yaml') tt = TextTransform() def load_model(config: Dict = config): state_dict_path = config.get('CHECKPOINT_PATH') checkpoint = torch.load(state_dict_path) model = SpeechRecognitionModel( n_cnn_layers = config.get('N_CNN_LAYERS'), n_rnn_layers = config.get('N_RNN_LAYERS'), rnn_dim = config.get('RNN_DIM'), n_class = config.get('N_CLASS'), n_feats = config.get('N_FEATS'), stride = config.get('STRIDE'), dropout = config.get('DROPOUT') )
import argparse from models.engine import load_engine from utils.utils import read_config if __name__ == "__main__": config = read_config("config.yml") engine = load_engine(config) engine.train()
write_new_file([train_argmax], [train_max], train_examples, train_examples[0][::train_step], training_params) _, dev_argmax, dev_max = calculate_cosine_similarity( dev_preds[0], train_preds[0]) logger.info("dev_argmax: {} \tdev_max: {}".format(len(dev_argmax), len(dev_max))) write_new_file([dev_argmax], [dev_max], dev_examples, train_examples[0][::train_step], validation_params) _, test_argmax, test_max = zip(*[ calculate_cosine_similarity(test_preds[i], train_preds[0]) for i in range(len(test_preds)) ]) logger.info("test_argmax: {} \ttest_max: {}".format( len(test_argmax[0]), len(test_max[0]))) write_new_file(test_argmax, test_max, test_examples, train_examples[0][::train_step], evaluation_params) if __name__ == "__main__": parser = argparse.ArgumentParser(description='Create Dataset.') parser.add_argument('--config', type=str, default="", help='Location of config file') args = parser.parse_args() config = utils.read_config(args.config) logger = utils.create_logger(config["global"]["log_file"]) create_dataset_for_context_retrieval(config)
# specify subplot and turn of axis ax = pyplot.subplot(4, 8, ix) ax.set_xticks([]) ax.set_yticks([]) # plot filter channel in grayscale print(filter.shape) pyplot.imshow(filter[:, :, 0, j]) ix += 1 pyplot.savefig( f'visualization/Layer_{layer_index}/subplot_32_filters.png') def visualize_filters(model, config): stable_mode = config.get('stable_mode') testset_size = config.get('N_test') print('Loading data set') data_parser = DataSetParser(stable_mode=stable_mode, config=config) X = data_parser.train_data[0][0] print(X.shape) X = np.expand_dims(X, axis=-1) print(X.shape) plot_feature_map(model, X) if __name__ == '__main__': print('CURRENT DIR: {}'.format(os.getcwd())) args = get_args() config = read_config(args) model = build_or_load_model(config) visualize_filters(model=model['train'], config=config)