import configparser import ui_utils import nlp_tools # allow gpu memory growth from tensorflow.compat.v1 import ConfigProto from tensorflow.compat.v1 import InteractiveSession config = ConfigProto() config.gpu_options.allow_growth = True session = InteractiveSession(config=config) config = configparser.ConfigParser() config.read('./config_core_train.cfg') OUTPUT_DIR = config.get('data', 'OUTPUT_DIR') FILE_NOTE = config.get('data', 'FILE_NOTE') APPLY_FILE = config.get('applying', 'APPLY_FILE') CLF_THRESHOLD = float(config.get('applying', 'CLF_THRESHOLD')) APPLY_BATCH_SIZE = int(config.get('applying', 'APPLY_BATCH_SIZE')) OUTPUT_PATH = f"{OUTPUT_DIR}{FILE_NOTE}/" SCORED_PATH = f"{OUTPUT_PATH}scored/" Path(SCORED_PATH).mkdir(parents=True, exist_ok=True) # initialize logger root_logger = logging.getLogger() formatter = logging.Formatter('%(asctime)s: %(levelname)s:: %(message)s') # prints to file logfile = f"{OUTPUT_PATH}apply_logs.log"
session = InteractiveSession(config=config) arg_parser = argparse.ArgumentParser( description='Run model training for each tagged label') arg_parser.add_argument('--config_path', metavar='config_path', default='./al_3_train.cfg', type=str, help='Path to the training config file', required=False) args = arg_parser.parse_args() config = configparser.ConfigParser() config.read(args.config_path) MAX_TAGS = int(config.get('training', 'MAX_TAGS')) BATCH_SIZE = int(config.get('training', 'BATCH_SIZE')) BUFFER_SIZE = int(config.get('training', 'BUFFER_SIZE')) INPUT_FILE = config.get('data', 'INPUT_FILE') OUTPUT_PATH = config.get('data', 'OUTPUT_PATH') EARLY_STOPPING_ROUNDS = int(config.get('training', 'EARLY_STOPPING_ROUNDS')) EMBED_TRAINABLE = bool(config.get('training', 'EMBED_TRAINABLE') == 'True') RANDOM_EMBED = bool(config.get('training', 'RANDOM_EMBED') == 'True') DROPOUT_LEVEL = float(config.get('training', 'DROPOUT_LEVEL')) LEARNING_RATE = float(config.get('training', 'LEARNING_RATE')) N_LSTM_UNITS = int(config.get('training', 'N_LSTM_UNITS')) N_FC_NEURONS = int(config.get('training', 'N_FC_NEURONS')) Path(OUTPUT_PATH).mkdir(parents=True, exist_ok=True) # initialize logger
from tensorflow.compat.v1 import InteractiveSession config = ConfigProto() config.gpu_options.allow_growth = True session = InteractiveSession(config=config) arg_parser = argparse.ArgumentParser(description='Run model scoring based on config file') arg_parser.add_argument('--config_path', metavar='config_path', default='./al_4_score.cfg', type=str, help='Path to the scoring config file', required=False) args = arg_parser.parse_args() config = configparser.ConfigParser() config.read(args.config_path) OUTPUT_PATH = config.get('data', 'OUTPUT_PATH') APPLY_FILE = config.get('data', 'APPLY_FILE') CLF_THRESHOLD = float(config.get('scoring', 'CLF_THRESHOLD')) APPLY_BATCH_SIZE = int(config.get('scoring', 'APPLY_BATCH_SIZE')) SCORE_W_GPU = bool(int(config.get('scoring', 'SCORE_W_GPU'))) SCORED_PATH = f"{OUTPUT_PATH.rstrip('/')}/scored/" Path(SCORED_PATH).mkdir(parents=True, exist_ok=True) # initialize logger root_logger = logging.getLogger() formatter = logging.Formatter('%(asctime)s: %(levelname)s:: %(message)s') # prints to file logfile = f"{OUTPUT_PATH.rstrip('/')}/scoring_logs.log"