def main():
conf, job_name = load_config()
init_logging()
# tf.logging.set_verbosity(3)
os.makedirs(conf.train.train_dir, exist_ok=True)
if job_name == "search":
axis_search(conf)
else:
train_model(job_name, conf, conf.train.hyper_params)
def main():
conf, args = load_config()
init_logging()
# tf.logging.set_verbosity(3)
os.makedirs(conf.train.train_dir, exist_ok=True)
train_model(
args.name,
conf,
conf.train.hyper_params,
grid_search=args.grid,
weights=args.weights,
)
def build_recognizer(BASE_PATH=BASE_PATH):
print("Loading Arabic dataset")
X_train, X_test, Y_train, Y_test = Arabic()
X_train, X_test = process_dataset(X_train, X_test)
ar_model_path = os.path.join(BASE_PATH, r"train/ar_model.h5")
print("Training Arabic model")
ar_model = train_model(X_train, Y_train, X_test, Y_test, ar_model_path)
print("Loading English dataset")
X_train, X_test, Y_train, Y_test = English()
X_train, X_test = process_dataset(X_train, X_test)
en_model_path = os.path.join(BASE_PATH, r"train/en_model.h5")
print("Training Arabic model")
en_model = train_model(X_train, Y_train, X_test, Y_test, en_model_path)
return ar_model, en_model
def start_training(user_model, concepts, verify_videos, model_params):
"""Start a training job with the correct parameters
"""
print("training")
train_model(
concepts,
verify_videos,
user_model,
model_params["annotation_collections"],
int(model_params["min_images"]),
int(model_params["epochs"]),
download_data=True,
verified_only=model_params["verified_only"],
include_tracking=model_params["include_tracking"],
)
def main():
"""
The start of the flow handles all initializations, configuration loading
and performs training or deployment based upon 'config.info.operation_type'
For more details on configuration options look up commnets in config.yaml
"""
parser = get_parser()
config = parser.parse_args(['--cfg', 'config.yaml'])
result_filing.init_config_vars(config)
run_id = config.info.run_id
logger = custom_logger.CustomLogger(run_id + ':' + file_id)
operation = config.info.operation_type
logger.info("Selected operation type %s." % (operation))
if operation == const.TRAIN_OP:
train.train_model(config)
elif operation == const.DEPLOY_OP:
test.test_model(config)
# Thresholding outliers
df_model_thresholded = prod.threshold_df_outliers_pre_model(df_model)
# Predicting and thresholding most confident predictions in number of recurring cycles (iterations)
df_preds = prod.cyclical_pred(
df_model_thresholded, thresh=0.95, enforce_first_cycle=True, norm_trfm=True, skew_trfm=False)
# Saving predictions to file
prod.write_preds(df_preds, file)
# -- !!! END !!! -- PRODUCTION BRANCH
# -- !!! START !!! -- TRAINING BRANCH
elif cfg.mode == 'train':
trn = train.train_model(train_lbls_filename=cfg.train_lbls_filename,
otg_cols_desc=cfg.otg_cols_desc, logs_cols_desc=cfg.logs_cols_desc)
# Geting labeled dataframe
lst_lv_outages = trn.get_train_lbl(get_unq_devs=True)
# getting subset for Train LV for concatenetaion all DFs in one DF containing only devices for which outages occured
df = df[df[cfg.dev_agg_col].isin(lst_lv_outages)]
dfs = dfs + tuple((df,))
if cfg.mode == 'train':
# Concat all logs in one DF
df = pd.concat(dfs, axis=0, sort=False, ignore_index=True).fillna(0)
# Grouping all logs in one to prepare train dataframe
df_model = trn.make_hist_df_train(
df, period=cfg.period, interval=cfg.interval)
def train_model(self, new_games):
model = copy.copy(self.champion.current_policy)
return train_model(model=model,
games=new_games,
min_num_games=self.championship_rounds)