def run_training_next_function(cmd_args: ArgNamespace):
from autopandas_v2.ml.networks.ggnn.models.sparse.base import SparseGGNN
from autopandas_v2.ml.networks.ggnn.utils import ParamsNamespace
train_path = cmd_args.train
valid_path = cmd_args.valid
model_path = cmd_args.modeldir
if not os.path.exists(model_path):
os.system('mkdir -p ' + model_path)
ggnn_args = ArgNamespace.from_namespace(cmd_args)
ggnn_args.train = train_path
ggnn_args.valid = valid_path
ggnn_args.outdir = model_path
ggnn_args.mode = 'train'
params = ParamsNamespace()
if cmd_args.config is not None:
with open(cmd_args.config, 'r') as f:
params.update(json.load(f))
if cmd_args.config_str is not None:
params.update(json.loads(cmd_args.config_str))
params.args = ParamsNamespace()
params.args.update(ggnn_args)
params.use_directed_edges = True
model = SparseGGNN.from_params(params)
model.run()
def setup_inference(self, model_dir: str):
# Load the params
with open('{}/params.pkl'.format(model_dir), 'rb') as f:
self.params = pickle.load(f)
self.params.args = ParamsNamespace()
self.model_dir: str = model_dir
model_path = '{}/model_best.pickle'.format(self.model_dir)
# Setup the model
self.build_graph_model(mode='testing', restore_file=model_path)
def default_params(cls):
return {
'args': ParamsNamespace(),
'num_epochs': 3000,
'patience': 25,
'lr': 0.001,
'clamp_gradient_norm': 1.0,
'out_layer_dropout_keep_prob': 1.0,
'hidden_size_node': 100,
'hidden_size_final_mlp': 100,
'num_timesteps': 4,
'tie_fwd_bkwd': True,
'use_directed_edges': True,
'random_seed': 0,
}
def run_analysis_helper(fname: str, identifier: str, cmd_args: ArgNamespace):
from autopandas_v2.generators.ml.networks.ggnn.ops.choice import ModelChoice
from autopandas_v2.generators.ml.networks.ggnn.ops.chain import ModelChain
from autopandas_v2.generators.ml.networks.ggnn.ops.select import ModelSelect
from autopandas_v2.generators.ml.networks.ggnn.ops.subsets import ModelSubsets
from autopandas_v2.generators.ml.networks.ggnn.ops.orderedsubsets import ModelOrderedSubsets
from autopandas_v2.ml.networks.ggnn.utils import ParamsNamespace
test_path = '{}/{}/{}.pkl'.format(cmd_args.test, fname, identifier)
model_path = '{}/{}/{}'.format(cmd_args.modeldir, fname, identifier)
if not os.path.exists(test_path):
raise Exception("Test data path {} does not exist".format(test_path))
os.system('mkdir -p ' + model_path)
ggnn_args = ArgNamespace.from_namespace(cmd_args)
ggnn_args.test = test_path
ggnn_args.outdir = model_path
ggnn_args.mode = 'train'
ggnn_args.model = model_path
params = ParamsNamespace()
if cmd_args.config is not None:
with open(cmd_args.config, 'r') as f:
params.update(json.load(f))
params.args = ParamsNamespace()
params.args.update(ggnn_args)
params.use_directed_edges = True
if identifier.startswith("choice"):
model = ModelChoice.from_params(params)
elif identifier.startswith("chain"):
model = ModelChain.from_params(params)
elif identifier.startswith("select"):
model = ModelSelect.from_params(params)
elif identifier.startswith("subsets"):
model = ModelSubsets.from_params(params)
elif identifier.startswith("orderedsubsets"):
model = ModelOrderedSubsets.from_params(params)
else:
raise NotImplementedError("Model not defined for operator {}".format(
identifier.split('_')[0]))
return model.run_analysis(test_path)
def run(cmd):
cmd_args = parse_args(cmd)
if cmd_args.subcommand == "sparse":
params = ParamsNamespace()
if cmd_args.config is not None:
with open(cmd_args.config, 'r') as f:
params.update(json.load(f))
params.args = ParamsNamespace()
params.args.update(cmd_args)
if cmd_args.rnn:
model = GGNNSeqStaticRNN.from_params(params)
else:
model = SparseGGNN.from_params(params)
model.run()
def run_training_generators_helper(fname: str, identifier: str,
cmd_args: ArgNamespace):
from autopandas_v2.generators.ml.networks.ggnn.ops.choice import ModelChoice
from autopandas_v2.generators.ml.networks.ggnn.ops.chain import ModelChain
from autopandas_v2.generators.ml.networks.ggnn.ops.select import ModelSelect
from autopandas_v2.generators.ml.networks.ggnn.ops.subsets import ModelSubsets
from autopandas_v2.generators.ml.networks.ggnn.ops.orderedsubsets import ModelOrderedSubsets
from autopandas_v2.ml.networks.ggnn.utils import ParamsNamespace
train_path = '{}/{}/{}.pkl'.format(cmd_args.train, fname, identifier)
valid_path = '{}/{}/{}.pkl'.format(cmd_args.valid, fname, identifier)
model_path = '{}/{}/{}'.format(cmd_args.modeldir, fname, identifier)
if not os.path.exists(train_path):
raise Exception(
"Training data path {} does not exist".format(train_path))
if not os.path.exists(valid_path):
raise Exception(
"Validation data path {} does not exist".format(valid_path))
if cmd_args.ignore_if_exists and os.path.exists(model_path +
'/model_best.pickle'):
logger.info(
"Skipping training for {}:{} as model already exists".format(
fname, identifier))
return
os.system('mkdir -p ' + model_path)
ggnn_args = ArgNamespace.from_namespace(cmd_args)
ggnn_args.train = train_path
ggnn_args.valid = valid_path
ggnn_args.outdir = model_path
ggnn_args.mode = 'train'
if cmd_args.restore_if_exists and os.path.exists(model_path +
'/model_best.pickle'):
ggnn_args.restore = model_path + '/model_best.pickle'
params = ParamsNamespace()
if cmd_args.config is not None:
with open(cmd_args.config, 'r') as f:
params.update(json.load(f))
if cmd_args.config_str is not None:
params.update(json.loads(cmd_args.config_str))
params.args = ParamsNamespace()
params.args.update(ggnn_args)
params.use_directed_edges = True
if identifier.startswith("choice"):
model = ModelChoice.from_params(params)
elif identifier.startswith("chain"):
model = ModelChain.from_params(params)
elif identifier.startswith("select"):
model = ModelSelect.from_params(params)
elif identifier.startswith("subsets"):
model = ModelSubsets.from_params(params)
elif identifier.startswith("orderedsubsets"):
model = ModelOrderedSubsets.from_params(params)
else:
raise NotImplementedError("Model not defined for operator {}".format(
identifier.split('_')[0]))
model.run()
def __init__(self):
self.params: ParamsNamespace = ParamsNamespace()
self.params.update(self.default_params())
