Exemple #1
0
def run_training_next_function(cmd_args: ArgNamespace):
    from autopandas_v2.ml.networks.ggnn.models.sparse.base import SparseGGNN
    from autopandas_v2.ml.networks.ggnn.utils import ParamsNamespace

    train_path = cmd_args.train
    valid_path = cmd_args.valid
    model_path = cmd_args.modeldir

    if not os.path.exists(model_path):
        os.system('mkdir -p ' + model_path)

    ggnn_args = ArgNamespace.from_namespace(cmd_args)
    ggnn_args.train = train_path
    ggnn_args.valid = valid_path
    ggnn_args.outdir = model_path
    ggnn_args.mode = 'train'

    params = ParamsNamespace()

    if cmd_args.config is not None:
        with open(cmd_args.config, 'r') as f:
            params.update(json.load(f))

    if cmd_args.config_str is not None:
        params.update(json.loads(cmd_args.config_str))

    params.args = ParamsNamespace()
    params.args.update(ggnn_args)
    params.use_directed_edges = True

    model = SparseGGNN.from_params(params)
    model.run()
Exemple #2
0
def run_analysis_helper(fname: str, identifier: str, cmd_args: ArgNamespace):
    from autopandas_v2.generators.ml.networks.ggnn.ops.choice import ModelChoice
    from autopandas_v2.generators.ml.networks.ggnn.ops.chain import ModelChain
    from autopandas_v2.generators.ml.networks.ggnn.ops.select import ModelSelect
    from autopandas_v2.generators.ml.networks.ggnn.ops.subsets import ModelSubsets
    from autopandas_v2.generators.ml.networks.ggnn.ops.orderedsubsets import ModelOrderedSubsets
    from autopandas_v2.ml.networks.ggnn.utils import ParamsNamespace

    test_path = '{}/{}/{}.pkl'.format(cmd_args.test, fname, identifier)
    model_path = '{}/{}/{}'.format(cmd_args.modeldir, fname, identifier)
    if not os.path.exists(test_path):
        raise Exception("Test data path {} does not exist".format(test_path))

    os.system('mkdir -p ' + model_path)
    ggnn_args = ArgNamespace.from_namespace(cmd_args)
    ggnn_args.test = test_path
    ggnn_args.outdir = model_path
    ggnn_args.mode = 'train'
    ggnn_args.model = model_path

    params = ParamsNamespace()

    if cmd_args.config is not None:
        with open(cmd_args.config, 'r') as f:
            params.update(json.load(f))

    params.args = ParamsNamespace()
    params.args.update(ggnn_args)
    params.use_directed_edges = True

    if identifier.startswith("choice"):
        model = ModelChoice.from_params(params)

    elif identifier.startswith("chain"):
        model = ModelChain.from_params(params)

    elif identifier.startswith("select"):
        model = ModelSelect.from_params(params)

    elif identifier.startswith("subsets"):
        model = ModelSubsets.from_params(params)

    elif identifier.startswith("orderedsubsets"):
        model = ModelOrderedSubsets.from_params(params)

    else:
        raise NotImplementedError("Model not defined for operator {}".format(
            identifier.split('_')[0]))

    return model.run_analysis(test_path)
Exemple #3
0
def run_training_generators_helper(fname: str, identifier: str,
                                   cmd_args: ArgNamespace):
    from autopandas_v2.generators.ml.networks.ggnn.ops.choice import ModelChoice
    from autopandas_v2.generators.ml.networks.ggnn.ops.chain import ModelChain
    from autopandas_v2.generators.ml.networks.ggnn.ops.select import ModelSelect
    from autopandas_v2.generators.ml.networks.ggnn.ops.subsets import ModelSubsets
    from autopandas_v2.generators.ml.networks.ggnn.ops.orderedsubsets import ModelOrderedSubsets
    from autopandas_v2.ml.networks.ggnn.utils import ParamsNamespace

    train_path = '{}/{}/{}.pkl'.format(cmd_args.train, fname, identifier)
    valid_path = '{}/{}/{}.pkl'.format(cmd_args.valid, fname, identifier)
    model_path = '{}/{}/{}'.format(cmd_args.modeldir, fname, identifier)
    if not os.path.exists(train_path):
        raise Exception(
            "Training data path {} does not exist".format(train_path))

    if not os.path.exists(valid_path):
        raise Exception(
            "Validation data path {} does not exist".format(valid_path))

    if cmd_args.ignore_if_exists and os.path.exists(model_path +
                                                    '/model_best.pickle'):
        logger.info(
            "Skipping training for {}:{} as model already exists".format(
                fname, identifier))
        return

    os.system('mkdir -p ' + model_path)
    ggnn_args = ArgNamespace.from_namespace(cmd_args)
    ggnn_args.train = train_path
    ggnn_args.valid = valid_path
    ggnn_args.outdir = model_path
    ggnn_args.mode = 'train'

    if cmd_args.restore_if_exists and os.path.exists(model_path +
                                                     '/model_best.pickle'):
        ggnn_args.restore = model_path + '/model_best.pickle'

    params = ParamsNamespace()

    if cmd_args.config is not None:
        with open(cmd_args.config, 'r') as f:
            params.update(json.load(f))

    if cmd_args.config_str is not None:
        params.update(json.loads(cmd_args.config_str))

    params.args = ParamsNamespace()
    params.args.update(ggnn_args)
    params.use_directed_edges = True

    if identifier.startswith("choice"):
        model = ModelChoice.from_params(params)

    elif identifier.startswith("chain"):
        model = ModelChain.from_params(params)

    elif identifier.startswith("select"):
        model = ModelSelect.from_params(params)

    elif identifier.startswith("subsets"):
        model = ModelSubsets.from_params(params)

    elif identifier.startswith("orderedsubsets"):
        model = ModelOrderedSubsets.from_params(params)

    else:
        raise NotImplementedError("Model not defined for operator {}".format(
            identifier.split('_')[0]))

    model.run()