from core.features.layers import wrap_extractor, get_default_processing
from core.models.input import SquareInput
# load config
options = args("fc")
config = importlib.import_module("configs." + options.config)
config = apply_options(config, options)
# data featurizers
extractor = config.extractor
featurizer = wrap_extractor(extractor)
preprocess = lambda cluster: (featurizer(cluster), cluster["nparts"], cluster[
"props"])
# get data
train_examples, dev_set, test_set, test_raw = make_datasets(
config, preprocess, preprocess)
# data processing
processing = get_default_processing(train_examples, extractor,
preprocess_y(1, 3),
config.preprocessing_mode)
# model
model = SquareInput(config, config.n_eta, config.n_phi, config.n_features)
model.build("light")
path1 = "results/20170318_161623/model.weights/"
path2 = "results/20170318_234936/model.weights/"
model.combine(path1, path2, processing, test_set)
ReLu, Conv2d, MaxPool, Combine, Reduce, Embedding
# load config
options = args("embeddings")
config = importlib.import_module("configs." + options.config)
config = apply_options(config, options)
# data featurizer
featurizer = embedding_features(config.modes)
preprocess = lambda cluster: (featurizer(cluster), cluster["nparts"], cluster[
"props"])
featurizer_raw = wrap_extractor(config.extractor)
preprocess_raw = lambda cluster: (featurizer_raw(cluster), cluster["nparts"])
# get data
train_examples, dev_set, test_set, test_raw = make_datasets(
config, preprocess, preprocess_raw)
# data processing
processing = get_default_processing(train_examples, config.n_features,
preprocess_y(1, 3), config.max_n_cells,
config.pad_tok, config.preprocessing_mode)
# model
model = EmbeddingsInput(config)
model.build("light")
path1 = "results/20170318_165602/model.weights/"
path2 = "results/20170318_234736/model.weights/"
model.combine(path1, path2, processing, test_set)
from core.utils.evaluate import baseline, f1score
from core.utils.data import get_xy
import tensorflow as tf
# load config
options = args("baseline")
config = importlib.import_module("configs." + options.config)
config = apply_options(config, options)
# data extraction
featurizer = simple_features(config.tops, config.feature_mode)
preprocess = lambda cluster: (featurizer(cluster), cluster["nparts"], cluster[
"props"])
featurizer_raw = wrap_extractor(config.extractor)
preprocess_raw = lambda cluster: (featurizer_raw(cluster), cluster["nparts"])
# get data
train_examples, dev_set, test_set, test_raw = make_datasets(
config, preprocess, preprocess_raw)
# data processing
processing = get_default_processing(train_examples, preprocess_y(1, 3),
config.preprocessing_mode)
# model
model = FlatInput(config, config.input_size)
model.build("light")
path1 = "results/20170318_132042/model.weights/"
path2 = "results/20170318_234539/model.weights/"
model.combine(path1, path2, processing, test_set)
from core.models.input import FlatInput
from core.utils.evaluate import baseline
from core.utils.data import get_xy
# load config
options = args("baseline")
config = importlib.import_module("configs." + options.config)
config = apply_options(config, options)
# data extraction
featurizer = simple_features(config.tops, config.feature_mode)
preprocess = lambda cluster: (featurizer(cluster), cluster["nparts"], cluster[
"props"])
featurizer_raw = wrap_extractor(config.extractor)
preprocess_raw = lambda cluster: (featurizer_raw(cluster), cluster["nparts"])
# get data
train_examples, dev_set, test_set, test_raw = make_datasets(
config, preprocess, preprocess_raw)
# data processing
processing = get_default_processing(
train_examples, preprocess_y(config.part_min, config.output_size),
config.preprocessing_mode)
# model
model = FlatInput(config, config.input_size)
model.build()
model.train(train_examples, dev_set, processing)
acc, base = model.evaluate(test_set, processing, test_raw,
featurized_export_result)
# data featurizer
featurizer = embedding_features(config.modes)
preprocess = lambda cluster: (featurizer(cluster), cluster["nparts"], cluster[
"props"])
featurizer_raw = wrap_extractor(config.extractor)
preprocess_raw = lambda cluster: (featurizer_raw(cluster), cluster["nparts"])
# get data
train_examples, dev_set, test_set, test_raw = make_datasets(
config, preprocess, preprocess_raw)
# data processing
processing = get_default_processing(
train_examples, config.n_features,
preprocess_y(config.part_min, config.output_size), config.max_n_cells,
config.pad_tok, config.preprocessing_mode)
# model
model = EmbeddingsInput(config)
model.build()
model.train(train_examples, dev_set, processing)
acc, base = model.evaluate(test_set, processing)
export_clustering(model,
config.embedding_node,
test_set,
processing,
config,
default=True,
n_components=3)
from core.utils.evaluate import featurized_export_result
from core.utils.preprocess import preprocess_y
from core.dataset.pickle import make_datasets
from core.features.layers import wrap_extractor, get_default_processing
from core.models.input import SquareInput
# load config
options = args("fc")
config = importlib.import_module("configs."+options.config)
config = apply_options(config, options)
# data featurizers
extractor = config.extractor
featurizer = wrap_extractor(extractor)
preprocess = lambda cluster: (featurizer(cluster), cluster["nparts"], cluster["props"])
# get data
train_examples, dev_set, test_set, test_raw = make_datasets(config, preprocess, preprocess)
# data processing
processing = get_default_processing(train_examples, extractor,
preprocess_y(config.part_min, config.output_size), config.preprocessing_mode)
# model
model = SquareInput(config, config.n_eta, config.n_phi, config.n_features)
model.build()
model.train(train_examples, dev_set, processing)
acc, base = model.evaluate(test_set, processing, test_raw, featurized_export_result)
# get data
train_examples, dev_set, test_set, test_raw = make_datasets(
config, preprocess, preprocess_raw)
all_ids = set()
for (x, y), i in train_examples:
all_ids.update(x["ids"])
# dynamically allocate the vocab size
config.n_cells = max(all_ids) + 3
config.unk_tok_id = config.n_cells - 1
config.pad_tok_id = config.n_cells - 2
for layer in config.layers:
if layer.__class__.__name__ == "Embedding":
print "Setting vocab_size to {}".format(config.n_cells)
layer.vocab_size = config.n_cells
# data processing
processing = get_default_processing(train_examples, config.n_features,
preprocess_y(config.output_size),
config.max_n_cells, config.n_cells - 3,
config.pad_tok_id, config.unk_tok_id,
config.pad_tok_feat, "none")
# model
model = IdInput(config)
model.build()
model.train(train_examples, dev_set, processing)
acc, base = model.evaluate(test_set, processing, test_raw,
featurized_export_result)