def main():
os_utils._makedirs(params["summary_dir"], force=True)
X_train, y_train, X_test, y_test = DataLoader.load(DATA_DIR, TASK_NAMES)
vocabulary = Vocabulary(max_num_word=params["max_num_word"])
words = []
for val in X_train.values():
words.extend(val)
vocabulary.fit(words)
params["max_num_word"] = vocabulary.max_num_word
for task_name in TASK_NAMES:
X_train[task_name] = vocabulary.transform(X_train[task_name])
X_test[task_name] = vocabulary.transform(X_test[task_name])
X_train[task_name] = pad_sequences(
X_train[task_name],
maxlen=params["max_seq_len_word"],
padding=params["pad_sequences_padding"],
truncating=params["pad_sequences_truncating"])
X_test[task_name] = pad_sequences(
X_test[task_name],
maxlen=params["max_seq_len_word"],
padding=params["pad_sequences_padding"],
truncating=params["pad_sequences_truncating"])
model = SharedPrivateModel(params, TASK_NAMES)
model.fit(X_train, y_train, X_valid=X_test, y_valid=y_test)
def main():
# source domain
print("load svhn")
svhn_images_train, _ = DataLoader.load_svhn(SVHN_DIR, "train_32x32.mat")
svhn_images_test, svhn_labels_test = DataLoader.load_svhn(SVHN_DIR, "test_32x32.mat")
svhn_images_extra, svhn_labels_extra = DataLoader.load_svhn(SVHN_DIR, "extra_32x32.mat")
auxiliary_data = {
"X_train": svhn_images_extra,
"y_train": svhn_labels_extra,
"X_test": svhn_images_test,
"y_test": svhn_labels_test,
}
# target domain
print("load mnist")
if not os.path.isfile(os.path.join(MNIST_DIR, "train.pkl")):
DataLoader.prepare_mnist(MNIST_DIR, "train")
mnist_images_train, _ = DataLoader.load_mnist(MNIST_DIR, "train")
# dtn model
print("init dtn")
os_utils._makedirs(params["summary_dir"], force=True)
os_utils._makedirs(params["log_dir"])
logger = log_utils._get_logger(params["log_dir"], "tf-%s.log" % time_utils._timestamp())
model = DomainTransferNet(params, logger)
print("fit dtn")
model.fit(auxiliary_data, Xs_train=svhn_images_train, Xt_train=mnist_images_train)
print("evaluate dtn")
model.evaluate(Xs=svhn_images_train, sample_batch=100, batch_size=100, sample_dir=SAMPLE_DIR)
def main(options):
os_utils._makedirs("../logs")
os_utils._makedirs("../output")
os_utils._makedirs(params["offline_model_dir"])
os_utils._makedirs(params["pb_model_dir"])
logger = log_utils._get_logger("../logs", "tf-%s.log" % time_utils._timestamp())
params["granularity"] = options.granularity
# save path
model_name = "augmentation_%s_%s_%s"%(str(options.augmentation), options.granularity, options.model)
path = config.SUB_DIR + "/" + model_name
os_utils._makedirs(path)
# load data
X_dev, X_valid, Q, X_itest= get_train_valid_test_data(options.augmentation)
# validation
model = get_model(options.model)(params, logger, init_embedding_matrix=init_embedding_matrix)
if os.path.exists(params["offline_model_dir"] + "/checkpoint"):
print('restoring model.......')
model.restore_session()
train_model = True
if train_model:
print('training model...')
model.fit(X_dev, Q, validation_data=X_valid, shuffle=True)
print('ready to save model....')
model.save_session()
print('model save done!')
y_pred_itest = model.predict_proba(X_itest, Q).flatten()
#print('build saving.....')
if not os.path.exists(params["pb_model_dir"]+'/1'):
build_model.build_save(model,str(1),params["pb_model_dir"])
#acu
assert(len(y_pred_itest)==len(X_itest["label"]))
print(len(y_pred_itest))
print(len(X_itest["label"]))
count = 0
for i in range(len(y_pred_itest)):
score = y_pred_itest[i]
if score > 0.5:
prob = 1
else:
prob = 0
if prob == X_itest["label"][i]:
count += 1
print(count/len(y_pred_itest))
# save for stacking
df = pd.DataFrame({"y_pred": y_pred_itest, "y_true": X_itest["label"]})
df.to_csv(path + "/valid.csv", index=False, header=True)
input('wait')
print('save done!')
def main():
model_type = None
if len(sys.argv) > 1:
model_type = sys.argv[1]
os_utils._makedirs("../logs")
os_utils._makedirs("../output")
logger = log_utils._get_logger("../logs", "tf-%s.log" % time_utils._timestamp())
# load data
Q = load_question(params)
dfTrain = load_train()
dfTest = load_test()
train_features = np.load(config.TRAIN_FEATURES_FILE)
test_features = np.load(config.TEST_FEATURES_FILE)
params["num_features"] = train_features.shape[1]
# load split
with open(config.SPLIT_FILE, "rb") as f:
train_idx, valid_idx = pkl.load(f)
# validation
X_train = get_model_data(dfTrain.loc[train_idx], train_features[train_idx], params)
X_valid = get_model_data(dfTrain.loc[valid_idx], train_features[valid_idx], params)
model = get_model(model_type)(params, logger, init_embedding_matrix=init_embedding_matrix)
model.fit(X_train, Q, validation_data=X_valid, shuffle=True)
# submit
X_train = get_model_data(dfTrain, train_features, params)
X_test = get_model_data(dfTest, test_features, params)
y_proba = np.zeros((dfTest.shape[0], params["n_runs"]), dtype=np.float32)
for run in range(params["n_runs"]):
params["random_seed"] = run
params["model_name"] = "semantic_model_%s"%str(run+1)
model = get_model(model_type)(params, logger, init_embedding_matrix=init_embedding_matrix)
model.fit(X_train, Q, validation_data=None, shuffle=True)
y_proba[:,run] = model.predict_proba(X_test, Q).flatten()
dfTest["y_pre"] = np.mean(y_proba[:,:(run+1)], axis=1)
dfTest[["y_pre"]].to_csv(config.SINGLE_SUB_FILE_PATTERN%(model_type, str(run+1)), header=True, index=False)
def main():
model_type = None
if len(sys.argv) > 1:
model_type = sys.argv[1]
os_utils._makedirs("../logs")
os_utils._makedirs("../output")
logger = log_utils._get_logger("../logs",
"tf-%s.log" % time_utils._timestamp())
Q = load_question(params)
dfTrain = load_train()
dfTest = load_test()
X_test = get_model_data(dfTest, params)
# shuffle training data
dfTrain = dfTrain.sample(frac=1.0)
# validation
train_ratio = 0.7
N = dfTrain.shape[0]
train_num = int(N * train_ratio)
X_train = get_model_data(dfTrain[:train_num], params)
X_valid = get_model_data(dfTrain[train_num:], params)
model = get_model(model_type)(params,
logger,
init_embedding_matrix=init_embedding_matrix)
model.fit(X_train, Q, validation_data=X_valid, shuffle=True)
# submit
X_train = get_model_data(dfTrain, params)
y_proba = np.zeros((dfTest.shape[0], params["n_runs"]), dtype=np.float32)
for run in range(params["n_runs"]):
params["random_seed"] = run
params["model_name"] = "semantic_model_%s" % str(run + 1)
model = get_model(model_type)(
params, logger, init_embedding_matrix=init_embedding_matrix)
model.fit(X_train, Q, validation_data=None, shuffle=True)
y_proba[:, run] = model.predict_proba(X_test, Q).flatten()
dfTest["y_pre"] = np.mean(y_proba[:, :(run + 1)], axis=1)
dfTest[["y_pre"]].to_csv(config.SUB_FILE_PATTERN % str(run + 1),
header=True,
index=False)
def main(options):
os_utils._makedirs("../logs")
os_utils._makedirs("../output")
logger = log_utils._get_logger("../logs",
"tf-%s.log" % time_utils._timestamp())
params["granularity"] = options.granularity
# save path
model_name = "augmentation_%s_%s_%s" % (str(
options.augmentation), options.granularity, options.model)
path = config.SUB_DIR + "/" + model_name
os_utils._makedirs(path)
# load data
X_dev, X_valid, X_train, X_test, Q = get_train_valid_test_data(
options.augmentation)
# validation
model = get_model(options.model)(
params, logger, init_embedding_matrix=init_embedding_matrix)
model.fit(X_dev, Q, validation_data=X_valid, shuffle=True)
y_pred_valid = model.predict_proba(X_valid, Q).flatten()
# save for stacking
df = pd.DataFrame({"y_pred": y_pred_valid, "y_true": X_valid["label"]})
df.to_csv(path + "/valid.csv", index=False, header=True)
# submission
y_proba = np.zeros((len(X_test["label"]), params["n_runs"]),
dtype=np.float32)
for run in range(params["n_runs"]):
params["random_seed"] = run
params["model_name"] = "semantic_model_%s" % str(run + 1)
model = get_model(options.model)(
params, logger, init_embedding_matrix=init_embedding_matrix)
model.fit(X_train, Q, validation_data=None, shuffle=True)
y_proba[:, run] = model.predict_proba(X_test, Q).flatten()
df = pd.DataFrame(
y_proba[:, :(run + 1)],
columns=["y_proba_%d" % (i + 1) for i in range(run + 1)])
df.to_csv(path + "/test.csv", index=False, header=True)