def test_all():
m = svmrank.Model({'-c': 3})
m.fit(train_xs, train_ys, train_groups)
fd, path = tempfile.mkstemp()
m.write(path)
m = svmrank.Model()
m.read(path)
test_preds = m.predict(test_xs, test_groups)
test_loss = m.loss(test_ys, test_preds, test_groups)
def test_write_read():
m = svmrank.Model({'-c': 3})
fd, path = tempfile.mkstemp()
ok = False
try:
m.write(path)
except ValueError:
ok = True
assert ok
m = svmrank.Model({'-c': 3})
m.fit(train_xs, train_ys, train_groups)
fd, path = tempfile.mkstemp()
m.write(path)
m = svmrank.Model()
m.read(path)
def test_predict():
m = svmrank.Model({'-c': 3})
ok = False
try:
preds = m.predict(test_xs, test_groups)
except ValueError:
ok = True
assert ok
m.fit(train_xs, train_ys, train_groups)
preds = m.predict(test_xs, test_groups)
def test_params():
m = svmrank.Model()
m.set_params({'-c': 3})
m._apply_params()
m.set_params({'-c': -1})
ok = False
try:
m._apply_params()
except ValueError:
ok = True
assert ok
def test_loss():
m = svmrank.Model({'-c': 3})
m.fit(train_xs, train_ys, train_groups)
# fd, path = tempfile.mkstemp()
# m.write(path)
# m = svmrank.Model()
# m.read(path)
train_preds = m.predict(train_xs, train_groups)
train_loss = m.loss(train_ys, train_preds, train_groups)
print(f"training loss: {train_loss}")
for y, pred, group in zip(train_ys, train_preds, train_groups):
print(f"group: {group} y: {y} pred: {pred}")
test_preds = m.predict(test_xs, test_groups)
test_loss = m.loss(test_ys, test_preds, test_groups)
print(f"test loss: {test_loss}")
for y, pred, group in zip(test_ys, test_preds, test_groups):
print(f"group: {group} y: {y} pred: {pred}")
log(f"Validation log-NDCG: {np.log(loss)}", logfile)
elif args.model == 'svmrank':
import svmrank
train_qids = np.repeat(np.arange(len(train_ncands)), train_ncands)
valid_qids = np.repeat(np.arange(len(valid_ncands)), valid_ncands)
# Training (includes hyper-parameter tuning)
best_loss = np.inf
best_model = None
for c in (1e-3, 1e-2, 1e-1, 1e0):
log(f"C: {c}", logfile)
model = svmrank.Model({
'-c': c * len(train_ncands), # c_light = c_rank / n
'-v': 1,
'-y': 0,
'-l': 2,
})
model.fit(train_x, train_y, train_qids)
loss = model.loss(train_y, model(train_x, train_qids), train_qids)
log(f" training loss: {loss}", logfile)
loss = model.loss(valid_y, model(valid_x, valid_qids), valid_qids)
log(f" validation loss: {loss}", logfile)
if loss < best_loss:
best_model = model
best_loss = loss
best_c = c
# save model
model.write(f"{running_dir}/model.txt")
log(f"Best model with C={best_c}, validation loss: {best_loss}",
policy['model'] = loaded_models[policy['name']]
# load ml-competitor models
for policy in branching_policies:
if policy['type'] == 'ml-competitor':
try:
with open(f"{policy['model']}/normalization.pkl", 'rb') as f:
policy['feat_shift'], policy['feat_scale'] = pickle.load(f)
except:
policy['feat_shift'], policy['feat_scale'] = 0, 1
with open(f"{policy['model']}/feat_specs.pkl", 'rb') as f:
policy['feat_specs'] = pickle.load(f)
if policy['name'].startswith('svmrank'):
policy['model'] = svmrank.Model().read(
f"{policy['model']}/model.txt")
else:
with open(f"{policy['model']}/model.pkl", 'rb') as f:
policy['model'] = pickle.load(f)
print("running SCIP...")
fieldnames = [
'policy',
'seed',
'type',
'instance',
'nnodes',
'nlps',
'stime',
'gap',
policy['batch_fun'] = load_batch_gcnn
else:
# load feature normalization parameters
try:
with open(
f"trained_models/{args.problem}/{policy['name']}/{seed}/normalization.pkl",
'rb') as f:
policy['feat_shift'], policy[
'feat_scale'] = pickle.load(f)
except:
policy['feat_shift'], policy['feat_scale'] = 0, 1
# load model
if policy_name.startswith('svmrank'):
policy['model'] = svmrank.Model().read(
f"trained_models/{args.problem}/{policy['name']}/{seed}/model.txt"
)
else:
with open(
f"trained_models/{args.problem}/{policy['name']}/{seed}/model.pkl",
'rb') as f:
policy['model'] = pickle.load(f)
# load feature specifications
with open(
f"trained_models/{args.problem}/{policy['name']}/{seed}/feat_specs.pkl",
'rb') as f:
feat_specs = pickle.load(f)
policy['batch_datatypes'] = [
tf.float32, tf.int32, tf.int32, tf.float32
def test_alloc_dealloc():
m = svmrank.Model()
def test_fit():
m = svmrank.Model({'-c': 3})
m.fit(train_xs, train_ys, train_groups)