def test_run_job():
os.environ[
'GOOGLE_APPLICATION_CREDENTIALS'] = "./google_cloud_storage_APIs/kaggle-pets-26ce7fa96dc3.json" # this should not be needed on the cloud
os.system("rm -r " + DATA_FOLDER)
os.system("rm -r " + RESULTS_FOLDER)
train.download_data()
name = "test_xgboost"
train.run_model(name)
assert os.path.isfile(os.path.join(RESULTS_FOLDER, name + ".bst"))
assert os.path.isfile(os.path.join(RESULTS_FOLDER, name + ".score"))
def main_cli():
parser = argparse.ArgumentParser('CEDR model re-ranking')
parser.add_argument('--model',
choices=train.MODEL_MAP.keys(),
default='vanilla_bert')
parser.add_argument('--datafiles', type=argparse.FileType('rt'), nargs='+')
# parser.add_argument('--run', type=argparse.FileType('rt'))
parser.add_argument('--passage',
choices=["first", "max", "sum"],
default="first")
parser.add_argument('--model_weights', type=argparse.FileType('rb'))
parser.add_argument('--out_path')
args = parser.parse_args()
model = train.MODEL_MAP[args.model]().cuda()
print("Reading testing file %s" % args.datafiles, flush=True)
testTable = pd.read_csv(args.datafiles[0],
sep='\t',
header=0,
error_bad_lines=False,
index_col=False)
print(testTable.columns.values)
if args.passage == "max" or args.passage == "sum" or args.passage == "first":
testTable = trainMZdataframe.applyPassaging(testTable, 150, 75, 0)
train.aggregation = args.passage
signal.signal(signal.SIGUSR1,
lambda sig, stack: traceback.print_stack(stack))
docs = {}
queries = {}
for index, row in testTable.iterrows():
queries[row['id_left']] = row['text_left']
docs[row['id_right']] = row['text_right']
dataset = (queries, docs)
test_run = {}
for index, row in testTable.iterrows():
test_run.setdefault(row['id_left'], {})[row['id_right']] = float(1)
if args.model_weights is not None:
model.load(args.model_weights)
#import pdb; pdb.set_trace()
train.run_model(model, dataset, test_run, args.out_path, desc='rerank')
def main_cli():
parser = argparse.ArgumentParser('CEDR model re-ranking')
parser.add_argument('--model',
choices=train.MODEL_MAP.keys(),
default='vanilla_bert')
parser.add_argument('--datafiles', type=argparse.FileType('rt'), nargs='+')
parser.add_argument('--run', type=argparse.FileType('rt'))
parser.add_argument('--model_weights', type=argparse.FileType('rb'))
parser.add_argument('--out_path', type=argparse.FileType('wt'))
args = parser.parse_args()
model = train.MODEL_MAP[args.model]().cuda()
dataset = data.read_datafiles(args.datafiles)
run = data.read_run_dict(args.run)
if args.model_weights is not None:
model.load(args.model_weights.name)
train.run_model(model, dataset, run, args.out_path.name, desc='rerank')
def inference(self):
weights = {
'attention_h': tf.Variable(tf.random_normal([self.dnc_hidden, 1]), name='attention_h_w'),
'attention_r': tf.Variable(tf.random_normal([self.dnc_hidden*2, 1]), name='attention_r_w'),
'attention_out': tf.Variable(tf.random_normal([self.dnc_hidden, self.attention_hidden]), name='attention_out_w'),
'out': tf.Variable(tf.random_normal([self.attention_hidden, self.relation_classes]), name='out_w')
}
biases = {
'attention': tf.Variable(tf.random_normal([self.max_sentence_len]), name='attention'),
'out': tf.Variable(tf.random_normal([self.relation_classes]), name='out_b')
}
#-------------------------------------- inputs ----------------------------------------------
sentence_input = tf.placeholder('float', [None, self.max_sentence_len, self.input_size], name='sentence_input')
sentence_input_T = tf.transpose(sentence_input, [1,0,2])
sentence_len_input = tf.placeholder('int32', [None], name='sentence_len_input')
dropout_input = tf.placeholder('float', [2], name='dropout_input')
# 按entity位置在rnn_out中查询，找到entity对应的rnn_out
entity_input = tf.placeholder('int32', [None,2], 'entity_input')
entity_input_onehot = tf.one_hot(entity_input, self.max_sentence_len)
#-------------------------------------- dnc ----------------------------------------------
dnc_out = dnc_train.run_model(sentence_input_T, self.dnc_hidden, sentence_len_input)
dnc_out = tf.transpose(dnc_out, [1,0,2])
# 按entity位置在dnc_out中查询，找到entity对应的rnn_out
entity_rnn_out = tf.matmul(entity_input_onehot,dnc_out)
entity_rnn_out = tf.concat([entity_rnn_out[:,0,:],entity_rnn_out[:,1,:]],1)
#-------------------------------------- attention_layer ----------------------------------------------
with tf.variable_scope('attention_layer'):
# attention rate:softmax(tanh(rnn_out*W1 + rnn_last*W2 + b))
# attention_r = tf.matmul(entity_rnn_out, weights['attention_r'])
# attention_r = tf.matmul(attention_r, tf.zeros([1,self.max_sentence_len])+1) #复制多份，适应句长
attention_h = tf.reshape(tf.matmul(tf.reshape(dnc_out,[-1,self.dnc_hidden]), weights['attention_h']), [-1,self.max_sentence_len])
attention_rate = tf.expand_dims(tf.nn.softmax(tf.nn.tanh(attention_h + biases['attention'])) ,1)
attention_out = tf.matmul(tf.reshape(tf.matmul(attention_rate, dnc_out), [-1,self.dnc_hidden]), weights['attention_out'])
#-------------------------------------- output ----------------------------------------------
with tf.variable_scope('output'):
out_dropout = tf.nn.dropout(attention_out, dropout_input[1])
# out_dropout = tf.nn.dropout(dnc_out[:,-1,:], dropout_input[1])
pred_label = tf.matmul(tf.nn.sigmoid(out_dropout), weights['out']) + biases['out']
return pred_label, sentence_input, sentence_len_input, entity_input, dropout_input
import numpy as np
import pickle
from model import FactorizationMachine
from train import run_model
x = np.array([[1, 0, 0, 1, 0, 0, 0, 0.3, 0.3, 0.3, 0.0, 13, 0, 0, 0, 0],
[1, 0, 0, 0, 1, 0, 0, 0.3, 0.3, 0.3, 0.0, 14, 1, 0, 0, 0],
[1, 0, 0, 0, 0, 1, 0, 0.3, 0.3, 0.3, 0.0, 16, 0, 1, 0, 0],
[0, 1, 0, 0, 0, 1, 0, 0.0, 0.0, 0.5, 0.5, 5, 0, 0, 0, 0],
[0, 1, 0, 0, 0, 0, 1, 0.0, 0.0, 0.5, 0.5, 8, 0, 0, 1, 0],
[0, 0, 1, 1, 0, 0, 0, 0.5, 0.0, 0.5, 0.0, 9, 0, 0, 0, 0],
[0, 0, 1, 0, 0, 1, 0, 0.5, 0.0, 0.5, 0.0, 12, 1, 0, 0, 0]],
dtype=np.float32)
y = np.array([5, 3, 1, 4, 5, 1, 5], dtype=np.float32)
model = FactorizationMachine(data_shape=16, latent_shape=10)
trained = run_model(model, x, y, epochs=200)
pickle.dump(trained, open("FM.model", "wb"))
def valid(net, loader):
criterion = torch.nn.CrossEntropyLoss()
_, acc = run_model(net, loader, criterion, None, train=False)
return -acc
def main_cli():
MODEL_MAP = train.modeling.MODEL_MAP
parser = argparse.ArgumentParser('CEDR model re-ranking')
parser.add_argument('--model', choices=MODEL_MAP.keys(), default='vanilla_bert')
parser.add_argument('--submodel1', choices=MODEL_MAP.keys(), default=None)
parser.add_argument('--submodel2', choices=MODEL_MAP.keys(), default=None)
parser.add_argument('--submodel3', choices=MODEL_MAP.keys(), default=None)
parser.add_argument('--submodel4', choices=MODEL_MAP.keys(), default=None)
parser.add_argument('--submodel5', choices=MODEL_MAP.keys(), default=None)
parser.add_argument('--submodel6', choices=MODEL_MAP.keys(), default=None)
parser.add_argument('--submodel7', choices=MODEL_MAP.keys(), default=None)
parser.add_argument('--submodel8', choices=MODEL_MAP.keys(), default=None)
parser.add_argument('--datafiles', type=argparse.FileType('rt'), nargs='+')
parser.add_argument('--run', type=argparse.FileType('rt'))
#parser.add_argument('--model_weights', type=argparse.FileType('rb'))
parser.add_argument('--model_weights', type=str, default=None)
parser.add_argument('--out_path', type=argparse.FileType('wt'))
parser.add_argument('--gpunum', type=str, default="0", help='gup number')
parser.add_argument('--random_seed', type=int, default=42, help='random seed')
args = parser.parse_args()
#setRandomSeed(args.random_seed)
os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"]=args.gpunum
print("GPU count=", torch.cuda.device_count())
#model = train.MODEL_MAP[args.model]().cuda()
if(args.model.startswith('duet')):
model = MODEL_MAP[args.model](
MODEL_MAP[args.submodel1](),
MODEL_MAP[args.submodel2]()
)
elif(args.model.startswith('trio')):
model = MODEL_MAP[args.model](
MODEL_MAP[args.submodel1](),
MODEL_MAP[args.submodel2](),
MODEL_MAP[args.submodel3]()
)
elif(args.model.startswith('quad')):
model = MODEL_MAP[args.model](
MODEL_MAP[args.submodel1](),
MODEL_MAP[args.submodel2](),
MODEL_MAP[args.submodel3](),
MODEL_MAP[args.submodel4]()
)
elif(args.model.startswith('octo')):
model = MODEL_MAP[args.model](
MODEL_MAP[args.submodel1](),
MODEL_MAP[args.submodel2](),
MODEL_MAP[args.submodel3](),
MODEL_MAP[args.submodel4](),
MODEL_MAP[args.submodel5](),
MODEL_MAP[args.submodel6](),
MODEL_MAP[args.submodel7](),
MODEL_MAP[args.submodel8]()
)
else:
model = MODEL_MAP[args.model]().cuda()
dataset = data.read_datafiles(args.datafiles)
run = data.read_run_dict(args.run)
if(args.model_weights is not None):
wts = args.model_weights.split(',')
if(len(wts) == 1):
model.load(wts[0])
elif(len(wts) == 2):
model.load_duet(wts[0], wts[1])
elif(len(wts) == 4):
model.load_quad(wts[0], wts[1], wts[2], wts[3])
elif(len(wts) == 6):
model.load_hexa(wts[0], wts[1], wts[2], wts[3], wts[4], wts[5])
elif(len(wts) == 8):
model.load_octo(wts[0], wts[1], wts[2], wts[3], wts[4], wts[5], wts[6], wts[7])
train.run_model(model, dataset, run, args.out_path.name, desc='rerank')
def run():
print("RUNNING")
coefs = run_model()
return ceofs