def bind_model(sess, FLAGS):
def save(dir_path, *args):
os.makedirs(dir_path, exist_ok=True)
saver = tf.train.Saver()
saver.save(sess, os.path.join(dir_path, 'model'))
with open(os.path.join(dir_path,FLAGS.map_file), "wb") as f:
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
def load(dir_path, *args):
global char_to_id
global id_to_tag
config = load_config(FLAGS.config_file)
logger = get_logger(FLAGS.log_file)
tf.get_variable_scope().reuse_variables()
with open(os.path.join(dir_path,FLAGS.map_file), "rb") as f:
char_to_id, _, __, id_to_tag = pickle.load(f)
saver = tf.train.Saver()
ckpt = tf.train.get_checkpoint_state(dir_path)
if ckpt and ckpt.model_checkpoint_path:
checkpoint = os.path.basename(ckpt.model_checkpoint_path)
saver.restore(sess, os.path.join(dir_path, checkpoint))
else:
raise NotImplemented('No checkpoint found!')
print ('model loaded!')
def infer(sentences, **kwargs):
config = load_config(FLAGS.config_file)
logger = get_logger(FLAGS.log_file)
reformed_sentences = [' '.join(sen[1]) for sen in sentences]
result = model.evaluate_lines(sess, inputs_from_sentences(reformed_sentences, char_to_id, FLAGS.max_char_length), id_to_tag)
'''
result = [
(0.0, ['ARG0', 'ARG3', '-']),
(0.0, ['ARG0', 'ARG1', '-'])
]
# evaluate_lines 함수는 문장 단위 분석 결과를 내어줍니다.
# len(result) : 문장의 갯수, 따라서 위 예제는 두 문장의 결과입니다.
# result[0] : 첫번째 문장의 분석 결과, result[1] : 두번째 문장의 분석 결과.
# 각 문장의 분석 결과는 다시 (prob, [labels])로 구성됩니다.
# prob에 해당하는 자료는 이번 task에서 사용하지 않습니다. 따라서 그 값이 결과에 영향을 미치지 않습니다.
# [labels]는 각 어절의 분석 결과를 담고 있습니다. 따라서 다음과 같이 구성됩니다.
## ['첫번째 어절의 분석 결과', '두번째 어절의 분석 결과', ...]
# 예를 들면 위 주어진 예제에서 첫번째 문장의 첫번째 어절은 'ARG0'을, 첫번째 문장의 두번째 어절은 'ARG3'을 argument label로 가집니다.
### 주의사항 ###
# 모든 어절의 결과를 제출하여야 합니다.
# 어절의 순서가 지켜져야 합니다. (첫번째 어절부터 순서대로 list 내에 위치하여야 합니다.)
'''
return result
if IS_ON_NSML:
nsml.bind(save=save, load=load, infer=infer)
def bind_nsml(model, tokenizer=None, args=None):
def save(path, **kwargs):
torch.save(model.state_dict(),
open(os.path.join(path, 'model.pt'), 'wb'))
def load(path, **kwargs):
model.load_state_dict(
torch.load(open(os.path.join(path, 'model.pt'), 'rb'),
map_location=lambda storage, loc: storage))
def infer(test_data, **kwargs):
'''
:param test_data: list of noisy sentences
:return: list of corrected sentences
'''
# 특수문자 지우기 일단 꺼놓음.
# test_data = preprocess_noisy_sentence_list(test_data)
tokenizer = ElectraTokenizer.from_pretrained(
"monologg/koelectra-base-v2-discriminator")
return correct_kcBert(model,
tokenizer,
test_data,
args,
length_limit=0.1)
import nsml
nsml.bind(save, load, infer)
def bind_model(model):
def save(dir_name):
os.makedirs(dir_name, exist_ok=True)
model.save_weights(os.path.join(dir_name, 'model'))
# model.save_weights(file_path,'model')
print('model saved!')
def load(dir_name):
model.load_weights(os.path.join(dir_name, 'model'))
print('model loaded!')
def infer(
data,
target_resolution=TARGET_RESOLUTION,
): ## test mode
##### DO NOT CHANGE ORDER OF TEST DATA #####
X = []
for i, d in enumerate(data):
# test 데이터를 training 데이터와 같이 전처리 하기
X.append(image_preprocessing(d, target_resolution, normalize=True))
X = np.array(X)
pred = model.predict(X)
pred = np.argmax(pred, axis=1) # 모델 예측 결과: 0-3
print('Prediction done!\n Saving the result...')
return pred
nsml.bind(save=save, load=load, infer=infer)
def bind_nsml(model, optimizer, config, sp, feature):
def save(dir_name, *args, **kwargs):
os.makedirs(dir_name, exist_ok=True)
state = {
"model": model.state_dict(),
"optimizer": optimizer.state_dict(),
}
torch.save(state, os.path.join(dir_name, "model"))
copyfile('vocab.model', os.path.join(dir_name, 'vocab.model'))
with open(os.path.join(dir_name, 'feature.pickle'), 'wb') as fp:
pickle.dump(feature, fp)
print("saved")
def load(dir_name, *args, **kwargs):
state = torch.load(os.path.join(dir_name, "model"))
model.load_state_dict(state["model"])
optimizer.load_state_dict(state["optimizer"])
sp.load(os.path.join(dir_name, 'vocab.model'))
feature.load_idf(os.path.join(dir_name, 'feature.pickle'))
print("loaded")
def infer(dataset_path):
return _infer(model, config, dataset_path, sp, feature)
nsml.bind(save=save, load=load, infer=infer)
def nsml_bind(trainer):
import nsml
if isinstance(trainer, MTTrainer):
model = trainer.model()
else:
model = trainer
def save(dir_path):
state = {
'model': model.state_dict(),
# 'optimizer': optimizer.state_dict()
}
torch.save(state, os.path.join(dir_path, 'model.pt'))
def load(dir_path):
state = torch.load(os.path.join(dir_path, 'model.pt'))
model.load_state_dict(state['model'], strict=False)
# if 'optimizer' in state and optimizer:
# optimizer.load_state_dict(state['optimizer'])
# print('optimizer loaded!')
print('model loaded!')
nsml.bind(save=save, load=load)
if isinstance(trainer, MTTrainer):
trainer.set_save_function(nsml_save_func)
def bind_model(model, class_to_save, optimizer=None):
def load(filename, **kwargs):
state = torch.load(os.path.join(filename, 'model.pt'))
model.load_state_dict(state['model'])
if 'optimizer' in state and optimizer:
optimizer.load_state_dict(state['optimizer'])
with open(os.path.join(filename, 'class.pkl'), 'rb') as fp:
temp_class = pickle.load(fp)
nsml.copy(temp_class, class_to_save)
print('Model loaded')
def save(filename, **kwargs):
state = {
'model': model.state_dict(),
'optimizer': optimizer.state_dict()
}
torch.save(state, os.path.join(filename, 'model.pt'))
with open(os.path.join(filename, 'class.pkl'), 'wb') as fp:
pickle.dump(class_to_save, fp)
def infer(input, top_k=100):
if isinstance(input, list):
print('input[0] :', input[0], 'type :', type(input[0]))
return input[0]
print('input :', input, 'type : ', type(input))
return input
nsml.bind(save=save, load=load, infer=infer)
def bind_model(model):
def save(dirname, *args):
if torch.cuda.device_count() > 1:
checkpoint = {'model': model.module.state_dict()}
else:
checkpoint = {'model': model.state_dict()}
torch.save(checkpoint, os.path.join(dirname, 'model.pt'))
def load(dirname, *args):
checkpoint = torch.load(os.path.join(dirname, 'model.pt'))
model.load_state_dict(checkpoint['model'])
def infer(raw_data, **kwargs):
model.eval()
examples = HateSpeech(raw_data).examples
examples = [ex.syllable_contents for ex in examples]
loader = torch.utils.data.DataLoader(examples, batch_size=1)
results = []
for data in loader:
dec_inputs = torch.tensor(data).long().cuda()
dec_inputs = dec_inputs.reshape(-1, len(dec_inputs))
index1 = dec_inputs == 1
index0 = dec_inputs == 0
dec_inputs[index1] = 0
dec_inputs[index0] = 1
pred = model(dec_inputs)[1].max(1)[1].tolist()
results += pred
return results
nsml.bind(save=save, load=load, infer=infer)
def bind_nsml(model, tokenizer=None, args=None, eos=False):
def save(path, **kwargs):
torch.save(model.state_dict(), open(os.path.join(path, 'model.pt'), 'wb'))
if args.tokenizer == 'char' and tokenizer is not None:
tokenizer.save(os.path.join(path, 'vocab.txt'))
def load(path, **kwargs):
model.load_state_dict(torch.load(open(os.path.join(path, 'model.pt'), 'rb'),
map_location=lambda storage, loc: storage))
if args.tokenizer == 'char' and tokenizer is not None:
tokenizer.load(os.path.join(path, 'vocab.txt'))
def infer(test_data, **kwargs):
'''
:param test_data: list of noisy sentences
:return: list of corrected sentences
'''
# 특수문자 지우기 일단 꺼놓음.
# test_data = preprocess_noisy_sentence_list(test_data)
if args.tokenizer == 'kobert':
tokenizer = KoBertTokenizer.from_pretrained('monologg/kobert')
return correct(model, tokenizer, test_data, args, eos=eos, length_limit=0.1)
import nsml
nsml.bind(save, load, infer)
def bind_model(model):
def save(dir_name):
os.makedirs(dir_name, exist_ok=True)
model.save_weights(os.path.join(dir_name, 'model'))
# model.save_weights(file_path,'model')
print('model saved!')
def load(dir_name):
model.load_weights(os.path.join(dir_name, 'model'))
print('model loaded!')
def infer(data, rescale=RESCALE, resize_factor=RESIZE): ## test mode
##### DO NOT CHANGE ORDER OF TEST DATA #####
X = []
for i, d in enumerate(data):
# test 데이터를 training 데이터와 같이 전처리 하기
X.append(image_preprocessing(d, rescale, resize_factor))
X = np.array(X)
pred = model.predict(X) # 모델 예측 결과: 0-3
print('Prediction done!\n Saving the result...')
if pred.shape[-1] > 1:
pred = pred.argmax(axis=-1)
else:
pred = (pred > 0.5).astype('int32')
return pred
nsml.bind(save=save, load=load, infer=infer)
def bind_nsml(model, tokenizer=None, args=None):
def save(path, **kwargs):
torch.save(model.state_dict(),
open(os.path.join(path, 'model.pt'), 'wb'))
if tokenizer is not None:
tokenizer.save(os.path.join(path, 'vocab.txt'))
def load(path, **kwargs):
model.load_state_dict(
torch.load(open(os.path.join(path, 'model.pt'), 'rb'),
map_location=lambda storage, loc: storage))
if tokenizer is not None:
tokenizer.load(os.path.join(path, 'vocab.txt'))
def infer(test_data, **kwargs):
'''
:param test_data: list of noisy sentences
:return: list of corrected sentences
'''
# 특수문자 지우기 일단 꺼놓음.
# test_data = preprocess_noisy_sentence_list(test_data)
return correct_kcBert(model, tokenizer, test_data, args)
import nsml
nsml.bind(save, load, infer)
def bind_nsml(model, optimizer, task):
def save(dir_name, *args, **kwargs):
os.makedirs(dir_name, exist_ok=True)
model.save_weights(os.path.join(dir_name, 'model'))
print('model saved!')
"""
state = {
'model': model.state_dict(), # 이부분 수정
'optimizer': optimizer.state_dict() # 이부분도 수정
}
torch.save(state, os.path.join(dir_name, 'model.ckpt'))
# tf.keras.models.save_model( model, os.path.join(dir_name,'model.ckpt'))
# tf.keras.models.save_model( model, filepath, overwrite=True, include_optimizer=True, save_format=None)
print('saved model checkpoints...!')
"""
def load(dir_name, *args, **kwargs):
model.load_weights(os.path.join(dir_name, 'model'))
print('model loaded')
"""
state = torch.load(os.path.join(dir_name, 'model.ckpt'))
model.load_state_dict(state['model'])
optimizer.load_state_dict(state['optimizer'])
# tf.keras.models.load_model(os.path.join(dir_name, 'model.ckpt'))
# tf.keras.models.load_model( filepath, custom_objects=None, compile=True)
print('loaded model checkpoints...!')
"""
def infer(root, phase):
return _infer(root, phase, model=model, task=task)
nsml.bind(save=save, load=load, infer=infer)
def bind_model(sess, config):
def save(dir_name, *args):
os.makedirs(dir_name, exist_ok=True)
saver = tf.train.Saver()
saver.save(sess, os.path.join(dir_name, 'model'))
def load(dir_name, *args):
saver = tf.train.Saver()
ckpt = tf.train.get_checkpoint_state(dir_name)
if ckpt and ckpt.model_checkpoint_path:
checkpoint = os.path.basename(ckpt.model_checkpoint_path)
saver.restore(sess, os.path.join(dir_name, checkpoint))
else:
raise NotImplemented('No checkpoint!')
print('Model loaded')
def infer(raw_data, **kwargs):
"""
:param raw_data: raw input (여기서는 문자열)을 입력받습니다
:param kwargs:
:return:
"""
queries = preprocess(raw_data, config.strmaxlen, hidden_layer_size)
pred = sess.run(hypothesis, feed_dict={x: queries})
clipped = np.array(pred > config.threshold, dtype=np.int)
# DONOTCHANGE: They are reserved for nsml
return list(zip(pred.flatten(), clipped.flatten()))
# DONOTCHANGE: They are reserved for nsml
nsml.bind(save=save, load=load, infer=infer)
def bind_model(model):
def save(dir_name):
os.makedirs(dir_name, exist_ok=True)
torch.save(model.state_dict(), os.path.join(dir_name, 'model'))
print('model saved!')
def load(dir_name):
model.load_state_dict(torch.load(os.path.join(dir_name, 'model')))
model.eval()
print('model loaded!')
def infer(image_path):
result = []
with torch.no_grad():
batch_loader = DataLoader(dataset=PathDataset(
image_path, labels=None, input_size=input_size),
batch_size=batch_size,
shuffle=False)
# Train the model
for i, images_list in enumerate(batch_loader):
y_hat_list = model([
images_list[j].to(device) for j in range(len(images_list))
])
y_hat = [
torch.softmax(y_hat_list[j], 1).data.cpu().numpy()
for j in range(len(images_list))
]
y_hat = np.mean(y_hat, 0)
result.extend(np.argmax(y_hat, axis=1))
print('predicted')
return np.array(result)
nsml.bind(save=save, load=load, infer=infer)
def bind_model(model_nsml):
def save(dir_name, **kwargs):
save_state_path = os.path.join(dir_name, 'state_dict.pkl')
state = {
'model': model_nsml.state_dict(),
}
torch.save(state, save_state_path)
def load(dir_name):
save_state_path = os.path.join(dir_name, 'state_dict.pkl')
state = torch.load(save_state_path)
model_nsml.load_state_dict(state['model'])
def infer(test_image_data_path, test_meta_data_path):
# DONOTCHANGE This Line
test_meta_data = pd.read_csv(test_meta_data_path, delimiter=',', header=0)
input_size=224 # you can change this according to your model.
batch_size=200 # you can change this. But when you use 'nsml submit --test' for test infer, there are only 200 number of data.
device = 0
we = 0.25
ensemble = [['team_62/airush1/320', '02'],['team_62/airush1/320','12'],['team_62/airush1/320','22'],['team_62/airush1/320','32']]
#ensemble = [['team_62/airush1/415', '03'],['team_62/airush1/415','13'],['team_62/airush1/415','23'],['team_62/airush1/415','33']]
predict_list = []
for i in range(4):
dataloader = DataLoader(
AIRushDataset(test_image_data_path, test_meta_data, label_path=None,
transform=transforms.Compose([transforms.Resize((input_size, input_size)), transforms.RandomRotation(20),transforms.ToTensor(),transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])),
batch_size=batch_size,
shuffle=False,
num_workers=4,
pin_memory=True)
# Let's do ensemble!!!
nsml.load(checkpoint=str(ensemble[i][1]),session=str(ensemble[i][0]))
# model load
model_nsml.to(device)
model_nsml.eval()
predict_output_list = []
with torch.no_grad():
for batch_idx, image in enumerate(dataloader):
image = image.to(device)
output = model_nsml(image).double()
output_prob = to_np(F.softmax(output, dim=1))
predict_output_list.append(output_prob * we)
predict_output_list = np.concatenate(predict_output_list,axis=0)
predict_list.append(predict_output_list)
predict_vector = np.argmax(np.sum(predict_list,axis=0), axis=1)
return predict_vector # this return type should be a numpy array which has shape of (138343)
# DONOTCHANGE: They are reserved for nsml
nsml.bind(save=save, load=load, infer=infer)
def bind_model(model, config):
# 학습한 모델을 저장하는 함수입니다.
def save(filename, *args):
checkpoint = {'model': model.state_dict()}
torch.save(checkpoint, filename)
# 저장한 모델을 불러올 수 있는 함수입니다.
def load(filename, *args):
checkpoint = torch.load(filename)
model.load_state_dict(checkpoint['model'])
print('Model loaded')
def infer(raw_data, **kwargs):
data = preprocess(raw_data, config.vocasize, config.minlen,
config.maxlen)
model.eval()
prediction = model(data)
point = prediction.data.squeeze(dim=1).tolist()
# DONOTCHANGE: They are reserved for nsml
# 리턴 결과는 [(confidence interval, 포인트)] 의 형태로 보내야만 리더보드에 올릴 수 있습니다. 리더보드 결과에 confidence interval의 값은 영향을 미치지 않습니다
return list(zip(np.zeros(len(point)), point))
# DONOTCHANGE: They are reserved for nsml
# nsml에서 지정한 함수에 접근할 수 있도록 하는 함수입니다.
nsml.bind(save=save, load=load, infer=infer)
def bind_nsml(model, tokenizer, my_args):
def save(dir_name, *args, **kwargs):
os.makedirs(dir_name, exist_ok=True)
torch.save(model.state_dict(), os.path.join(dir_name, 'model.pt'))
torch.save(tokenizer, os.path.join(dir_name, 'tokenizer'))
torch.save(my_args, os.path.join(dir_name, "my_args.bin"))
logger.info("Save model & tokenizer & args at {}".format(dir_name))
def load(dir_name, *args, **kwargs):
state = torch.load(os.path.join(dir_name, 'model.pt'))
model.load_state_dict(state)
temp_my_args = torch.load(os.path.join(dir_name, "my_args.bin"))
nsml.copy(temp_my_args, my_args)
temp_tokenizer = torch.load(os.path.join(dir_name, 'tokenizer'))
nsml.copy(temp_tokenizer, tokenizer)
logger.info("Load model & tokenizer & args from {}".format(dir_name))
def infer(root_path):
"""NSML will record f1-score based on predictions from this method."""
result = _infer(model, tokenizer, my_args, root_path)
for line in result:
assert type(tuple(line)) == tuple and len(
line) == 2, "Wrong infer result: {}".format(line)
return result
nsml.bind(save=save, load=load, infer=infer)
def bind_model(model, optimizer=None):
def load(filename, **kwargs):
state = torch.load(os.path.join(filename, 'model.pt'))
model.load_state_dict(state['model'])
if 'optimizer' in state and optimizer:
optimizer.load_state_dict(state['optimizer'])
print('Model loaded')
def save(filename, **kwargs):
state = {
'model': model.state_dict(),
'optimizer': optimizer.state_dict()
}
torch.save(state, os.path.join(filename, 'model.pt'))
def infer(wav_path):
model.eval()
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
input = get_spectrogram_feature(wav_path).unsqueeze(0)
input = input.to(device)
logit = model(input_variable=input,
input_lengths=None,
teacher_forcing_ratio=0)
logit = torch.stack(logit, dim=1).to(device)
y_hat = logit.max(-1)[1]
hyp = label_to_string(y_hat)
return hyp[0]
nsml.bind(save=save, load=load,
infer=infer) # 'nsml.bind' function must be called at the end.
def bind_model(model, config):
# 학습한 모델을 저장하는 함수입니다.
def save(filename, *args):
model.save_weights(filename)
# 저장한 모델을 불러올 수 있는 함수입니다.
def load(filename, *args):
model.load_weights(filename)
print('Model loaded')
def infer(raw_data, **kwargs):
"""
:param raw_data: raw input (여기서는 문자열)을 입력받습니다
:param kwargs:
:return:
"""
# dataset.py에서 작성한 preprocess 함수를 호출하여, 문자열을 벡터로 변환합니다
preprocessed_data_pre = preprocess_pre(raw_data, config.strmaxlen)
preprocessed_data_post = preprocess_post(raw_data, config.strmaxlen)
# 저장한 모델에 입력값을 넣고 prediction 결과를 리턴받습니다
point = model.predict([preprocessed_data_pre,
preprocessed_data_post])[-1]
point[point > 10.] = 10.
point[point < 1.] = 1.
point = point.squeeze(axis=1).tolist()
# DONOTCHANGE: They are reserved for nsml
# 리턴 결과는 [(confidence interval, 포인트)] 의 형태로 보내야만 리더보드에 올릴 수 있습니다. 리더보드 결과에 confidence interval의 값은 영향을 미치지 않습니다
return list(zip(np.zeros(len(point)), point))
# DONOTCHANGE: They are reserved for nsml
# nsml에서 지정한 함수에 접근할 수 있도록 하는 함수입니다.
nsml.bind(save=save, load=load, infer=infer)
def bind_model(model, config):
# 학습한 모델을 저장하는 함수입니다.
def save(filename, *args):
checkpoint = {
'model': model.state_dict()
}
torch.save(checkpoint, filename)
# 저장한 모델을 불러올 수 있는 함수입니다.
def load(filename, *args):
checkpoint = torch.load(filename)
model.load_state_dict(checkpoint['model'])
print('Model loaded')
def infer(raw_data, **kwargs):
"""
:param raw_data: raw input (여기서는 문자열)을 입력받습니다
:param kwargs:
:return:
"""
# dataset.py에서 작성한 preprocess 함수를 호출하여, 문자열을 벡터로 변환합니다
preprocessed_data = preprocess(raw_data, config.strmaxlen)
model.eval()
# 저장한 모델에 입력값을 넣고 prediction 결과를 리턴받습니다
output_prediction = model(preprocessed_data[0])
point = output_prediction.data.squeeze(dim=1).tolist()
# DONOTCHANGE: They are reserved for nsml
# 리턴 결과는 [(confidence interval, 포인트)] 의 형태로 보내야만 리더보드에 올릴 수 있습니다. 리더보드 결과에 confidence interval의 값은 영향을 미치지 않습니다
return list(zip(np.zeros(len(point)), point))
# DONOTCHANGE: They are reserved for nsml
# nsml에서 지정한 함수에 접근할 수 있도록 하는 함수입니다.
nsml.bind(save=save, load=load, infer=infer)
def bind_nsml(model_sc, model_qa, tokenizer, my_args):
def save(dir_name, *args, **kwargs):
os.makedirs(dir_name, exist_ok=True)
torch.save(model_sc.state_dict(), os.path.join(dir_name, 'model_sc.pt'))
torch.save(model_qa.state_dict(), os.path.join(dir_name, 'model_qa.pt'))
logger.info("Save model at {}".format(dir_name))
def load(dir_name, *args, **kwargs):
state_sc = torch.load(os.path.join(dir_name, 'model_sc.pt'))
model_sc.load_state_dict(state_sc)
state_qa = torch.load(os.path.join(dir_name, 'model_qa.pt'))
model_qa.load_state_dict(state_qa)
logger.info("Load model from {}".format(dir_name))
def infer(root_path):
"""NSML will record f1-score based on predictions from this method."""
result = _infer(model_sc, model_qa, tokenizer, my_args, root_path)
for line in result:
assert type(tuple(line)) == tuple and len(line) == 2, "Wrong infer result: {}".format(line)
return result
nsml.bind(save=save, load=load, infer=infer)
def bind_models(models): # for ensemble
def save(dirname, *args):
checkpoint = {
f'model{i}': model.state_dict() for i, model in enumerate(models)
}
torch.save(checkpoint, os.path.join(dirname, 'model.pt'))
def load(dirname, *args):
checkpoint = torch.load(os.path.join(dirname, 'model.pt'))
for i, model in enumerate(models):
model.load_state_dict(checkpoint[f'model{i}'])
def infer(raw_data, **kwargs):
for model in models:
model.eval()
examples = HateSpeech(raw_data).examples
tensors = [torch.tensor(ex.syllable_contents, device='cuda').reshape([-1, 1]) for ex in examples]
results = []
for ex in tensors:
total = 0
for model in models: # vote
total += torch.where(model(ex) > 0.5, torch.tensor(1.0, device='cuda'), torch.tensor(0.0, device='cuda'))
total = total / len(models) # len must be an odd number
results.append(total.tolist())
return results
nsml.bind(save=save, load=load, infer=infer)
def bind_nsml(model, optimizer):
def save(dir_name, *args, **kwargs):
if not isinstance(dir_name, str):
dir_name = str(dir_name)
os.makedirs(dir_name, exist_ok=True)
state = {
'model': model.state_dict(),
'optimizer': optimizer.state_dict()
}
fname = os.path.join(dir_name, 'model.pth')
torch.save(state, fname)
print('saved')
def load(dir_name, *args, **kwargs):
if not isinstance(dir_name, str):
dir_name = str(dir_name)
fname = os.path.join(dir_name, 'model.pth')
state = torch.load(fname)
model.load_state_dict(state['model'])
optimizer.load_state_dict(state['optimizer'])
print('loaded')
def infer(root_path):
return _infer(model, root_path)
if use_nsml:
nsml.bind(save=save, load=load, infer=infer)
return save, load
def bind_model(model):
def save(dirname, *args):
if torch.cuda.device_count() > 1:
checkpoint = {'model': model.module.state_dict()}
else:
checkpoint = {'model': model.state_dict()}
torch.save(checkpoint, os.path.join(dirname, 'model.pt'))
def load(dirname, *args):
checkpoint = torch.load(os.path.join(dirname, 'model.pt'))
model.load_state_dict(checkpoint['model'])
def infer(raw_data, **kwargs):
model.eval()
examples = HateSpeech(raw_data).examples
tensors = [
torch.tensor(ex.syllable_contents, device='cuda')
for ex in examples
]
results = []
for tensor in tensors:
tensor = tensor.reshape(-1, len(tensor))
index1 = tensor == 1
index0 = tensor == 0
tensor[index1] = 0
tensor[index0] = 1
pred = model(
tensor,
torch.tensor([[3]] *
len(tensor)).cuda())[0].sigmoid()[0][0].tolist()
results.append(pred)
return results
nsml.bind(save=save, load=load, infer=infer)
def bind_model(sess, config):
# 학습한 모델을 저장하는 함수입니다.
def save(dir_name, *args):
# directory
os.makedirs(dir_name, exist_ok=True)
saver = tf.train.Saver()
saver.save(sess, os.path.join(dir_name, 'model'))
# 저장한 모델을 불러올 수 있는 함수입니다.
def load(dir_name, *args):
saver = tf.train.Saver()
# find checkpoint
ckpt = tf.train.get_checkpoint_state(dir_name)
if ckpt and ckpt.model_checkpoint_path:
checkpoint = os.path.basename(ckpt.model_checkpoint_path)
saver.restore(sess, os.path.join(dir_name, checkpoint))
else:
raise NotImplemented('No checkpoint!')
print('Model loaded')
def infer(raw_data, **kwargs):
left_preprocessed_data, right_preprocessed_data = preprocess(raw_data, config.strmaxlen)
# 저장한 모델에 입력값을 넣고 prediction 결과를 리턴받습니다
pred = sess.run(output_sigmoid, feed_dict={x_1: left_preprocessed_data, x_2: right_preprocessed_data})
clipped = np.array(pred > config.threshold, dtype=np.int)
return list(zip(pred.flatten(), clipped.flatten()))
# DONOTCHANGE: They are reserved for nsml
# nsml에서 지정한 함수에 접근할 수 있도록 하는 함수입니다.
nsml.bind(save=save, load=load, infer=infer)
def bind_model(model):
def save(dir_name):
os.makedirs(dir_name, exist_ok=True)
torch.save(model.state_dict(), os.path.join(dir_name, 'model'))
print('model saved!')
def load(dir_name):
model.load_state_dict(torch.load(os.path.join(dir_name, 'model')))
model.eval()
print('model loaded!')
def infer(data): ## test mode
X = ImagePreprocessing(data)
X = np.array(X)
X = np.expand_dims(X, axis=1)
##### DO NOT CHANGE ORDER OF TEST DATA #####
with torch.no_grad():
X = torch.from_numpy(X).float().to(device)
pred = model.forward(X)
prob, pred_cls = torch.max(pred, 1)
pred_cls = pred_cls.tolist()
#pred_cls = pred_cls.data.cpu().numpy()
print('Prediction done!\n Saving the result...')
return pred_cls
nsml.bind(save=save, load=load, infer=infer)
def bind_nsml(model, optimizer, scheduler):
def save(dir_name, *args, **kwargs):
os.makedirs(dir_name, exist_ok=True)
state = {
'model': model.state_dict(),
'optimizer': optimizer.state_dict(),
'scheduler': scheduler.state_dict()
}
torch.save(state, os.path.join(dir_name, 'model.pth'))
def load(dir_name, *args, **kwargs):
state = torch.load(os.path.join(dir_name, 'model.pth'))
model.load_state_dict({k.replace('module.', ''): v for k, v in state['model'].items()})
optimizer.load_state_dict(state['optimizer'])
scheduler.load_state_dict(state['scheduler'])
print('loaded')
def infer(root_path, top_k=1):
if C.get()['mode'] != 'test':
return _infer(model, root_path)
if C.get()['infer_mode'] == 'ret':
return _infer_ret(model, root_path)
else:
return _infer(model, root_path)
nsml.bind(save=save, load=load, infer=infer)
def bind_model(sess):
def save(dir_name):
os.makedirs(dir_name, exist_ok=True)
saver = tf.train.Saver()
saver.save(sess,
os.path.join(dir_name, 'model'),
global_step=model.global_step)
def load(dir_name):
saver = tf.train.Saver()
ckpt = tf.train.get_checkpoint_state(dir_name)
if ckpt and ckpt.model_checkpoint_path:
checkpoint = os.path.basename(ckpt.model_checkpoint_path)
saver.restore(sess, os.path.join(dir_name, checkpoint))
else:
raise NotImplemented('No checkpoint!')
print('model loaded!')
def infer(input, **kwargs):
pred = []
# 학습용 데이터셋 구성
dataset.parameter["train_lines"] = len(input)
dataset.make_input_data(input)
reverse_tag = {
v: k
for k, v in dataset.necessary_data["ner_tag"].items()
}
# 테스트 셋을 측정한다.
for morph, ne_dict, character, seq_len, char_len, _, step in dataset.get_data_batch_size(
len(input), False):
feed_dict = {
model.morph: morph,
model.ne_dict: ne_dict,
model.character: character,
model.sequence: seq_len,
model.character_len: char_len,
model.dropout_rate: 1.0
}
viters = sess.run(model.viterbi_sequence, feed_dict=feed_dict)
for index, viter in zip(range(0, len(viters)), viters):
pred.append(
get_ner_bi_tag_list_in_sentence(reverse_tag, viter,
seq_len[index]))
# 최종 output 포맷 예시
# [(0.0, ['NUM_B', '-', '-', '-']),
# (0.0, ['PER_B', 'PER_I', 'CVL_B', 'NUM_B', '-', '-', '-', '-', '-', '-']),
# ( ), ( )
# ]
padded_array = np.zeros(len(pred))
return list(zip(padded_array, pred))
# DO NOT CHANGE
nsml.bind(save=save, load=load, infer=infer)
def bind_model(model):
def save(dir_name):
os.makedirs(dir_name, exist_ok=True)
torch.save(model.state_dict(),os.path.join(dir_name, 'model'))
print('model saved!')
def load(dir_name):
model.load_state_dict(torch.load(os.path.join(dir_name, 'model')))
model.eval()
print('model loaded!')
def infer(image_path):
result1 = []
result2 = []
result3 = []
result4 = []
result5 = []
with torch.no_grad():
test_transform = tv.transforms.Compose([
tv.transforms.ToPILImage(mode = 'RGB'),
tv.transforms.Resize(512),
tv.transforms.RandomCrop(256),
tv.transforms.ToTensor(),
tv.transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
batch_dataset = PathDataset(image_path, labels=None, test_mode= True, transform = test_transform)
batch_loader = DataLoader(dataset=batch_dataset,
batch_size=batch_size,shuffle=False)
# Train the model
for i, images in enumerate(batch_loader):
y_hat = model(images.to(device)).cpu().numpy()
result1.extend(np.argmax(y_hat, axis=1))
for i, images in enumerate(batch_loader):
y_hat = model(images.to(device)).cpu().numpy()
result2.extend(np.argmax(y_hat, axis=1))
for i, images in enumerate(batch_loader):
y_hat = model(images.to(device)).cpu().numpy()
result3.extend(np.argmax(y_hat, axis=1))
for i, images in enumerate(batch_loader):
y_hat = model(images.to(device)).cpu().numpy()
result4.extend(np.argmax(y_hat, axis=1))
for i, images in enumerate(batch_loader):
y_hat = model(images.to(device)).cpu().numpy()
result5.extend(np.argmax(y_hat, axis=1))
total_result = np.array([result1,result2,result3,result4,result5])
before_result = list(np.count_nonzero(total_result, axis=0))
after_result = [1 if x>=3 else 0 for x in before_result]
result = after_result
print('predicted')
return np.array(result)
nsml.bind(save=save, load=load, infer=infer)
def bind_model(model):
def save(dir_name):
os.makedirs(dir_name, exist_ok=True)
model.save(os.path.join(dir_name, 'model'))
#model.save_weights(os.path.join(dir_name, 'model'))
print('model saved!')
def load(file_path):
model = load_model(file_path)
#model.load_weights(file_path)
print('model loaded!')
def infer(queries, _):
test_path = DATASET_PATH + '/test/test_data'
db = [
os.path.join(test_path, 'reference', path)
for path in os.listdir(os.path.join(test_path, 'reference'))
]
queries = [v.split('/')[-1].split('.')[0] for v in queries]
db = [v.split('/')[-1].split('.')[0] for v in db]
queries.sort()
db.sort()
print('infer start qeries length:', len(queries), 'db length:',
len(db))
queries, query_vecs, references, reference_vecs = get_feature(
model, queries, db)
print('feature shape:', query_vecs.shape, reference_vecs.shape)
## l2 normalization
query_vecs = l2_normalize(query_vecs)
reference_vecs = l2_normalize(reference_vecs)
print('l2 complete')
# Calculate cosine similarity
sim_matrix = np.dot(query_vecs, reference_vecs.T)
print('cal_dot_complete sim matrix shape : ', sim_matrix.shape)
indices = np.argsort(sim_matrix, axis=1)
indices = np.flip(indices, axis=1)
print('indices shape:', indices.shape)
retrieval_results = {}
print('queries length:', len(queries), 'references length',
len(references))
for (i, query) in enumerate(queries):
# print(indices[i].shape)
ranked_list = [references[k] for k in indices[i]]
# print(i, query, ranked_list)
ranked_list = ranked_list[:1000]
retrieval_results[query] = ranked_list
print('done')
return list(
zip(range(len(retrieval_results)), retrieval_results.items()))
# DONOTCHANGE: They are reserved for nsml
nsml.bind(save=save, load=load, infer=infer)
def bind_model(sess, config, model, vocabulary):
# 학습한 모델을 저장하는 함수입니다.
def save(dir_name, *args):
# directory
os.makedirs(dir_name, exist_ok=True)
saver = tf.train.Saver()
saver.save(sess, os.path.join(dir_name, 'model'))
#print('save_seq vocabulary = {}'.format(vocabulary))
#print('save_seq vocabulary.voc = {}'.format(vocabulary.voc))
with open(os.path.join(dir_name, 'voc.pkl'), 'wb') as f:
pickle.dump(vocabulary, f)
# 저장한 모델을 불러올 수 있는 함수입니다.
def load(dir_name, *args):
saver = tf.train.Saver()
# find checkpoint
ckpt = tf.train.get_checkpoint_state(dir_name)
if ckpt and ckpt.model_checkpoint_path:
checkpoint = os.path.basename(ckpt.model_checkpoint_path)
saver.restore(sess, os.path.join(dir_name, checkpoint))
else:
raise NotImplemented('No checkpoint!')
print('Model loaded')
with open(os.path.join(dir_name, 'voc.pkl'), 'rb') as f:
vocabularyf = pickle.load(f)
vocabulary.append(vocabularyf)
def infer(raw_data, **kwargs):
"""
:param raw_data: raw input (여기서는 문자열)을 입력받습니다
:param kwargs:
:return:
"""
preprocessed_data = data_helpers.load_data_infer(
raw_data, vocabulary[0])
feed_dict = {
model.input_x: preprocessed_data,
model.dropout_keep_prob: 1.0
}
pred = sess.run(model.predictions, feed_dict=feed_dict)
pred = pred + 1
con = np.zeros(len(pred))
result = list(zip(con, pred))
# DONOTCHANGE: They are reserved for nsml
# 리턴 결과는 [(확률, 0 or 1)] 의 형태로 보내야만 리더보드에 올릴 수 있습니다. 리더보드 결과에 확률의 값은 영향을 미치지 않습니다
# [(1, 7.4), (1, 2.3)]
return result
# DONOTCHANGE: They are reserved for nsml
# nsml에서 지정한 함수에 접근할 수 있도록 하는 함수입니다.
nsml.bind(save=save, load=load, infer=infer)
def __init__(self,
policy,
ob_space,
ac_space,
nenvs,
total_timesteps,
nprocs=32,
nscripts=16,
nsteps=20,
nstack=4,
ent_coef=0.1,
vf_coef=0.5,
vf_fisher_coef=1.0,
lr=0.25,
max_grad_norm=0.001,
kfac_clip=0.001,
lrschedule='linear',
alpha=0.99,
epsilon=1e-5):
config = tf.ConfigProto(
allow_soft_placement=True,
intra_op_parallelism_threads=nprocs,
inter_op_parallelism_threads=nprocs)
config.gpu_options.allow_growth = True
self.sess = sess = tf.Session(config=config)
nsml.bind(sess=sess)
#nact = ac_space.n
nbatch = nenvs * nsteps
A = tf.placeholder(tf.int32, [nbatch])
XY0 = tf.placeholder(tf.int32, [nbatch])
XY1 = tf.placeholder(tf.int32, [nbatch])
# ADV == TD_TARGET - values
ADV = tf.placeholder(tf.float32, [nbatch])
TD_TARGET = tf.placeholder(tf.float32, [nbatch])
PG_LR = tf.placeholder(tf.float32, [])
VF_LR = tf.placeholder(tf.float32, [])
self.model = step_model = policy(
sess, ob_space, ac_space, nenvs, 1, nstack, reuse=False)
self.model2 = train_model = policy(
sess, ob_space, ac_space, nenvs, nsteps, nstack, reuse=True)
# Policy 1 : Base Action : train_model.pi label = A
script_mask = tf.concat(
[
tf.zeros([nscripts * nsteps, 1]),
tf.ones([(nprocs - nscripts) * nsteps, 1])
],
axis=0)
pi = train_model.pi
pac_weight = script_mask * (tf.nn.softmax(pi) - 1.0) + 1.0
pac_weight = tf.reduce_sum(pac_weight * tf.one_hot(A, depth=3), axis=1)
neglogpac = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=pi, labels=A)
neglogpac *= tf.stop_gradient(pac_weight)
inv_A = 1.0 - tf.cast(A, tf.float32)
xy0_mask = tf.cast(A, tf.float32)
xy1_mask = tf.cast(A, tf.float32)
condition0 = tf.equal(xy0_mask, 2)
xy0_mask = tf.where(condition0, tf.ones(tf.shape(xy0_mask)), xy0_mask)
xy0_mask = 1.0 - xy0_mask
condition1 = tf.equal(xy1_mask, 2)
xy1_mask = tf.where(condition1, tf.zeros(tf.shape(xy1_mask)), xy1_mask)
# One hot representation of chosen marine.
# [batch_size, 2]
pi_xy0 = train_model.pi_xy0
pac_weight = script_mask * (tf.nn.softmax(pi_xy0) - 1.0) + 1.0
pac_weight = tf.reduce_sum(
pac_weight * tf.one_hot(XY0, depth=1024), axis=1)
logpac_xy0 = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=pi_xy0, labels=XY0)
logpac_xy0 *= tf.stop_gradient(pac_weight)
logpac_xy0 *= tf.cast(xy0_mask, tf.float32)
pi_xy1 = train_model.pi_xy1
pac_weight = script_mask * (tf.nn.softmax(pi_xy1) - 1.0) + 1.0
pac_weight = tf.reduce_sum(
pac_weight * tf.one_hot(XY0, depth=1024), axis=1)
# 1D? 2D?
logpac_xy1 = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=pi_xy1, labels=XY1)
logpac_xy1 *= tf.stop_gradient(pac_weight)
logpac_xy1 *= tf.cast(xy1_mask, tf.float32)
pg_loss = tf.reduce_mean(ADV * neglogpac)
pg_loss_xy0 = tf.reduce_mean(ADV * logpac_xy0)
pg_loss_xy1 = tf.reduce_mean(ADV * logpac_xy1)
vf_ = tf.squeeze(train_model.vf)
vf_r = tf.concat(
[
tf.ones([nscripts * nsteps, 1]),
tf.zeros([(nprocs - nscripts) * nsteps, 1])
],
axis=0) * TD_TARGET
vf_masked = vf_ * script_mask + vf_r
#vf_mask[0:nscripts * nsteps] = R[0:nscripts * nsteps]
vf_loss = tf.reduce_mean(mse(vf_masked, TD_TARGET))
entropy_a = tf.reduce_mean(cat_entropy(train_model.pi))
entropy_xy0 = tf.reduce_mean(cat_entropy(train_model.pi_xy0))
entropy_xy1 = tf.reduce_mean(cat_entropy(train_model.pi_xy1))
entropy = entropy_a + entropy_xy0 + entropy_xy1
loss = pg_loss - entropy * ent_coef + vf_loss * vf_coef
params = find_trainable_variables("model")
grads = tf.gradients(loss, params)
if max_grad_norm is not None:
grads, _ = tf.clip_by_global_norm(grads, max_grad_norm)
grads = list(zip(grads, params))
trainer = tf.train.RMSPropOptimizer(
learning_rate=lr, decay=alpha, epsilon=epsilon)
_train = trainer.apply_gradients(grads)
self.logits = logits = train_model.pi
# xy0
self.params_common = params_common = tf.get_collection(
tf.GraphKeys.TRAINABLE_VARIABLES, scope='model/common')
self.params_xy0 = params_xy0 = tf.get_collection(
tf.GraphKeys.TRAINABLE_VARIABLES,
scope='model/xy0') + params_common
train_loss_xy0 = pg_loss_xy0 - entropy * ent_coef + vf_coef * vf_loss
self.grads_check_xy0 = grads_xy0 = tf.gradients(
train_loss_xy0, params_xy0)
if max_grad_norm is not None:
grads_xy0, _ = tf.clip_by_global_norm(grads_xy0, max_grad_norm)
grads_xy0 = list(zip(grads_xy0, params_xy0))
trainer_xy0 = tf.train.RMSPropOptimizer(
learning_rate=lr, decay=alpha, epsilon=epsilon)
_train_xy0 = trainer_xy0.apply_gradients(grads_xy0)
# xy1
self.params_xy1 = params_xy1 = tf.get_collection(
tf.GraphKeys.TRAINABLE_VARIABLES,
scope='model/xy1') + params_common
train_loss_xy1 = pg_loss_xy1 - entropy * ent_coef + vf_coef * vf_loss
self.grads_check_xy1 = grads_xy1 = tf.gradients(
train_loss_xy1, params_xy1)
if max_grad_norm is not None:
grads_xy1, _ = tf.clip_by_global_norm(grads_xy1, max_grad_norm)
grads_xy1 = list(zip(grads_xy1, params_xy1))
trainer_xy1 = tf.train.RMSPropOptimizer(
learning_rate=lr, decay=alpha, epsilon=epsilon)
_train_xy1 = trainer_xy1.apply_gradients(grads_xy1)
self.lr = Scheduler(v=lr, nvalues=total_timesteps, schedule=lrschedule)
def train(obs, states, td_targets, masks, actions, xy0, xy1, values):
advs = td_targets - values
for step in range(len(obs)):
cur_lr = self.lr.value()
td_map = {
train_model.X: obs,
A: actions,
XY0: xy0,
XY1: xy1,
ADV: advs,
TD_TARGET: td_targets,
PG_LR: cur_lr
}
if states != []:
td_map[train_model.S] = states
td_map[train_model.M] = masks
policy_loss, value_loss, policy_entropy, _, \
policy_loss_xy0, policy_entropy_xy0, _, \
policy_loss_xy1, policy_entropy_xy1, _ = sess.run(
[pg_loss, vf_loss, entropy, _train,
pg_loss_xy0, entropy_xy0, _train_xy0,
pg_loss_xy1, entropy_xy1, _train_xy1],
td_map)
return policy_loss, value_loss, policy_entropy, \
policy_loss_xy0, policy_entropy_xy0, \
policy_loss_xy1, policy_entropy_xy1
def save(save_path):
ps = sess.run(params)
joblib.dump(ps, save_path)
def load(load_path):
loaded_params = joblib.load(load_path)
restores = []
for p, loaded_p in zip(params, loaded_params):
restores.append(p.assign(loaded_p))
sess.run(restores)
self.train = train
self.save = save
self.load = load
self.train_model = train_model
self.step_model = step_model
self.step = step_model.step
self.value = step_model.value
self.initial_state = step_model.initial_state
print("global_variables_initializer start")
tf.global_variables_initializer().run(session=sess)
print("global_variables_initializer complete")
