def bind_model(sess, FLAGS): def save(dir_path, *args): os.makedirs(dir_path, exist_ok=True) saver = tf.train.Saver() saver.save(sess, os.path.join(dir_path, 'model')) with open(os.path.join(dir_path,FLAGS.map_file), "wb") as f: pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f) def load(dir_path, *args): global char_to_id global id_to_tag config = load_config(FLAGS.config_file) logger = get_logger(FLAGS.log_file) tf.get_variable_scope().reuse_variables() with open(os.path.join(dir_path,FLAGS.map_file), "rb") as f: char_to_id, _, __, id_to_tag = pickle.load(f) saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state(dir_path) if ckpt and ckpt.model_checkpoint_path: checkpoint = os.path.basename(ckpt.model_checkpoint_path) saver.restore(sess, os.path.join(dir_path, checkpoint)) else: raise NotImplemented('No checkpoint found!') print ('model loaded!') def infer(sentences, **kwargs): config = load_config(FLAGS.config_file) logger = get_logger(FLAGS.log_file) reformed_sentences = [' '.join(sen[1]) for sen in sentences] result = model.evaluate_lines(sess, inputs_from_sentences(reformed_sentences, char_to_id, FLAGS.max_char_length), id_to_tag) ''' result = [ (0.0, ['ARG0', 'ARG3', '-']), (0.0, ['ARG0', 'ARG1', '-']) ] # evaluate_lines 함수는 문장 단위 분석 결과를 내어줍니다. # len(result) : 문장의 갯수, 따라서 위 예제는 두 문장의 결과입니다. # result[0] : 첫번째 문장의 분석 결과, result[1] : 두번째 문장의 분석 결과. # 각 문장의 분석 결과는 다시 (prob, [labels])로 구성됩니다. # prob에 해당하는 자료는 이번 task에서 사용하지 않습니다. 따라서 그 값이 결과에 영향을 미치지 않습니다. # [labels]는 각 어절의 분석 결과를 담고 있습니다. 따라서 다음과 같이 구성됩니다. ## ['첫번째 어절의 분석 결과', '두번째 어절의 분석 결과', ...] # 예를 들면 위 주어진 예제에서 첫번째 문장의 첫번째 어절은 'ARG0'을, 첫번째 문장의 두번째 어절은 'ARG3'을 argument label로 가집니다. ### 주의사항 ### # 모든 어절의 결과를 제출하여야 합니다. # 어절의 순서가 지켜져야 합니다. (첫번째 어절부터 순서대로 list 내에 위치하여야 합니다.) ''' return result if IS_ON_NSML: nsml.bind(save=save, load=load, infer=infer)
def bind_nsml(model, tokenizer=None, args=None): def save(path, **kwargs): torch.save(model.state_dict(), open(os.path.join(path, 'model.pt'), 'wb')) def load(path, **kwargs): model.load_state_dict( torch.load(open(os.path.join(path, 'model.pt'), 'rb'), map_location=lambda storage, loc: storage)) def infer(test_data, **kwargs): ''' :param test_data: list of noisy sentences :return: list of corrected sentences ''' # 특수문자 지우기 일단 꺼놓음. # test_data = preprocess_noisy_sentence_list(test_data) tokenizer = ElectraTokenizer.from_pretrained( "monologg/koelectra-base-v2-discriminator") return correct_kcBert(model, tokenizer, test_data, args, length_limit=0.1) import nsml nsml.bind(save, load, infer)
def bind_model(model): def save(dir_name): os.makedirs(dir_name, exist_ok=True) model.save_weights(os.path.join(dir_name, 'model')) # model.save_weights(file_path,'model') print('model saved!') def load(dir_name): model.load_weights(os.path.join(dir_name, 'model')) print('model loaded!') def infer( data, target_resolution=TARGET_RESOLUTION, ): ## test mode ##### DO NOT CHANGE ORDER OF TEST DATA ##### X = [] for i, d in enumerate(data): # test 데이터를 training 데이터와 같이 전처리 하기 X.append(image_preprocessing(d, target_resolution, normalize=True)) X = np.array(X) pred = model.predict(X) pred = np.argmax(pred, axis=1) # 모델 예측 결과: 0-3 print('Prediction done!\n Saving the result...') return pred nsml.bind(save=save, load=load, infer=infer)
def bind_nsml(model, optimizer, config, sp, feature): def save(dir_name, *args, **kwargs): os.makedirs(dir_name, exist_ok=True) state = { "model": model.state_dict(), "optimizer": optimizer.state_dict(), } torch.save(state, os.path.join(dir_name, "model")) copyfile('vocab.model', os.path.join(dir_name, 'vocab.model')) with open(os.path.join(dir_name, 'feature.pickle'), 'wb') as fp: pickle.dump(feature, fp) print("saved") def load(dir_name, *args, **kwargs): state = torch.load(os.path.join(dir_name, "model")) model.load_state_dict(state["model"]) optimizer.load_state_dict(state["optimizer"]) sp.load(os.path.join(dir_name, 'vocab.model')) feature.load_idf(os.path.join(dir_name, 'feature.pickle')) print("loaded") def infer(dataset_path): return _infer(model, config, dataset_path, sp, feature) nsml.bind(save=save, load=load, infer=infer)
def nsml_bind(trainer): import nsml if isinstance(trainer, MTTrainer): model = trainer.model() else: model = trainer def save(dir_path): state = { 'model': model.state_dict(), # 'optimizer': optimizer.state_dict() } torch.save(state, os.path.join(dir_path, 'model.pt')) def load(dir_path): state = torch.load(os.path.join(dir_path, 'model.pt')) model.load_state_dict(state['model'], strict=False) # if 'optimizer' in state and optimizer: # optimizer.load_state_dict(state['optimizer']) # print('optimizer loaded!') print('model loaded!') nsml.bind(save=save, load=load) if isinstance(trainer, MTTrainer): trainer.set_save_function(nsml_save_func)
def bind_model(model, class_to_save, optimizer=None): def load(filename, **kwargs): state = torch.load(os.path.join(filename, 'model.pt')) model.load_state_dict(state['model']) if 'optimizer' in state and optimizer: optimizer.load_state_dict(state['optimizer']) with open(os.path.join(filename, 'class.pkl'), 'rb') as fp: temp_class = pickle.load(fp) nsml.copy(temp_class, class_to_save) print('Model loaded') def save(filename, **kwargs): state = { 'model': model.state_dict(), 'optimizer': optimizer.state_dict() } torch.save(state, os.path.join(filename, 'model.pt')) with open(os.path.join(filename, 'class.pkl'), 'wb') as fp: pickle.dump(class_to_save, fp) def infer(input, top_k=100): if isinstance(input, list): print('input[0] :', input[0], 'type :', type(input[0])) return input[0] print('input :', input, 'type : ', type(input)) return input nsml.bind(save=save, load=load, infer=infer)
def bind_model(model): def save(dirname, *args): if torch.cuda.device_count() > 1: checkpoint = {'model': model.module.state_dict()} else: checkpoint = {'model': model.state_dict()} torch.save(checkpoint, os.path.join(dirname, 'model.pt')) def load(dirname, *args): checkpoint = torch.load(os.path.join(dirname, 'model.pt')) model.load_state_dict(checkpoint['model']) def infer(raw_data, **kwargs): model.eval() examples = HateSpeech(raw_data).examples examples = [ex.syllable_contents for ex in examples] loader = torch.utils.data.DataLoader(examples, batch_size=1) results = [] for data in loader: dec_inputs = torch.tensor(data).long().cuda() dec_inputs = dec_inputs.reshape(-1, len(dec_inputs)) index1 = dec_inputs == 1 index0 = dec_inputs == 0 dec_inputs[index1] = 0 dec_inputs[index0] = 1 pred = model(dec_inputs)[1].max(1)[1].tolist() results += pred return results nsml.bind(save=save, load=load, infer=infer)
def bind_nsml(model, tokenizer=None, args=None, eos=False): def save(path, **kwargs): torch.save(model.state_dict(), open(os.path.join(path, 'model.pt'), 'wb')) if args.tokenizer == 'char' and tokenizer is not None: tokenizer.save(os.path.join(path, 'vocab.txt')) def load(path, **kwargs): model.load_state_dict(torch.load(open(os.path.join(path, 'model.pt'), 'rb'), map_location=lambda storage, loc: storage)) if args.tokenizer == 'char' and tokenizer is not None: tokenizer.load(os.path.join(path, 'vocab.txt')) def infer(test_data, **kwargs): ''' :param test_data: list of noisy sentences :return: list of corrected sentences ''' # 특수문자 지우기 일단 꺼놓음. # test_data = preprocess_noisy_sentence_list(test_data) if args.tokenizer == 'kobert': tokenizer = KoBertTokenizer.from_pretrained('monologg/kobert') return correct(model, tokenizer, test_data, args, eos=eos, length_limit=0.1) import nsml nsml.bind(save, load, infer)
def bind_model(model): def save(dir_name): os.makedirs(dir_name, exist_ok=True) model.save_weights(os.path.join(dir_name, 'model')) # model.save_weights(file_path,'model') print('model saved!') def load(dir_name): model.load_weights(os.path.join(dir_name, 'model')) print('model loaded!') def infer(data, rescale=RESCALE, resize_factor=RESIZE): ## test mode ##### DO NOT CHANGE ORDER OF TEST DATA ##### X = [] for i, d in enumerate(data): # test 데이터를 training 데이터와 같이 전처리 하기 X.append(image_preprocessing(d, rescale, resize_factor)) X = np.array(X) pred = model.predict(X) # 모델 예측 결과: 0-3 print('Prediction done!\n Saving the result...') if pred.shape[-1] > 1: pred = pred.argmax(axis=-1) else: pred = (pred > 0.5).astype('int32') return pred nsml.bind(save=save, load=load, infer=infer)
def bind_nsml(model, tokenizer=None, args=None): def save(path, **kwargs): torch.save(model.state_dict(), open(os.path.join(path, 'model.pt'), 'wb')) if tokenizer is not None: tokenizer.save(os.path.join(path, 'vocab.txt')) def load(path, **kwargs): model.load_state_dict( torch.load(open(os.path.join(path, 'model.pt'), 'rb'), map_location=lambda storage, loc: storage)) if tokenizer is not None: tokenizer.load(os.path.join(path, 'vocab.txt')) def infer(test_data, **kwargs): ''' :param test_data: list of noisy sentences :return: list of corrected sentences ''' # 특수문자 지우기 일단 꺼놓음. # test_data = preprocess_noisy_sentence_list(test_data) return correct_kcBert(model, tokenizer, test_data, args) import nsml nsml.bind(save, load, infer)
def bind_nsml(model, optimizer, task): def save(dir_name, *args, **kwargs): os.makedirs(dir_name, exist_ok=True) model.save_weights(os.path.join(dir_name, 'model')) print('model saved!') """ state = { 'model': model.state_dict(), # 이부분 수정 'optimizer': optimizer.state_dict() # 이부분도 수정 } torch.save(state, os.path.join(dir_name, 'model.ckpt')) # tf.keras.models.save_model( model, os.path.join(dir_name,'model.ckpt')) # tf.keras.models.save_model( model, filepath, overwrite=True, include_optimizer=True, save_format=None) print('saved model checkpoints...!') """ def load(dir_name, *args, **kwargs): model.load_weights(os.path.join(dir_name, 'model')) print('model loaded') """ state = torch.load(os.path.join(dir_name, 'model.ckpt')) model.load_state_dict(state['model']) optimizer.load_state_dict(state['optimizer']) # tf.keras.models.load_model(os.path.join(dir_name, 'model.ckpt')) # tf.keras.models.load_model( filepath, custom_objects=None, compile=True) print('loaded model checkpoints...!') """ def infer(root, phase): return _infer(root, phase, model=model, task=task) nsml.bind(save=save, load=load, infer=infer)
def bind_model(sess, config): def save(dir_name, *args): os.makedirs(dir_name, exist_ok=True) saver = tf.train.Saver() saver.save(sess, os.path.join(dir_name, 'model')) def load(dir_name, *args): saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state(dir_name) if ckpt and ckpt.model_checkpoint_path: checkpoint = os.path.basename(ckpt.model_checkpoint_path) saver.restore(sess, os.path.join(dir_name, checkpoint)) else: raise NotImplemented('No checkpoint!') print('Model loaded') def infer(raw_data, **kwargs): """ :param raw_data: raw input (여기서는 문자열)을 입력받습니다 :param kwargs: :return: """ queries = preprocess(raw_data, config.strmaxlen, hidden_layer_size) pred = sess.run(hypothesis, feed_dict={x: queries}) clipped = np.array(pred > config.threshold, dtype=np.int) # DONOTCHANGE: They are reserved for nsml return list(zip(pred.flatten(), clipped.flatten())) # DONOTCHANGE: They are reserved for nsml nsml.bind(save=save, load=load, infer=infer)
def bind_model(model): def save(dir_name): os.makedirs(dir_name, exist_ok=True) torch.save(model.state_dict(), os.path.join(dir_name, 'model')) print('model saved!') def load(dir_name): model.load_state_dict(torch.load(os.path.join(dir_name, 'model'))) model.eval() print('model loaded!') def infer(image_path): result = [] with torch.no_grad(): batch_loader = DataLoader(dataset=PathDataset( image_path, labels=None, input_size=input_size), batch_size=batch_size, shuffle=False) # Train the model for i, images_list in enumerate(batch_loader): y_hat_list = model([ images_list[j].to(device) for j in range(len(images_list)) ]) y_hat = [ torch.softmax(y_hat_list[j], 1).data.cpu().numpy() for j in range(len(images_list)) ] y_hat = np.mean(y_hat, 0) result.extend(np.argmax(y_hat, axis=1)) print('predicted') return np.array(result) nsml.bind(save=save, load=load, infer=infer)
def bind_model(model_nsml): def save(dir_name, **kwargs): save_state_path = os.path.join(dir_name, 'state_dict.pkl') state = { 'model': model_nsml.state_dict(), } torch.save(state, save_state_path) def load(dir_name): save_state_path = os.path.join(dir_name, 'state_dict.pkl') state = torch.load(save_state_path) model_nsml.load_state_dict(state['model']) def infer(test_image_data_path, test_meta_data_path): # DONOTCHANGE This Line test_meta_data = pd.read_csv(test_meta_data_path, delimiter=',', header=0) input_size=224 # you can change this according to your model. batch_size=200 # you can change this. But when you use 'nsml submit --test' for test infer, there are only 200 number of data. device = 0 we = 0.25 ensemble = [['team_62/airush1/320', '02'],['team_62/airush1/320','12'],['team_62/airush1/320','22'],['team_62/airush1/320','32']] #ensemble = [['team_62/airush1/415', '03'],['team_62/airush1/415','13'],['team_62/airush1/415','23'],['team_62/airush1/415','33']] predict_list = [] for i in range(4): dataloader = DataLoader( AIRushDataset(test_image_data_path, test_meta_data, label_path=None, transform=transforms.Compose([transforms.Resize((input_size, input_size)), transforms.RandomRotation(20),transforms.ToTensor(),transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])), batch_size=batch_size, shuffle=False, num_workers=4, pin_memory=True) # Let's do ensemble!!! nsml.load(checkpoint=str(ensemble[i][1]),session=str(ensemble[i][0])) # model load model_nsml.to(device) model_nsml.eval() predict_output_list = [] with torch.no_grad(): for batch_idx, image in enumerate(dataloader): image = image.to(device) output = model_nsml(image).double() output_prob = to_np(F.softmax(output, dim=1)) predict_output_list.append(output_prob * we) predict_output_list = np.concatenate(predict_output_list,axis=0) predict_list.append(predict_output_list) predict_vector = np.argmax(np.sum(predict_list,axis=0), axis=1) return predict_vector # this return type should be a numpy array which has shape of (138343) # DONOTCHANGE: They are reserved for nsml nsml.bind(save=save, load=load, infer=infer)
def bind_model(model, config): # 학습한 모델을 저장하는 함수입니다. def save(filename, *args): checkpoint = {'model': model.state_dict()} torch.save(checkpoint, filename) # 저장한 모델을 불러올 수 있는 함수입니다. def load(filename, *args): checkpoint = torch.load(filename) model.load_state_dict(checkpoint['model']) print('Model loaded') def infer(raw_data, **kwargs): data = preprocess(raw_data, config.vocasize, config.minlen, config.maxlen) model.eval() prediction = model(data) point = prediction.data.squeeze(dim=1).tolist() # DONOTCHANGE: They are reserved for nsml # 리턴 결과는 [(confidence interval, 포인트)] 의 형태로 보내야만 리더보드에 올릴 수 있습니다. 리더보드 결과에 confidence interval의 값은 영향을 미치지 않습니다 return list(zip(np.zeros(len(point)), point)) # DONOTCHANGE: They are reserved for nsml # nsml에서 지정한 함수에 접근할 수 있도록 하는 함수입니다. nsml.bind(save=save, load=load, infer=infer)
def bind_nsml(model, tokenizer, my_args): def save(dir_name, *args, **kwargs): os.makedirs(dir_name, exist_ok=True) torch.save(model.state_dict(), os.path.join(dir_name, 'model.pt')) torch.save(tokenizer, os.path.join(dir_name, 'tokenizer')) torch.save(my_args, os.path.join(dir_name, "my_args.bin")) logger.info("Save model & tokenizer & args at {}".format(dir_name)) def load(dir_name, *args, **kwargs): state = torch.load(os.path.join(dir_name, 'model.pt')) model.load_state_dict(state) temp_my_args = torch.load(os.path.join(dir_name, "my_args.bin")) nsml.copy(temp_my_args, my_args) temp_tokenizer = torch.load(os.path.join(dir_name, 'tokenizer')) nsml.copy(temp_tokenizer, tokenizer) logger.info("Load model & tokenizer & args from {}".format(dir_name)) def infer(root_path): """NSML will record f1-score based on predictions from this method.""" result = _infer(model, tokenizer, my_args, root_path) for line in result: assert type(tuple(line)) == tuple and len( line) == 2, "Wrong infer result: {}".format(line) return result nsml.bind(save=save, load=load, infer=infer)
def bind_model(model, optimizer=None): def load(filename, **kwargs): state = torch.load(os.path.join(filename, 'model.pt')) model.load_state_dict(state['model']) if 'optimizer' in state and optimizer: optimizer.load_state_dict(state['optimizer']) print('Model loaded') def save(filename, **kwargs): state = { 'model': model.state_dict(), 'optimizer': optimizer.state_dict() } torch.save(state, os.path.join(filename, 'model.pt')) def infer(wav_path): model.eval() device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') input = get_spectrogram_feature(wav_path).unsqueeze(0) input = input.to(device) logit = model(input_variable=input, input_lengths=None, teacher_forcing_ratio=0) logit = torch.stack(logit, dim=1).to(device) y_hat = logit.max(-1)[1] hyp = label_to_string(y_hat) return hyp[0] nsml.bind(save=save, load=load, infer=infer) # 'nsml.bind' function must be called at the end.
def bind_model(model, config): # 학습한 모델을 저장하는 함수입니다. def save(filename, *args): model.save_weights(filename) # 저장한 모델을 불러올 수 있는 함수입니다. def load(filename, *args): model.load_weights(filename) print('Model loaded') def infer(raw_data, **kwargs): """ :param raw_data: raw input (여기서는 문자열)을 입력받습니다 :param kwargs: :return: """ # dataset.py에서 작성한 preprocess 함수를 호출하여, 문자열을 벡터로 변환합니다 preprocessed_data_pre = preprocess_pre(raw_data, config.strmaxlen) preprocessed_data_post = preprocess_post(raw_data, config.strmaxlen) # 저장한 모델에 입력값을 넣고 prediction 결과를 리턴받습니다 point = model.predict([preprocessed_data_pre, preprocessed_data_post])[-1] point[point > 10.] = 10. point[point < 1.] = 1. point = point.squeeze(axis=1).tolist() # DONOTCHANGE: They are reserved for nsml # 리턴 결과는 [(confidence interval, 포인트)] 의 형태로 보내야만 리더보드에 올릴 수 있습니다. 리더보드 결과에 confidence interval의 값은 영향을 미치지 않습니다 return list(zip(np.zeros(len(point)), point)) # DONOTCHANGE: They are reserved for nsml # nsml에서 지정한 함수에 접근할 수 있도록 하는 함수입니다. nsml.bind(save=save, load=load, infer=infer)
def bind_model(model, config): # 학습한 모델을 저장하는 함수입니다. def save(filename, *args): checkpoint = { 'model': model.state_dict() } torch.save(checkpoint, filename) # 저장한 모델을 불러올 수 있는 함수입니다. def load(filename, *args): checkpoint = torch.load(filename) model.load_state_dict(checkpoint['model']) print('Model loaded') def infer(raw_data, **kwargs): """ :param raw_data: raw input (여기서는 문자열)을 입력받습니다 :param kwargs: :return: """ # dataset.py에서 작성한 preprocess 함수를 호출하여, 문자열을 벡터로 변환합니다 preprocessed_data = preprocess(raw_data, config.strmaxlen) model.eval() # 저장한 모델에 입력값을 넣고 prediction 결과를 리턴받습니다 output_prediction = model(preprocessed_data[0]) point = output_prediction.data.squeeze(dim=1).tolist() # DONOTCHANGE: They are reserved for nsml # 리턴 결과는 [(confidence interval, 포인트)] 의 형태로 보내야만 리더보드에 올릴 수 있습니다. 리더보드 결과에 confidence interval의 값은 영향을 미치지 않습니다 return list(zip(np.zeros(len(point)), point)) # DONOTCHANGE: They are reserved for nsml # nsml에서 지정한 함수에 접근할 수 있도록 하는 함수입니다. nsml.bind(save=save, load=load, infer=infer)
def bind_nsml(model_sc, model_qa, tokenizer, my_args): def save(dir_name, *args, **kwargs): os.makedirs(dir_name, exist_ok=True) torch.save(model_sc.state_dict(), os.path.join(dir_name, 'model_sc.pt')) torch.save(model_qa.state_dict(), os.path.join(dir_name, 'model_qa.pt')) logger.info("Save model at {}".format(dir_name)) def load(dir_name, *args, **kwargs): state_sc = torch.load(os.path.join(dir_name, 'model_sc.pt')) model_sc.load_state_dict(state_sc) state_qa = torch.load(os.path.join(dir_name, 'model_qa.pt')) model_qa.load_state_dict(state_qa) logger.info("Load model from {}".format(dir_name)) def infer(root_path): """NSML will record f1-score based on predictions from this method.""" result = _infer(model_sc, model_qa, tokenizer, my_args, root_path) for line in result: assert type(tuple(line)) == tuple and len(line) == 2, "Wrong infer result: {}".format(line) return result nsml.bind(save=save, load=load, infer=infer)
def bind_models(models): # for ensemble def save(dirname, *args): checkpoint = { f'model{i}': model.state_dict() for i, model in enumerate(models) } torch.save(checkpoint, os.path.join(dirname, 'model.pt')) def load(dirname, *args): checkpoint = torch.load(os.path.join(dirname, 'model.pt')) for i, model in enumerate(models): model.load_state_dict(checkpoint[f'model{i}']) def infer(raw_data, **kwargs): for model in models: model.eval() examples = HateSpeech(raw_data).examples tensors = [torch.tensor(ex.syllable_contents, device='cuda').reshape([-1, 1]) for ex in examples] results = [] for ex in tensors: total = 0 for model in models: # vote total += torch.where(model(ex) > 0.5, torch.tensor(1.0, device='cuda'), torch.tensor(0.0, device='cuda')) total = total / len(models) # len must be an odd number results.append(total.tolist()) return results nsml.bind(save=save, load=load, infer=infer)
def bind_nsml(model, optimizer): def save(dir_name, *args, **kwargs): if not isinstance(dir_name, str): dir_name = str(dir_name) os.makedirs(dir_name, exist_ok=True) state = { 'model': model.state_dict(), 'optimizer': optimizer.state_dict() } fname = os.path.join(dir_name, 'model.pth') torch.save(state, fname) print('saved') def load(dir_name, *args, **kwargs): if not isinstance(dir_name, str): dir_name = str(dir_name) fname = os.path.join(dir_name, 'model.pth') state = torch.load(fname) model.load_state_dict(state['model']) optimizer.load_state_dict(state['optimizer']) print('loaded') def infer(root_path): return _infer(model, root_path) if use_nsml: nsml.bind(save=save, load=load, infer=infer) return save, load
def bind_model(model): def save(dirname, *args): if torch.cuda.device_count() > 1: checkpoint = {'model': model.module.state_dict()} else: checkpoint = {'model': model.state_dict()} torch.save(checkpoint, os.path.join(dirname, 'model.pt')) def load(dirname, *args): checkpoint = torch.load(os.path.join(dirname, 'model.pt')) model.load_state_dict(checkpoint['model']) def infer(raw_data, **kwargs): model.eval() examples = HateSpeech(raw_data).examples tensors = [ torch.tensor(ex.syllable_contents, device='cuda') for ex in examples ] results = [] for tensor in tensors: tensor = tensor.reshape(-1, len(tensor)) index1 = tensor == 1 index0 = tensor == 0 tensor[index1] = 0 tensor[index0] = 1 pred = model( tensor, torch.tensor([[3]] * len(tensor)).cuda())[0].sigmoid()[0][0].tolist() results.append(pred) return results nsml.bind(save=save, load=load, infer=infer)
def bind_model(sess, config): # 학습한 모델을 저장하는 함수입니다. def save(dir_name, *args): # directory os.makedirs(dir_name, exist_ok=True) saver = tf.train.Saver() saver.save(sess, os.path.join(dir_name, 'model')) # 저장한 모델을 불러올 수 있는 함수입니다. def load(dir_name, *args): saver = tf.train.Saver() # find checkpoint ckpt = tf.train.get_checkpoint_state(dir_name) if ckpt and ckpt.model_checkpoint_path: checkpoint = os.path.basename(ckpt.model_checkpoint_path) saver.restore(sess, os.path.join(dir_name, checkpoint)) else: raise NotImplemented('No checkpoint!') print('Model loaded') def infer(raw_data, **kwargs): left_preprocessed_data, right_preprocessed_data = preprocess(raw_data, config.strmaxlen) # 저장한 모델에 입력값을 넣고 prediction 결과를 리턴받습니다 pred = sess.run(output_sigmoid, feed_dict={x_1: left_preprocessed_data, x_2: right_preprocessed_data}) clipped = np.array(pred > config.threshold, dtype=np.int) return list(zip(pred.flatten(), clipped.flatten())) # DONOTCHANGE: They are reserved for nsml # nsml에서 지정한 함수에 접근할 수 있도록 하는 함수입니다. nsml.bind(save=save, load=load, infer=infer)
def bind_model(model): def save(dir_name): os.makedirs(dir_name, exist_ok=True) torch.save(model.state_dict(), os.path.join(dir_name, 'model')) print('model saved!') def load(dir_name): model.load_state_dict(torch.load(os.path.join(dir_name, 'model'))) model.eval() print('model loaded!') def infer(data): ## test mode X = ImagePreprocessing(data) X = np.array(X) X = np.expand_dims(X, axis=1) ##### DO NOT CHANGE ORDER OF TEST DATA ##### with torch.no_grad(): X = torch.from_numpy(X).float().to(device) pred = model.forward(X) prob, pred_cls = torch.max(pred, 1) pred_cls = pred_cls.tolist() #pred_cls = pred_cls.data.cpu().numpy() print('Prediction done!\n Saving the result...') return pred_cls nsml.bind(save=save, load=load, infer=infer)
def bind_nsml(model, optimizer, scheduler): def save(dir_name, *args, **kwargs): os.makedirs(dir_name, exist_ok=True) state = { 'model': model.state_dict(), 'optimizer': optimizer.state_dict(), 'scheduler': scheduler.state_dict() } torch.save(state, os.path.join(dir_name, 'model.pth')) def load(dir_name, *args, **kwargs): state = torch.load(os.path.join(dir_name, 'model.pth')) model.load_state_dict({k.replace('module.', ''): v for k, v in state['model'].items()}) optimizer.load_state_dict(state['optimizer']) scheduler.load_state_dict(state['scheduler']) print('loaded') def infer(root_path, top_k=1): if C.get()['mode'] != 'test': return _infer(model, root_path) if C.get()['infer_mode'] == 'ret': return _infer_ret(model, root_path) else: return _infer(model, root_path) nsml.bind(save=save, load=load, infer=infer)
def bind_model(sess): def save(dir_name): os.makedirs(dir_name, exist_ok=True) saver = tf.train.Saver() saver.save(sess, os.path.join(dir_name, 'model'), global_step=model.global_step) def load(dir_name): saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state(dir_name) if ckpt and ckpt.model_checkpoint_path: checkpoint = os.path.basename(ckpt.model_checkpoint_path) saver.restore(sess, os.path.join(dir_name, checkpoint)) else: raise NotImplemented('No checkpoint!') print('model loaded!') def infer(input, **kwargs): pred = [] # 학습용 데이터셋 구성 dataset.parameter["train_lines"] = len(input) dataset.make_input_data(input) reverse_tag = { v: k for k, v in dataset.necessary_data["ner_tag"].items() } # 테스트 셋을 측정한다. for morph, ne_dict, character, seq_len, char_len, _, step in dataset.get_data_batch_size( len(input), False): feed_dict = { model.morph: morph, model.ne_dict: ne_dict, model.character: character, model.sequence: seq_len, model.character_len: char_len, model.dropout_rate: 1.0 } viters = sess.run(model.viterbi_sequence, feed_dict=feed_dict) for index, viter in zip(range(0, len(viters)), viters): pred.append( get_ner_bi_tag_list_in_sentence(reverse_tag, viter, seq_len[index])) # 최종 output 포맷 예시 # [(0.0, ['NUM_B', '-', '-', '-']), # (0.0, ['PER_B', 'PER_I', 'CVL_B', 'NUM_B', '-', '-', '-', '-', '-', '-']), # ( ), ( ) # ] padded_array = np.zeros(len(pred)) return list(zip(padded_array, pred)) # DO NOT CHANGE nsml.bind(save=save, load=load, infer=infer)
def bind_model(model): def save(dir_name): os.makedirs(dir_name, exist_ok=True) torch.save(model.state_dict(),os.path.join(dir_name, 'model')) print('model saved!') def load(dir_name): model.load_state_dict(torch.load(os.path.join(dir_name, 'model'))) model.eval() print('model loaded!') def infer(image_path): result1 = [] result2 = [] result3 = [] result4 = [] result5 = [] with torch.no_grad(): test_transform = tv.transforms.Compose([ tv.transforms.ToPILImage(mode = 'RGB'), tv.transforms.Resize(512), tv.transforms.RandomCrop(256), tv.transforms.ToTensor(), tv.transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) batch_dataset = PathDataset(image_path, labels=None, test_mode= True, transform = test_transform) batch_loader = DataLoader(dataset=batch_dataset, batch_size=batch_size,shuffle=False) # Train the model for i, images in enumerate(batch_loader): y_hat = model(images.to(device)).cpu().numpy() result1.extend(np.argmax(y_hat, axis=1)) for i, images in enumerate(batch_loader): y_hat = model(images.to(device)).cpu().numpy() result2.extend(np.argmax(y_hat, axis=1)) for i, images in enumerate(batch_loader): y_hat = model(images.to(device)).cpu().numpy() result3.extend(np.argmax(y_hat, axis=1)) for i, images in enumerate(batch_loader): y_hat = model(images.to(device)).cpu().numpy() result4.extend(np.argmax(y_hat, axis=1)) for i, images in enumerate(batch_loader): y_hat = model(images.to(device)).cpu().numpy() result5.extend(np.argmax(y_hat, axis=1)) total_result = np.array([result1,result2,result3,result4,result5]) before_result = list(np.count_nonzero(total_result, axis=0)) after_result = [1 if x>=3 else 0 for x in before_result] result = after_result print('predicted') return np.array(result) nsml.bind(save=save, load=load, infer=infer)
def bind_model(model): def save(dir_name): os.makedirs(dir_name, exist_ok=True) model.save(os.path.join(dir_name, 'model')) #model.save_weights(os.path.join(dir_name, 'model')) print('model saved!') def load(file_path): model = load_model(file_path) #model.load_weights(file_path) print('model loaded!') def infer(queries, _): test_path = DATASET_PATH + '/test/test_data' db = [ os.path.join(test_path, 'reference', path) for path in os.listdir(os.path.join(test_path, 'reference')) ] queries = [v.split('/')[-1].split('.')[0] for v in queries] db = [v.split('/')[-1].split('.')[0] for v in db] queries.sort() db.sort() print('infer start qeries length:', len(queries), 'db length:', len(db)) queries, query_vecs, references, reference_vecs = get_feature( model, queries, db) print('feature shape:', query_vecs.shape, reference_vecs.shape) ## l2 normalization query_vecs = l2_normalize(query_vecs) reference_vecs = l2_normalize(reference_vecs) print('l2 complete') # Calculate cosine similarity sim_matrix = np.dot(query_vecs, reference_vecs.T) print('cal_dot_complete sim matrix shape : ', sim_matrix.shape) indices = np.argsort(sim_matrix, axis=1) indices = np.flip(indices, axis=1) print('indices shape:', indices.shape) retrieval_results = {} print('queries length:', len(queries), 'references length', len(references)) for (i, query) in enumerate(queries): # print(indices[i].shape) ranked_list = [references[k] for k in indices[i]] # print(i, query, ranked_list) ranked_list = ranked_list[:1000] retrieval_results[query] = ranked_list print('done') return list( zip(range(len(retrieval_results)), retrieval_results.items())) # DONOTCHANGE: They are reserved for nsml nsml.bind(save=save, load=load, infer=infer)
def bind_model(sess, config, model, vocabulary): # 학습한 모델을 저장하는 함수입니다. def save(dir_name, *args): # directory os.makedirs(dir_name, exist_ok=True) saver = tf.train.Saver() saver.save(sess, os.path.join(dir_name, 'model')) #print('save_seq vocabulary = {}'.format(vocabulary)) #print('save_seq vocabulary.voc = {}'.format(vocabulary.voc)) with open(os.path.join(dir_name, 'voc.pkl'), 'wb') as f: pickle.dump(vocabulary, f) # 저장한 모델을 불러올 수 있는 함수입니다. def load(dir_name, *args): saver = tf.train.Saver() # find checkpoint ckpt = tf.train.get_checkpoint_state(dir_name) if ckpt and ckpt.model_checkpoint_path: checkpoint = os.path.basename(ckpt.model_checkpoint_path) saver.restore(sess, os.path.join(dir_name, checkpoint)) else: raise NotImplemented('No checkpoint!') print('Model loaded') with open(os.path.join(dir_name, 'voc.pkl'), 'rb') as f: vocabularyf = pickle.load(f) vocabulary.append(vocabularyf) def infer(raw_data, **kwargs): """ :param raw_data: raw input (여기서는 문자열)을 입력받습니다 :param kwargs: :return: """ preprocessed_data = data_helpers.load_data_infer( raw_data, vocabulary[0]) feed_dict = { model.input_x: preprocessed_data, model.dropout_keep_prob: 1.0 } pred = sess.run(model.predictions, feed_dict=feed_dict) pred = pred + 1 con = np.zeros(len(pred)) result = list(zip(con, pred)) # DONOTCHANGE: They are reserved for nsml # 리턴 결과는 [(확률, 0 or 1)] 의 형태로 보내야만 리더보드에 올릴 수 있습니다. 리더보드 결과에 확률의 값은 영향을 미치지 않습니다 # [(1, 7.4), (1, 2.3)] return result # DONOTCHANGE: They are reserved for nsml # nsml에서 지정한 함수에 접근할 수 있도록 하는 함수입니다. nsml.bind(save=save, load=load, infer=infer)
def __init__(self, policy, ob_space, ac_space, nenvs, total_timesteps, nprocs=32, nscripts=16, nsteps=20, nstack=4, ent_coef=0.1, vf_coef=0.5, vf_fisher_coef=1.0, lr=0.25, max_grad_norm=0.001, kfac_clip=0.001, lrschedule='linear', alpha=0.99, epsilon=1e-5): config = tf.ConfigProto( allow_soft_placement=True, intra_op_parallelism_threads=nprocs, inter_op_parallelism_threads=nprocs) config.gpu_options.allow_growth = True self.sess = sess = tf.Session(config=config) nsml.bind(sess=sess) #nact = ac_space.n nbatch = nenvs * nsteps A = tf.placeholder(tf.int32, [nbatch]) XY0 = tf.placeholder(tf.int32, [nbatch]) XY1 = tf.placeholder(tf.int32, [nbatch]) # ADV == TD_TARGET - values ADV = tf.placeholder(tf.float32, [nbatch]) TD_TARGET = tf.placeholder(tf.float32, [nbatch]) PG_LR = tf.placeholder(tf.float32, []) VF_LR = tf.placeholder(tf.float32, []) self.model = step_model = policy( sess, ob_space, ac_space, nenvs, 1, nstack, reuse=False) self.model2 = train_model = policy( sess, ob_space, ac_space, nenvs, nsteps, nstack, reuse=True) # Policy 1 : Base Action : train_model.pi label = A script_mask = tf.concat( [ tf.zeros([nscripts * nsteps, 1]), tf.ones([(nprocs - nscripts) * nsteps, 1]) ], axis=0) pi = train_model.pi pac_weight = script_mask * (tf.nn.softmax(pi) - 1.0) + 1.0 pac_weight = tf.reduce_sum(pac_weight * tf.one_hot(A, depth=3), axis=1) neglogpac = tf.nn.sparse_softmax_cross_entropy_with_logits( logits=pi, labels=A) neglogpac *= tf.stop_gradient(pac_weight) inv_A = 1.0 - tf.cast(A, tf.float32) xy0_mask = tf.cast(A, tf.float32) xy1_mask = tf.cast(A, tf.float32) condition0 = tf.equal(xy0_mask, 2) xy0_mask = tf.where(condition0, tf.ones(tf.shape(xy0_mask)), xy0_mask) xy0_mask = 1.0 - xy0_mask condition1 = tf.equal(xy1_mask, 2) xy1_mask = tf.where(condition1, tf.zeros(tf.shape(xy1_mask)), xy1_mask) # One hot representation of chosen marine. # [batch_size, 2] pi_xy0 = train_model.pi_xy0 pac_weight = script_mask * (tf.nn.softmax(pi_xy0) - 1.0) + 1.0 pac_weight = tf.reduce_sum( pac_weight * tf.one_hot(XY0, depth=1024), axis=1) logpac_xy0 = tf.nn.sparse_softmax_cross_entropy_with_logits( logits=pi_xy0, labels=XY0) logpac_xy0 *= tf.stop_gradient(pac_weight) logpac_xy0 *= tf.cast(xy0_mask, tf.float32) pi_xy1 = train_model.pi_xy1 pac_weight = script_mask * (tf.nn.softmax(pi_xy1) - 1.0) + 1.0 pac_weight = tf.reduce_sum( pac_weight * tf.one_hot(XY0, depth=1024), axis=1) # 1D? 2D? logpac_xy1 = tf.nn.sparse_softmax_cross_entropy_with_logits( logits=pi_xy1, labels=XY1) logpac_xy1 *= tf.stop_gradient(pac_weight) logpac_xy1 *= tf.cast(xy1_mask, tf.float32) pg_loss = tf.reduce_mean(ADV * neglogpac) pg_loss_xy0 = tf.reduce_mean(ADV * logpac_xy0) pg_loss_xy1 = tf.reduce_mean(ADV * logpac_xy1) vf_ = tf.squeeze(train_model.vf) vf_r = tf.concat( [ tf.ones([nscripts * nsteps, 1]), tf.zeros([(nprocs - nscripts) * nsteps, 1]) ], axis=0) * TD_TARGET vf_masked = vf_ * script_mask + vf_r #vf_mask[0:nscripts * nsteps] = R[0:nscripts * nsteps] vf_loss = tf.reduce_mean(mse(vf_masked, TD_TARGET)) entropy_a = tf.reduce_mean(cat_entropy(train_model.pi)) entropy_xy0 = tf.reduce_mean(cat_entropy(train_model.pi_xy0)) entropy_xy1 = tf.reduce_mean(cat_entropy(train_model.pi_xy1)) entropy = entropy_a + entropy_xy0 + entropy_xy1 loss = pg_loss - entropy * ent_coef + vf_loss * vf_coef params = find_trainable_variables("model") grads = tf.gradients(loss, params) if max_grad_norm is not None: grads, _ = tf.clip_by_global_norm(grads, max_grad_norm) grads = list(zip(grads, params)) trainer = tf.train.RMSPropOptimizer( learning_rate=lr, decay=alpha, epsilon=epsilon) _train = trainer.apply_gradients(grads) self.logits = logits = train_model.pi # xy0 self.params_common = params_common = tf.get_collection( tf.GraphKeys.TRAINABLE_VARIABLES, scope='model/common') self.params_xy0 = params_xy0 = tf.get_collection( tf.GraphKeys.TRAINABLE_VARIABLES, scope='model/xy0') + params_common train_loss_xy0 = pg_loss_xy0 - entropy * ent_coef + vf_coef * vf_loss self.grads_check_xy0 = grads_xy0 = tf.gradients( train_loss_xy0, params_xy0) if max_grad_norm is not None: grads_xy0, _ = tf.clip_by_global_norm(grads_xy0, max_grad_norm) grads_xy0 = list(zip(grads_xy0, params_xy0)) trainer_xy0 = tf.train.RMSPropOptimizer( learning_rate=lr, decay=alpha, epsilon=epsilon) _train_xy0 = trainer_xy0.apply_gradients(grads_xy0) # xy1 self.params_xy1 = params_xy1 = tf.get_collection( tf.GraphKeys.TRAINABLE_VARIABLES, scope='model/xy1') + params_common train_loss_xy1 = pg_loss_xy1 - entropy * ent_coef + vf_coef * vf_loss self.grads_check_xy1 = grads_xy1 = tf.gradients( train_loss_xy1, params_xy1) if max_grad_norm is not None: grads_xy1, _ = tf.clip_by_global_norm(grads_xy1, max_grad_norm) grads_xy1 = list(zip(grads_xy1, params_xy1)) trainer_xy1 = tf.train.RMSPropOptimizer( learning_rate=lr, decay=alpha, epsilon=epsilon) _train_xy1 = trainer_xy1.apply_gradients(grads_xy1) self.lr = Scheduler(v=lr, nvalues=total_timesteps, schedule=lrschedule) def train(obs, states, td_targets, masks, actions, xy0, xy1, values): advs = td_targets - values for step in range(len(obs)): cur_lr = self.lr.value() td_map = { train_model.X: obs, A: actions, XY0: xy0, XY1: xy1, ADV: advs, TD_TARGET: td_targets, PG_LR: cur_lr } if states != []: td_map[train_model.S] = states td_map[train_model.M] = masks policy_loss, value_loss, policy_entropy, _, \ policy_loss_xy0, policy_entropy_xy0, _, \ policy_loss_xy1, policy_entropy_xy1, _ = sess.run( [pg_loss, vf_loss, entropy, _train, pg_loss_xy0, entropy_xy0, _train_xy0, pg_loss_xy1, entropy_xy1, _train_xy1], td_map) return policy_loss, value_loss, policy_entropy, \ policy_loss_xy0, policy_entropy_xy0, \ policy_loss_xy1, policy_entropy_xy1 def save(save_path): ps = sess.run(params) joblib.dump(ps, save_path) def load(load_path): loaded_params = joblib.load(load_path) restores = [] for p, loaded_p in zip(params, loaded_params): restores.append(p.assign(loaded_p)) sess.run(restores) self.train = train self.save = save self.load = load self.train_model = train_model self.step_model = step_model self.step = step_model.step self.value = step_model.value self.initial_state = step_model.initial_state print("global_variables_initializer start") tf.global_variables_initializer().run(session=sess) print("global_variables_initializer complete")