def answerer(input_string, tp='big'): my_extractor = extractor() my_extractor.from_string(input_string) my_responser = responser() obj1, obj2, predicates = my_extractor.get_params() print("len(obj1), len(obj2)", len(obj1), len(obj2)) print("obj1, obj2, predicates", obj1, obj2, predicates) if (len(obj1) > 0 and len(obj2) > 0): response = my_responser.get_response( first_object=obj1, second_object=obj2, fast_search=True, aspects=predicates, weights=[1 for predicate in predicates]) try: response_json = response.json() except: return ("smth wrong in response, please try again") try: my_diviner = diviner(tp=tp) print(1) my_diviner.create_from_json(response_json, predicates) print(2) except: return ("smth wrong in diviner, please try again") try: answer = my_diviner.generate_advice() print("answer", answer) #del my_extractor,my_diviner, my_responser return answer except: #del my_extractor,my_diviner, my_responser return ("smth wrong in answer generation, please try again") elif (len(obj1) > 0 and len(obj2) == 0): print("len(obj1) > 0 and len(obj2) == 0") response = my_responser.get_response( first_object=obj1, second_object='and', fast_search=True, aspects=predicates, weights=[1 for predicate in predicates]) try: response_json = response.json() my_diviner = diviner(tp="big") my_diviner.create_from_json(response_json, predicates) answer = my_diviner.generate_advice(is_object_single=True) print("answer", answer) #del my_extractor,my_diviner, my_responser return answer except: #del my_extractor,my_diviner, my_responser return ("smth wrong in response, please try again") else: return ("We can't recognize objects for comparision")
#device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") #LM_CAM = load_cam_model(device) #Cam = diviner(tp = 'cam', model = LM_CAM, device = device) #print ("loaded cam") device = torch.device("cuda:2" if torch.cuda.is_available() else "cpu") from ctrl_generation import initialize_model model_type = "ctrl" #PUT NAME OF NEEDED MODEL length = 200 #MODEL LENGTH device = torch.device("cuda:3" if torch.cuda.is_available() else "cpu") model, tokenizer, length = initialize_model(model_type, length, device) CTRL = diviner(tp = 'ctrl', model = model, device = device, tokenizer = tokenizer) #my_extractor = extractor(my_device = 0) print ("loaded extractor") def main(): df = pd.DataFrame(columns=['Object 1', 'Object 2', 'Question', 'Best Answer', 'Answers']) with open('yahoo_answers_positive_questions.csv', 'r') as file: reader = csv.reader(file) for ind, row in enumerate(reader): d = {'Object 1': row[0], 'Object 2': row[1], 'Question': row[2], 'Best Answer': row[3], 'Answers': [elem for elem in row[3:]]} if (ind > 0): df = df.append(d, ignore_index=True) gpt_answ_list = []
from generation.generation import diviner from my_functions import extractor from my_functions import responser from cam_summarize import load_cam_model from text_gen_big import load_big_model from text_gen import load_small_model #device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") #LM_CAM = load_cam_model(device) #Cam = diviner(tp = 'cam', model = LM_CAM, device = device) #print ("loaded cam") device = torch.device("cuda:3" if torch.cuda.is_available() else "cpu") LM_SMALL = load_small_model(device) GPT2Small = diviner(tp='small', model=LM_SMALL, device=device) #GPT2Small_vs = diviner(tp = 'small_vs', model = LM_SMALL, device = device) #GPT2Small_vs_str = diviner(tp = 'small_vs_str', model = LM_SMALL, device = device) print("loaded gpt2") Templ = diviner(tp='templates', model='', device=device) my_extractor = extractor(my_device=3) print("loaded extractor") def main(): df = pd.DataFrame( columns=['Object 1', 'Object 2', 'Question', 'Best Answer', 'Answers']) with open('yahoo_answers_positive_questions.csv', 'r') as file:
from flasgger import Swagger, LazyString, LazyJSONEncoder from flask_restful import Api, Resource, reqparse from flask import make_response from nltk.tokenize import sent_tokenize, word_tokenize import random import json from flask import jsonify import json """Models""" from generation.generation import diviner from my_functions import extractor from my_functions import responser GPT2Big = diviner('big', device = 5) GPT2Small = diviner('small', device = 6) Templ = diviner('templates', device = 5) Cam = diviner('cam', device = 6) class ReverseProxied(object): def __init__(self, app): self.app = app def __call__(self, environ, start_response): script_name = environ.get('HTTP_X_SCRIPT_NAME', '') if script_name:
sys.path.insert(0, "/notebook/cqas") sys.path.insert(0, "/notebook/cqas/generation/gpt-2-Pytorch") sys.path.insert(0, "/notebook/cqas/generation/Student") sys.path.insert(0, "/notebook/cqas/generation/pytorch_transformers") from generation.generation import diviner from my_functions import extractor from my_functions import responser from cam_summarize import load_cam_model from text_gen_big import load_big_model from text_gen import load_small_model device = torch.device("cuda:3" if torch.cuda.is_available() else "cpu") LM_CAM = load_cam_model(device) Cam = diviner(tp='cam', model=LM_CAM, device=device) print("loaded cam") #device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") #LM_SMALL = load_small_model(device) #GPT2Small = diviner(tp = 'small', model = LM_SMALL, device = device) #GPT2Small_vs = diviner(tp = 'small_vs', model = LM_SMALL, device = device) #GPT2Small_vs_str = diviner(tp = 'small_vs_str', model = LM_SMALL, device = device) print("loaded gpt2") Templ = diviner(tp='templates', model='', device=device) my_extractor = extractor(my_device=5) print("loaded extractor")
parser.add_argument( '--input', help= 'input phrases as a list form, i.e [qwestion1, qwestion2, qwestion3]') args = parser.parse_args() x = ast.literal_eval(args.input) words = create_sequence_from_sentence([ 'what is better amazon or itunes for showing', 'what is better mouse or rat', 'what is easier to make bread o pizza' ]) model = TaggerFactory.load(PATH_TO_PRETRAINED + MODEL_NAME) tags = model.predict_tags_from_words(words) objects_list = [] for elem in list(zip(words, tags)): objects = get_objects(elem[0], elem[1]) assert len(objects) >= 2, "We have %d objects to compare" % ( len(objects)) objects_list.append((objects[0], objects[1])) for obj0, obj1 in objects_list: response = get_response(obj0, obj1, False) response_json = response.json() Merlin = diviner() Merlin.create_from_json(response_json) Merlin.generate_advice() print('\nThe end.')
from generation.Student.cam_summarize import load_cam_model #from text_gen_big import load_big_model #from text_gen import load_small_model from generation.Student.ctrl_generation import initialize_model from generation.Student.ctrl_generation import generate_text_from_condition model_type = "ctrl" #PUT NAME OF NEEDED MODEL length = 100 #MODEL LENGTH import configparser config_parser = configparser.ConfigParser() config_parser.read('config.ini') config = config_parser['DEV'] device = torch.device("cuda:2" if torch.cuda.is_available() else "cpu") #LM_CAM = load_cam_model(device) Cam = diviner(tp='cam', model='', device=device) print("loaded cam") #device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") #LM_SMALL = load_small_model(device) #GPT2Small = diviner(tp = 'small', model = LM_SMALL, device = device) #print ("loaded gpt2") device = torch.device("cuda:2" if torch.cuda.is_available() else "cpu") model, tokenizer, length = initialize_model(model_type, length, device=device) CTRL = diviner(tp='ctrl', model=model, device=device, tokenizer=tokenizer) print("loaded ctrl") #device = torch.device("cuda:6" if torch.cuda.is_available() else "cpu") #LM_BIG, tokenizer_big = load_big_model(device)
sys.path.insert(0, "/notebook/cqas") sys.path.insert(0, "/notebook/cqas/generation/gpt-2-Pytorch") sys.path.insert(0, "/notebook/cqas/generation/Student") sys.path.insert(0, "/notebook/cqas/generation/pytorch_transformers") from generation.generation import diviner from my_functions import extractor from my_functions import responser from cam_summarize import load_cam_model from text_gen_big import load_big_model from text_gen import load_small_model device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") LM_CAM = load_cam_model(device) Cam = diviner(tp = 'cam', model = LM_CAM, device = device) print ("loaded cam") device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") LM_SMALL = load_small_model(device) GPT2Small = diviner(tp = 'small', model = LM_SMALL, device = device) GPT2Small_vs = diviner(tp = 'small_vs', model = LM_SMALL, device = device) GPT2Small_vs_str = diviner(tp = 'small_vs_str', model = LM_SMALL, device = device) print ("loaded gpt2") Templ = diviner(tp = 'templates', model = '', device = device) my_extractor = extractor(my_device = 0) print ("loaded extractor")
sys.path.insert(0, "/notebook/cqas") sys.path.insert(0, "/notebook/cqas/generation/gpt-2-Pytorch") sys.path.insert(0, "/notebook/cqas/generation/Student") sys.path.insert(0, "/notebook/cqas/generation/pytorch_transformers") from generation.generation import diviner from my_functions import extractor from my_functions import responser from cam_summarize import load_cam_model from text_gen_big import load_big_model from text_gen import load_small_model device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") LM_CAM = load_cam_model(device) Cam = diviner(tp='cam', model=LM_CAM, device=device) print("loaded cam") from evaluation.gen import generate_one_answer_with_defined_objects df = pd.DataFrame( columns=['Object 1', 'Object 2', 'Question', 'Best Answer', 'Answers']) def main(): df = pd.DataFrame( columns=['Object 1', 'Object 2', 'Question', 'Best Answer', 'Answers']) with open('yahoo_answers_positive_questions.csv', 'r') as file: reader = csv.reader(file) for ind, row in enumerate(reader):