def main(image_path, nsamples, temperature): # Generate questions sess = gpt2.start_tf_sess() gpt2.load_gpt2(sess) config = yaml.load(open('config.yaml', 'r'), Loader=yaml.FullLoader) checkpoint_path = os.path.join(config['project_root_dir'], config['checkpoint_path']) vocab_file_path = os.path.join(config['project_root_dir'], config['vocab_file_path']) captioner = Captioner(sess, checkpoint_path, vocab_file_path) caption = captioner.caption(image_path) questions = gpt2_gen_questions(sess, caption, nsamples=nsamples, temperature=temperature) # Print generated questions print('----------\nQuestions:') for i, question in enumerate(questions): print('%d. %s' % (i + 1, question))
class WindowWidget(QtWidgets.QWidget): def __init__(self, voice): super(WindowWidget, self).__init__() self.sess = None self.captioner = None self.prepare_questioner() self.prepare_captioner() self.threadpool = QtCore.QThreadPool() self.questioner_running = False self.applying_output = False global graph graph = tf.get_default_graph() if voice: self.tts = TTS() else: self.tts = None # Viewing region self.viewing_region = QtWidgets.QLabel(self) layout = QtWidgets.QHBoxLayout() layout.addWidget(self.viewing_region) # Load button self.load_button = QtWidgets.QPushButton('Load image') self.load_button.clicked.connect(self.load_button_clicked) right_sidebar = QtWidgets.QVBoxLayout() right_sidebar.addWidget(self.load_button) # Extra instructions region self.instr_region = QtWidgets.QLabel(self) self.instr_region.setText('Or drop an image onto this window.') right_sidebar.addWidget(self.instr_region) right_sidebar.addStretch() # Progress bar self.progress = QtWidgets.QProgressBar(self) self.progress.setMaximum(100) right_sidebar.addWidget(self.progress) self.progress.hide() # Text region self.text_region = QtWidgets.QLabel(self) self.text_region.setFrameStyle(QtWidgets.QFrame.Panel | QtWidgets.QFrame.Sunken) self.text_region.setWordWrap(True) self.text_region.setMargin(8) self.text_region.setText('...') right_sidebar.addWidget(self.text_region) layout.addLayout(right_sidebar) # Launch self.setWindowIcon(QtGui.QIcon(os.path.join('icon', 'question.png'))) self.setLayout(layout) self.setAcceptDrops(True) self.show() def prepare_questioner(self): self.sess = gpt2.start_tf_sess() gpt2.load_gpt2(self.sess) def prepare_captioner(self): config = yaml.load(open('config.yaml', 'r'), Loader=yaml.FullLoader) checkpoint_path = os.path.join(config['project_root_dir'], config['checkpoint_path']) vocab_file_path = os.path.join(config['project_root_dir'], config['vocab_file_path']) self.captioner = Captioner(self.sess, checkpoint_path, vocab_file_path) def load_button_clicked(self): if self.questioner_running or self.applying_output: print("Can't load an image right now. Questioner is busy.") else: image_path, _ = QtWidgets.QFileDialog.getOpenFileName( self, 'Open file') if image_path: self.load_image(image_path) def load_image(self, image_path): pixmap = QtGui.QPixmap(image_path) pixmap = pixmap.scaled(500, 500, QtCore.Qt.KeepAspectRatio) self.viewing_region.setPixmap(pixmap) self.text_region.setText('Questioner is working.') self.adjustSize() self.questioner_running = True worker = Worker(self.run_questioner, image_path) worker.signals.finished.connect(self.questioner_finished) worker.signals.error.connect(self.questioner_failed) worker.signals.result.connect(self.apply_questioner_output) self.threadpool.start(worker) def run_questioner(self, image_path): global graph with graph.as_default(): # this is run on a separate thread caption = self.captioner.caption(image_path) questions = gpt2_gen_questions(self.sess, caption, nsamples=1, temperature=0.7) return questions[0] if len(questions) > 0 else '' def questioner_finished(self): self.questioner_running = False def questioner_failed(self, e): print(e) def apply_questioner_output(self, question): self.applying_output = True if len(question) > 0: self.text_region.setText(question) if self.tts: self.progress.show() self.tts.speak(question, self.tts_callback) self.progress.hide() self.applying_output = False def tts_callback(self, i, seq_len, batch_size, gen_rate): percentage = i * 100 / seq_len self.progress.setValue(percentage) def dragEnterEvent(self, evt): if evt.mimeData().hasUrls: evt.accept() else: evt.ignore() def dragMoveEvent(self, evt): if evt.mimeData().hasUrls: evt.accept() else: evt.ignore() def dropEvent(self, evt): if evt.mimeData().hasUrls \ and not self.questioner_running \ and not self.applying_output: evt.setDropAction(QtCore.Qt.CopyAction) evt.accept() for url in evt.mimeData().urls(): if op_sys == 'Darwin': image_path = str( NSURL.URLWithString_(str( url.toString())).filePathURL().path()) else: image_path = str(url.toLocalFile()) self.load_image(image_path) else: evt.ignore()