def _interactive_input_fn(hparams, decode_hp, Q): """Generator that reads from the terminal and yields "interactive inputs". Due to temporary limitations in tf.learn, if we don't want to reload the whole graph, then we are stuck encoding all of the input as one fixed-size numpy array. We yield int32 arrays with shape [const_array_size]. The format is: [num_samples, decode_length, len(input ids), <input ids>, <padding>] Args: hparams: model hparams decode_hp: decode hparams Yields: numpy arrays Raises: Exception: when `input_type` is invalid. """ num_samples = decode_hp.num_samples if decode_hp.num_samples > 0 else 1 decode_length = 1 #go one more input_type = "text" p_hparams = hparams.problem_hparams has_input = "inputs" in p_hparams.modality vocabulary = p_hparams.vocabulary["inputs" if has_input else "targets"] # This should be longer than the longest input. const_array_size = 10000 # Import readline if available for command line editing and recall. print('*' * 100) print('getting...') input_string = Q.get() print('got!', input_string) input_ids = vocabulary.encode(input_string) if has_input: input_ids.append(text_encoder.EOS_ID) x = [num_samples, decode_length, len(input_ids)] + input_ids assert len(x) < const_array_size x += [0] * (const_array_size - len(x)) features = { "inputs": np.array(x).astype(np.int32), } for k, v in six.iteritems( problem_lib.problem_hparams_to_features(p_hparams)): features[k] = np.array(v).astype(np.int32) print('yielded') yield features
def __interactive_input_fn(self): num_samples = self.decode_hp.num_samples if self.decode_hp.num_samples > 0\ else 1 decode_length = self.decode_hp.extra_length input_type = "text" p_hparams = self.hparams.problem_hparams has_input = "inputs" in p_hparams.input_modality vocabulary = p_hparams.vocabulary["inputs" if has_input else "targets"] # Import readline if available for command line editing and recall. try: import readline # pylint: disable=g-import-not-at-top,unused-variable except ImportError: pass while True: features = { "inputs": np.array(self.inputs).astype(np.int32), } for k, v in six.iteritems(problem_hparams_to_features(p_hparams)): features[k] = np.array(v).astype(np.int32) yield features
def _interactive_input_fn(hparams, decode_length=1024, input_string="\documentclass"): num_samples = 1 input_type = "text" p_hparams = hparams.problem_hparams has_input = "inputs" in p_hparams.modality vocabulary = p_hparams.vocabulary["inputs" if has_input else "targets"] # This should be longer than the longest input. const_array_size = 10000 input_ids = vocabulary.encode(input_string) if has_input: input_ids.append(text_encoder.EOS_ID) x = [num_samples, decode_length, len(input_ids)] + input_ids assert len(x) < const_array_size x += [0] * (const_array_size - len(x)) features = { "inputs": np.array(x).astype(np.int32), } for k, v in six.iteritems(problem_lib.problem_hparams_to_features(p_hparams)): features[k] = np.array(v).astype(np.int32) yield features
def __interactive_input_fn(self): num_samples = self.decode_hp.num_samples if self.decode_hp.num_samples > 0\ else 1 decode_length = self.decode_hp.extra_length input_type = "text" p_hparams = self.hparams.problem_hparams has_input = "inputs" in p_hparams.input_modality vocabulary = p_hparams.vocabulary["inputs" if has_input else "targets"] # This should be longer than the longest input. const_array_size = 10000 # Import readline if available for command line editing and recall. try: import readline # pylint: disable=g-import-not-at-top,unused-variable except ImportError: pass while True: features = { "inputs": np.array(self.inputs).astype(np.int32), } for k, v in six.iteritems(problem_hparams_to_features(p_hparams)): features[k] = np.array(v).astype(np.int32) yield features
def _interactive_input_fn(hparams, decode_hp): """Generator that reads from the terminal and yields "interactive inputs". Due to temporary limitations in tf.learn, if we don't want to reload the whole graph, then we are stuck encoding all of the input as one fixed-size numpy array. We yield int32 arrays with shape [const_array_size]. The format is: [num_samples, decode_length, len(input ids), <input ids>, <padding>] Args: hparams: model hparams decode_hp: decode hparams Yields: numpy arrays Raises: Exception: when `input_type` is invalid. """ num_samples = decode_hp.num_samples if decode_hp.num_samples > 0 else 1 decode_length = decode_hp.extra_length input_type = "text" p_hparams = hparams.problem_hparams has_input = "inputs" in p_hparams.input_modality vocabulary = p_hparams.vocabulary["inputs" if has_input else "targets"] # This should be longer than the longest input. const_array_size = 10000 # Import readline if available for command line editing and recall. try: import readline # pylint: disable=g-import-not-at-top,unused-variable except ImportError: pass while True: prompt = ( "INTERACTIVE MODE num_samples=%d decode_length=%d \n" " it=<input_type> ('text' or 'image' or 'label', default: " "text)\n" " ns=<num_samples> (changes number of samples, default: 1)\n" " dl=<decode_length> (changes decode length, default: 100)\n" " <%s> (decode)\n" " q (quit)\n" ">" % (num_samples, decode_length, "source_string" if has_input else "target_prefix")) input_string = input(prompt) if input_string == "q": return elif input_string[:3] == "ns=": num_samples = int(input_string[3:]) elif input_string[:3] == "dl=": decode_length = int(input_string[3:]) elif input_string[:3] == "it=": input_type = input_string[3:] else: if input_type == "text": input_ids = vocabulary.encode(input_string) if has_input: input_ids.append(text_encoder.EOS_ID) x = [num_samples, decode_length, len(input_ids)] + input_ids assert len(x) < const_array_size x += [0] * (const_array_size - len(x)) features = { "inputs": np.array(x).astype(np.int32), } elif input_type == "image": input_path = input_string img = vocabulary.encode(input_path) features = { "inputs": img.astype(np.int32), } elif input_type == "label": input_ids = [int(input_string)] x = [num_samples, decode_length, len(input_ids)] + input_ids features = { "inputs": np.array(x).astype(np.int32), } else: raise Exception("Unsupported input type.") for k, v in six.iteritems( problem_lib.problem_hparams_to_features(p_hparams)): features[k] = np.array(v).astype(np.int32) yield features
def _fill_problem_hparams_features(self, features): if features is not None: for k, v in six.iteritems( problem_hparams_to_features(self._problem_hparams)): if k not in features: features[k] = tf.constant(v, name=k)
def _interactive_input_fn(hparams, decode_hp): """Generator that reads from the terminal and yields "interactive inputs". Due to temporary limitations in tf.learn, if we don't want to reload the whole graph, then we are stuck encoding all of the input as one fixed-size numpy array. We yield int32 arrays with shape [const_array_size]. The format is: [num_samples, decode_length, len(input ids), <input ids>, <padding>] Args: hparams: model hparams decode_hp: decode hparams Yields: numpy arrays Raises: Exception: when `input_type` is invalid. """ num_samples = decode_hp.num_samples if decode_hp.num_samples > 0 else 1 decode_length = decode_hp.extra_length input_type = "text" p_hparams = hparams.problem_hparams has_input = "inputs" in p_hparams.modality vocabulary = p_hparams.vocabulary["inputs" if has_input else "targets"] # This should be longer than the longest input. const_array_size = 10000 # Import readline if available for command line editing and recall. try: import readline # pylint: disable=g-import-not-at-top,unused-variable except ImportError: pass while True: prompt = ("INTERACTIVE MODE num_samples=%d decode_length=%d \n" " it=<input_type> ('text' or 'image' or 'label', default: " "text)\n" " ns=<num_samples> (changes number of samples, default: 1)\n" " dl=<decode_length> (changes decode length, default: 100)\n" " <%s> (decode)\n" " q (quit)\n" ">" % (num_samples, decode_length, "source_string" if has_input else "target_prefix")) input_string = input(prompt) if input_string == "q": return elif input_string[:3] == "ns=": num_samples = int(input_string[3:]) elif input_string[:3] == "dl=": decode_length = int(input_string[3:]) elif input_string[:3] == "it=": input_type = input_string[3:] else: if input_type == "text": input_ids = vocabulary.encode(input_string) if has_input: input_ids.append(text_encoder.EOS_ID) x = [num_samples, decode_length, len(input_ids)] + input_ids assert len(x) < const_array_size x += [0] * (const_array_size - len(x)) features = { "inputs": np.array(x).astype(np.int32), } elif input_type == "image": input_path = input_string img = vocabulary.encode(input_path) features = { "inputs": img.astype(np.int32), } elif input_type == "label": input_ids = [int(input_string)] x = [num_samples, decode_length, len(input_ids)] + input_ids features = { "inputs": np.array(x).astype(np.int32), } else: raise Exception("Unsupported input type.") for k, v in six.iteritems( problem_lib.problem_hparams_to_features(p_hparams)): features[k] = np.array(v).astype(np.int32) yield features
def _fill_problem_hparams_features(self, features): if features is not None: for k, v in six.iteritems( problem_hparams_to_features(self._problem_hparams)): if k not in features: features[k] = tf.constant(v, name=k)