def _print_train_info(num_samples_or_steps, val_samples_or_steps, is_dataset): increment = 'steps' if is_dataset else 'samples' msg = 'Train on {0} {increment}'.format(num_samples_or_steps, increment=increment) if val_samples_or_steps: msg += ', validate on {0} {increment}'.format(val_samples_or_steps, increment=increment) io_utils.print_msg(msg)
def _print_train_info(num_samples_or_steps, val_samples_or_steps, is_dataset): increment = "steps" if is_dataset else "samples" msg = "Train on {0} {increment}".format(num_samples_or_steps, increment=increment) if val_samples_or_steps: msg += ", validate on {0} {increment}".format(val_samples_or_steps, increment=increment) io_utils.print_msg(msg)
def test_print_msg(self): enabled = io_utils.is_interactive_logging_enabled() io_utils.disable_interactive_logging() self.assertFalse(io_utils.is_interactive_logging_enabled()) with self.assertLogs(level="INFO") as logged: io_utils.print_msg("Testing Message") self.assertIn("Testing Message", logged.output[0]) io_utils.enable_interactive_logging() self.assertTrue(io_utils.is_interactive_logging_enabled()) with self.captureWritesToStream(sys.stdout) as printed: io_utils.print_msg("Testing Message") self.assertEqual("Testing Message\n", printed.contents()) if enabled: io_utils.enable_interactive_logging() else: io_utils.disable_interactive_logging()
def plot_model(model, to_file='model.png', show_shapes=False, show_dtype=False, show_layer_names=True, rankdir='TB', expand_nested=False, dpi=96, layer_range=None, show_layer_activations=False): """Converts a Keras model to dot format and save to a file. Example: ```python input = tf.keras.Input(shape=(100,), dtype='int32', name='input') x = tf.keras.layers.Embedding( output_dim=512, input_dim=10000, input_length=100)(input) x = tf.keras.layers.LSTM(32)(x) x = tf.keras.layers.Dense(64, activation='relu')(x) x = tf.keras.layers.Dense(64, activation='relu')(x) x = tf.keras.layers.Dense(64, activation='relu')(x) output = tf.keras.layers.Dense(1, activation='sigmoid', name='output')(x) model = tf.keras.Model(inputs=[input], outputs=[output]) dot_img_file = '/tmp/model_1.png' tf.keras.utils.plot_model(model, to_file=dot_img_file, show_shapes=True) ``` Args: model: A Keras model instance to_file: File name of the plot image. show_shapes: whether to display shape information. show_dtype: whether to display layer dtypes. show_layer_names: whether to display layer names. rankdir: `rankdir` argument passed to PyDot, a string specifying the format of the plot: 'TB' creates a vertical plot; 'LR' creates a horizontal plot. expand_nested: Whether to expand nested models into clusters. dpi: Dots per inch. layer_range: input of `list` containing two `str` items, which is the starting layer name and ending layer name (both inclusive) indicating the range of layers for which the plot will be generated. It also accepts regex patterns instead of exact name. In such case, start predicate will be the first element it matches to `layer_range[0]` and the end predicate will be the last element it matches to `layer_range[1]`. By default `None` which considers all layers of model. Note that you must pass range such that the resultant subgraph must be complete. show_layer_activations: Display layer activations (only for layers that have an `activation` property). Raises: ImportError: if graphviz or pydot are not available. ValueError: if `plot_model` is called before the model is built. Returns: A Jupyter notebook Image object if Jupyter is installed. This enables in-line display of the model plots in notebooks. """ if not model.built: raise ValueError( 'This model has not yet been built. ' 'Build the model first by calling `build()` or by calling ' 'the model on a batch of data.') if not check_graphviz(): message = ( 'You must install pydot (`pip install pydot`) ' 'and install graphviz ' '(see instructions at https://graphviz.gitlab.io/download/) ' 'for plot_model to work.') if 'IPython.core.magics.namespace' in sys.modules: # We don't raise an exception here in order to avoid crashing notebook # tests where graphviz is not available. io_utils.print_msg(message) return else: raise ImportError(message) dot = model_to_dot(model, show_shapes=show_shapes, show_dtype=show_dtype, show_layer_names=show_layer_names, rankdir=rankdir, expand_nested=expand_nested, dpi=dpi, layer_range=layer_range, show_layer_activations=show_layer_activations) to_file = io_utils.path_to_string(to_file) if dot is None: return _, extension = os.path.splitext(to_file) if not extension: extension = 'png' else: extension = extension[1:] # Save image to disk. dot.write(to_file, format=extension) # Return the image as a Jupyter Image object, to be displayed in-line. # Note that we cannot easily detect whether the code is running in a # notebook, and thus we always return the Image if Jupyter is available. if extension != 'pdf': try: from IPython import display return display.Image(filename=to_file) except ImportError: pass
def get_file(fname=None, origin=None, untar=False, md5_hash=None, file_hash=None, cache_subdir='datasets', hash_algorithm='auto', extract=False, archive_format='auto', cache_dir=None): """Downloads a file from a URL if it not already in the cache. By default the file at the url `origin` is downloaded to the cache_dir `~/.keras`, placed in the cache_subdir `datasets`, and given the filename `fname`. The final location of a file `example.txt` would therefore be `~/.keras/datasets/example.txt`. Files in tar, tar.gz, tar.bz, and zip formats can also be extracted. Passing a hash will verify the file after download. The command line programs `shasum` and `sha256sum` can compute the hash. Example: ```python path_to_downloaded_file = tf.keras.utils.get_file( "flower_photos", "https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz", untar=True) ``` Args: fname: Name of the file. If an absolute path `/path/to/file.txt` is specified the file will be saved at that location. If `None`, the name of the file at `origin` will be used. origin: Original URL of the file. untar: Deprecated in favor of `extract` argument. boolean, whether the file should be decompressed md5_hash: Deprecated in favor of `file_hash` argument. md5 hash of the file for verification file_hash: The expected hash string of the file after download. The sha256 and md5 hash algorithms are both supported. cache_subdir: Subdirectory under the Keras cache dir where the file is saved. If an absolute path `/path/to/folder` is specified the file will be saved at that location. hash_algorithm: Select the hash algorithm to verify the file. options are `'md5'`, `'sha256'`, and `'auto'`. The default 'auto' detects the hash algorithm in use. extract: True tries extracting the file as an Archive, like tar or zip. archive_format: Archive format to try for extracting the file. Options are `'auto'`, `'tar'`, `'zip'`, and `None`. `'tar'` includes tar, tar.gz, and tar.bz files. The default `'auto'` corresponds to `['tar', 'zip']`. None or an empty list will return no matches found. cache_dir: Location to store cached files, when None it defaults to the default directory `~/.keras/`. Returns: Path to the downloaded file """ if origin is None: raise ValueError( 'Please specify the "origin" argument (URL of the file ' 'to download).') if cache_dir is None: cache_dir = os.path.join(os.path.expanduser('~'), '.keras') if md5_hash is not None and file_hash is None: file_hash = md5_hash hash_algorithm = 'md5' datadir_base = os.path.expanduser(cache_dir) if not os.access(datadir_base, os.W_OK): datadir_base = os.path.join('/tmp', '.keras') datadir = os.path.join(datadir_base, cache_subdir) _makedirs_exist_ok(datadir) fname = io_utils.path_to_string(fname) if not fname: fname = os.path.basename(urlsplit(origin).path) if not fname: raise ValueError( f"Can't parse the file name from the origin provided: '{origin}'." "Please specify the `fname` as the input param.") if untar: if fname.endswith('.tar.gz'): fname = pathlib.Path(fname) # The 2 `.with_suffix()` are because of `.tar.gz` as pathlib # considers it as 2 suffixes. fname = fname.with_suffix('').with_suffix('') fname = str(fname) untar_fpath = os.path.join(datadir, fname) fpath = untar_fpath + '.tar.gz' else: fpath = os.path.join(datadir, fname) download = False if os.path.exists(fpath): # File found; verify integrity if a hash was provided. if file_hash is not None: if not validate_file(fpath, file_hash, algorithm=hash_algorithm): io_utils.print_msg( 'A local file was found, but it seems to be ' f'incomplete or outdated because the {hash_algorithm} ' f'file hash does not match the original value of {file_hash} ' 'so we will re-download the data.') download = True else: download = True if download: io_utils.print_msg(f'Downloading data from {origin}') class DLProgbar: """Manage progress bar state for use in urlretrieve.""" def __init__(self): self.progbar = None self.finished = False def __call__(self, block_num, block_size, total_size): if not self.progbar: if total_size == -1: total_size = None self.progbar = Progbar(total_size) current = block_num * block_size if current < total_size: self.progbar.update(current) elif not self.finished: self.progbar.update(self.progbar.target) self.finished = True error_msg = 'URL fetch failure on {}: {} -- {}' try: try: urlretrieve(origin, fpath, DLProgbar()) except urllib.error.HTTPError as e: raise Exception(error_msg.format(origin, e.code, e.msg)) except urllib.error.URLError as e: raise Exception(error_msg.format(origin, e.errno, e.reason)) except (Exception, KeyboardInterrupt) as e: if os.path.exists(fpath): os.remove(fpath) raise # Validate download if succeeded and user provided an expected hash # Security conscious users would get the hash of the file from a separate # channel and pass it to this API to prevent MITM / corruption: if os.path.exists(fpath) and file_hash is not None: if not validate_file(fpath, file_hash, algorithm=hash_algorithm): raise ValueError( f'Incomplete or corrupted file detected. The {hash_algorithm} ' f'file hash does not match the provided value of {file_hash}.' ) if untar: if not os.path.exists(untar_fpath): _extract_archive(fpath, datadir, archive_format='tar') return untar_fpath if extract: _extract_archive(fpath, datadir, archive_format) return fpath
def model_to_dot(model, show_shapes=False, show_dtype=False, show_layer_names=True, rankdir='TB', expand_nested=False, dpi=96, subgraph=False, layer_range=None, show_layer_activations=False): """Convert a Keras model to dot format. Args: model: A Keras model instance. show_shapes: whether to display shape information. show_dtype: whether to display layer dtypes. show_layer_names: whether to display layer names. rankdir: `rankdir` argument passed to PyDot, a string specifying the format of the plot: 'TB' creates a vertical plot; 'LR' creates a horizontal plot. expand_nested: whether to expand nested models into clusters. dpi: Dots per inch. subgraph: whether to return a `pydot.Cluster` instance. layer_range: input of `list` containing two `str` items, which is the starting layer name and ending layer name (both inclusive) indicating the range of layers for which the `pydot.Dot` will be generated. It also accepts regex patterns instead of exact name. In such case, start predicate will be the first element it matches to `layer_range[0]` and the end predicate will be the last element it matches to `layer_range[1]`. By default `None` which considers all layers of model. Note that you must pass range such that the resultant subgraph must be complete. show_layer_activations: Display layer activations (only for layers that have an `activation` property). Returns: A `pydot.Dot` instance representing the Keras model or a `pydot.Cluster` instance representing nested model if `subgraph=True`. Raises: ValueError: if `model_to_dot` is called before the model is built. ImportError: if graphviz or pydot are not available. """ if not model.built: raise ValueError( 'This model has not yet been built. ' 'Build the model first by calling `build()` or by calling ' 'the model on a batch of data.') from keras.layers import wrappers from keras.engine import sequential from keras.engine import functional if not check_pydot(): message = ( 'You must install pydot (`pip install pydot`) ' 'and install graphviz ' '(see instructions at https://graphviz.gitlab.io/download/) ' 'for plot_model/model_to_dot to work.') if 'IPython.core.magics.namespace' in sys.modules: # We don't raise an exception here in order to avoid crashing notebook # tests where graphviz is not available. io_utils.print_msg(message) return else: raise ImportError(message) if subgraph: dot = pydot.Cluster(style='dashed', graph_name=model.name) dot.set('label', model.name) dot.set('labeljust', 'l') else: dot = pydot.Dot() dot.set('rankdir', rankdir) dot.set('concentrate', True) dot.set('dpi', dpi) dot.set_node_defaults(shape='record') if layer_range is not None: if len(layer_range) != 2: raise ValueError( 'layer_range must be of shape (2,). Received: ' f'layer_range = {layer_range} of length {len(layer_range)}') if (not isinstance(layer_range[0], str) or not isinstance(layer_range[1], str)): raise ValueError('layer_range should contain string type only. ' f'Received: {layer_range}') layer_range = get_layer_index_bound_by_layer_name(model, layer_range) if layer_range[0] < 0 or layer_range[1] > len(model.layers): raise ValueError( 'Both values in layer_range should be in range (0, ' f'{len(model.layers)}. Received: {layer_range}') sub_n_first_node = {} sub_n_last_node = {} sub_w_first_node = {} sub_w_last_node = {} layers = model.layers if not model._is_graph_network: node = pydot.Node(str(id(model)), label=model.name) dot.add_node(node) return dot elif isinstance(model, sequential.Sequential): if not model.built: model.build() layers = super(sequential.Sequential, model).layers # Create graph nodes. for i, layer in enumerate(layers): if (layer_range) and (i < layer_range[0] or i > layer_range[1]): continue layer_id = str(id(layer)) # Append a wrapped layer's label to node's label, if it exists. layer_name = layer.name class_name = layer.__class__.__name__ if isinstance(layer, wrappers.Wrapper): if expand_nested and isinstance(layer.layer, functional.Functional): submodel_wrapper = model_to_dot(layer.layer, show_shapes, show_dtype, show_layer_names, rankdir, expand_nested, subgraph=True) # sub_w : submodel_wrapper sub_w_nodes = submodel_wrapper.get_nodes() sub_w_first_node[layer.layer.name] = sub_w_nodes[0] sub_w_last_node[layer.layer.name] = sub_w_nodes[-1] dot.add_subgraph(submodel_wrapper) else: layer_name = '{}({})'.format(layer_name, layer.layer.name) child_class_name = layer.layer.__class__.__name__ class_name = '{}({})'.format(class_name, child_class_name) if expand_nested and isinstance(layer, functional.Functional): submodel_not_wrapper = model_to_dot(layer, show_shapes, show_dtype, show_layer_names, rankdir, expand_nested, subgraph=True) # sub_n : submodel_not_wrapper sub_n_nodes = submodel_not_wrapper.get_nodes() sub_n_first_node[layer.name] = sub_n_nodes[0] sub_n_last_node[layer.name] = sub_n_nodes[-1] dot.add_subgraph(submodel_not_wrapper) # Create node's label. label = class_name # Rebuild the label as a table including the layer's activation. if (show_layer_activations and hasattr(layer, 'activation') and layer.activation is not None): label = '{%s|%s}' % (label, activations.serialize( layer.activation)) # Rebuild the label as a table including the layer's name. if show_layer_names: label = '%s|%s' % (layer_name, label) # Rebuild the label as a table including the layer's dtype. if show_dtype: def format_dtype(dtype): if dtype is None: return '?' else: return str(dtype) label = '%s|%s' % (label, format_dtype(layer.dtype)) # Rebuild the label as a table including input/output shapes. if show_shapes: def format_shape(shape): return str(shape).replace(str(None), 'None') try: outputlabels = format_shape(layer.output_shape) except AttributeError: outputlabels = '?' if hasattr(layer, 'input_shape'): inputlabels = format_shape(layer.input_shape) elif hasattr(layer, 'input_shapes'): inputlabels = ', '.join( [format_shape(ishape) for ishape in layer.input_shapes]) else: inputlabels = '?' label = '{%s}|{input:|output:}|{{%s}}|{{%s}}' % ( label, inputlabels, outputlabels) if not expand_nested or not isinstance(layer, functional.Functional): node = pydot.Node(layer_id, label=label) dot.add_node(node) # Connect nodes with edges. for i, layer in enumerate(layers): if (layer_range) and (i <= layer_range[0] or i > layer_range[1]): continue layer_id = str(id(layer)) for i, node in enumerate(layer._inbound_nodes): node_key = layer.name + '_ib-' + str(i) if node_key in model._network_nodes: for inbound_layer in tf.nest.flatten(node.inbound_layers): inbound_layer_id = str(id(inbound_layer)) if not expand_nested: assert dot.get_node(inbound_layer_id) assert dot.get_node(layer_id) add_edge(dot, inbound_layer_id, layer_id) else: # if inbound_layer is not Model or wrapped Model if (not isinstance(inbound_layer, functional.Functional) and not is_wrapped_model(inbound_layer)): # if current layer is not Model or wrapped Model if (not isinstance(layer, functional.Functional) and not is_wrapped_model(layer)): assert dot.get_node(inbound_layer_id) assert dot.get_node(layer_id) add_edge(dot, inbound_layer_id, layer_id) # if current layer is Model elif isinstance(layer, functional.Functional): add_edge( dot, inbound_layer_id, sub_n_first_node[layer.name].get_name()) # if current layer is wrapped Model elif is_wrapped_model(layer): add_edge(dot, inbound_layer_id, layer_id) name = sub_w_first_node[ layer.layer.name].get_name() add_edge(dot, layer_id, name) # if inbound_layer is Model elif isinstance(inbound_layer, functional.Functional): name = sub_n_last_node[ inbound_layer.name].get_name() if isinstance(layer, functional.Functional): output_name = sub_n_first_node[ layer.name].get_name() add_edge(dot, name, output_name) else: add_edge(dot, name, layer_id) # if inbound_layer is wrapped Model elif is_wrapped_model(inbound_layer): inbound_layer_name = inbound_layer.layer.name add_edge( dot, sub_w_last_node[inbound_layer_name].get_name(), layer_id) return dot
def update(self, current, values=None, finalize=None): """Updates the progress bar. Args: current: Index of current step. values: List of tuples: `(name, value_for_last_step)`. If `name` is in `stateful_metrics`, `value_for_last_step` will be displayed as-is. Else, an average of the metric over time will be displayed. finalize: Whether this is the last update for the progress bar. If `None`, defaults to `current >= self.target`. """ if finalize is None: if self.target is None: finalize = False else: finalize = current >= self.target values = values or [] for k, v in values: if k not in self._values_order: self._values_order.append(k) if k not in self.stateful_metrics: # In the case that progress bar doesn't have a target value in # the first epoch, both on_batch_end and on_epoch_end will be # called, which will cause 'current' and 'self._seen_so_far' to # have the same value. Force the minimal value to 1 here, # otherwise stateful_metric will be 0s. value_base = max(current - self._seen_so_far, 1) if k not in self._values: self._values[k] = [v * value_base, value_base] else: self._values[k][0] += v * value_base self._values[k][1] += value_base else: # Stateful metrics output a numeric value. This representation # means "take an average from a single value" but keeps the # numeric formatting. self._values[k] = [v, 1] self._seen_so_far = current message = "" now = time.time() info = " - %.0fs" % (now - self._start) if current == self.target: self._time_at_epoch_end = now if self.verbose == 1: if now - self._last_update < self.interval and not finalize: return prev_total_width = self._total_width if self._dynamic_display: message += "\b" * prev_total_width message += "\r" else: message += "\n" if self.target is not None: numdigits = int(np.log10(self.target)) + 1 bar = ("%" + str(numdigits) + "d/%d [") % (current, self.target) prog = float(current) / self.target prog_width = int(self.width * prog) if prog_width > 0: bar += "=" * (prog_width - 1) if current < self.target: bar += ">" else: bar += "=" bar += "." * (self.width - prog_width) bar += "]" else: bar = "%7d/Unknown" % current self._total_width = len(bar) message += bar time_per_unit = self._estimate_step_duration(current, now) if self.target is None or finalize: info += self._format_time(time_per_unit, self.unit_name) else: eta = time_per_unit * (self.target - current) if eta > 3600: eta_format = "%d:%02d:%02d" % ( eta // 3600, (eta % 3600) // 60, eta % 60, ) elif eta > 60: eta_format = "%d:%02d" % (eta // 60, eta % 60) else: eta_format = "%ds" % eta info = " - ETA: %s" % eta_format for k in self._values_order: info += " - %s:" % k if isinstance(self._values[k], list): avg = np.mean(self._values[k][0] / max(1, self._values[k][1])) if abs(avg) > 1e-3: info += " %.4f" % avg else: info += " %.4e" % avg else: info += " %s" % self._values[k] self._total_width += len(info) if prev_total_width > self._total_width: info += " " * (prev_total_width - self._total_width) if finalize: info += "\n" message += info io_utils.print_msg(message, line_break=False) message = "" elif self.verbose == 2: if finalize: numdigits = int(np.log10(self.target)) + 1 count = ("%" + str(numdigits) + "d/%d") % (current, self.target) info = count + info for k in self._values_order: info += " - %s:" % k avg = np.mean(self._values[k][0] / max(1, self._values[k][1])) if avg > 1e-3: info += " %.4f" % avg else: info += " %.4e" % avg if self._time_at_epoch_end: time_per_epoch = (self._time_at_epoch_end - self._time_at_epoch_start) avg_time_per_step = time_per_epoch / self.target self._time_at_epoch_start = now self._time_at_epoch_end = None info += " -" + self._format_time(time_per_epoch, "epoch") info += " -" + self._format_time(avg_time_per_step, self.unit_name) info += "\n" message += info io_utils.print_msg(message, line_break=False) message = "" self._last_update = now
def write_ckpt_to_h5(path_h5, path_ckpt, keras_model, use_ema=True): """Map the weights in checkpoint file (tf) to h5 file (keras). Args: path_h5: str, path to output hdf5 file to write weights loaded from ckpt files. path_ckpt: str, path to the ckpt files (e.g. 'efficientnet-b0/model.ckpt') that records efficientnet weights from original repo https://github.com/tensorflow/tpu/tree/master/models/official/efficientnet keras_model: keras model, built from keras.applications efficientnet functions (e.g. EfficientNetB0) use_ema: Bool, whether to use ExponentialMovingAverage result or not """ model_name_keras = keras_model.name model_name_tf = model_name_keras.replace('efficientnet', 'efficientnet-') keras_weight_names = [w.name for w in keras_model.weights] tf_weight_names = get_variable_names_from_ckpt(path_ckpt) keras_blocks = get_keras_blocks(keras_weight_names) tf_blocks = get_tf_blocks(tf_weight_names) io_utils.print_msg('check variables match in each block') for keras_block, tf_block in zip(keras_blocks, tf_blocks): check_match(keras_block, tf_block, keras_weight_names, tf_weight_names, model_name_tf) io_utils.print_msg('{} and {} match.'.format(tf_block, keras_block)) block_mapping = {x[0]: x[1] for x in zip(keras_blocks, tf_blocks)} changed_weights = 0 for w in keras_model.weights: if 'block' in w.name: # example: 'block1a_dwconv/depthwise_kernel:0' -> 'block1a' keras_block = w.name.split('/')[0].split('_')[0] tf_block = block_mapping[keras_block] tf_name = keras_name_to_tf_name_block( w.name, keras_block=keras_block, tf_block=tf_block, use_ema=use_ema, model_name_tf=model_name_tf) elif any([x in w.name for x in ['stem', 'top', 'predictions', 'probs']]): tf_name = keras_name_to_tf_name_stem_top( w.name, use_ema=use_ema, model_name_tf=model_name_tf) elif 'normalization' in w.name: io_utils.print_msg( f'Skipping variable {w.name}: normalization is a Keras ' 'preprocessing layer, which does not exist in the TF ckpt.') continue else: raise ValueError('{} failed to parse.'.format(w.name)) try: w_tf = tf.train.load_variable(path_ckpt, tf_name) if (w.value().numpy() != w_tf).any(): w.assign(w_tf) changed_weights += 1 except ValueError as e: if any([x in w.name for x in ['top', 'predictions', 'probs']]): warnings.warn( 'Fail to load top layer variable {}' 'from {} because of {}.'.format(w.name, tf_name, e), stacklevel=2) else: raise ValueError('Fail to load {} from {}'.format(w.name, tf_name)) total_weights = len(keras_model.weights) io_utils.print_msg(f'{changed_weights}/{total_weights} weights updated') keras_model.save_weights(path_h5)