def garbage_collect_exports(export_dir_base, exports_to_keep): """Deletes older exports, retaining only a given number of the most recent. Export subdirectories are assumed to be named with monotonically increasing integers; the most recent are taken to be those with the largest values. Args: export_dir_base: the base directory under which each export is in a versioned subdirectory. exports_to_keep: the number of recent exports to retain. """ if exports_to_keep is None: return keep_filter = gc.largest_export_versions(exports_to_keep) delete_filter = gc.negation(keep_filter) # Export dir must not end with / or it will break the re match below. if export_dir_base.endswith('/'): export_dir_base = export_dir_base[:-1] # create a simple parser that pulls the export_version from the directory. def parser(path): match = re.match('^' + export_dir_base + '/(\\d{13})$', path.path) if not match: return None return path._replace(export_version=int(match.group(1))) for p in delete_filter(gc.get_paths(export_dir_base, parser=parser)): gfile.DeleteRecursively(p.path)
def garbage_collect_exports(export_dir_base, exports_to_keep): """Deletes older exports, retaining only a given number of the most recent. Export subdirectories are assumed to be named with monotonically increasing integers; the most recent are taken to be those with the largest values. Args: export_dir_base: the base directory under which each export is in a versioned subdirectory. exports_to_keep: the number of recent exports to retain. """ if exports_to_keep is None: return keep_filter = gc.largest_export_versions(exports_to_keep) delete_filter = gc.negation(keep_filter) # create a simple parser that pulls the export_version from the directory. def parser(path): filename = os.path.basename(path.path) if not (len(filename) == 10 and filename.isdigit()): return None return path._replace(export_version=int(filename)) for p in delete_filter(gc.get_paths(export_dir_base, parser=parser)): gfile.DeleteRecursively(p.path)
def testUnion(self): paths = [] for i in xrange(10): paths.append(gc.Path("/foo", i)) f = gc.union(gc.largest_export_versions(3), gc.mod_export_version(3)) self.assertEquals( f(paths), [ gc.Path("/foo", 0), gc.Path("/foo", 3), gc.Path("/foo", 6), gc.Path("/foo", 7), gc.Path("/foo", 8), gc.Path("/foo", 9) ])
def get_most_recent_export(export_dir_base): """Locate the most recent SavedModel export in a directory of many exports. This method assumes that SavedModel subdirectories are named as a timestamp (seconds from epoch), as produced by get_timestamped_export_dir(). Args: export_dir_base: A base directory containing multiple timestamped directories. Returns: A gc.Path, whith is just a namedtuple of (path, export_version). """ select_filter = gc.largest_export_versions(1) results = select_filter(gc.get_paths(export_dir_base, parser=_export_version_parser)) return next(iter(results or []), None)
def get_most_recent_export(export_dir_base): """Locate the most recent SavedModel export in a directory of many exports. This method assumes that SavedModel subdirectories are named as a timestamp (seconds from epoch), as produced by get_timestamped_export_dir(). Args: export_dir_base: A base directory containing multiple timestamped directories. Returns: A gc.Path, with is just a namedtuple of (path, export_version). """ select_filter = gc.largest_export_versions(1) results = select_filter( gc.get_paths(export_dir_base, parser=_export_version_parser)) return next(iter(results or []), None)
def garbage_collect_exports(export_dir_base, exports_to_keep): """Deletes older exports, retaining only a given number of the most recent. Export subdirectories are assumed to be named with monotonically increasing integers; the most recent are taken to be those with the largest values. Args: export_dir_base: the base directory under which each export is in a versioned subdirectory. exports_to_keep: the number of recent exports to retain. """ if exports_to_keep is None: return keep_filter = gc.largest_export_versions(exports_to_keep) delete_filter = gc.negation(keep_filter) for p in delete_filter(gc.get_paths(export_dir_base, parser=_export_version_parser)): gfile.DeleteRecursively(p.path)
def testLargestExportVersionsDoesNotDeleteZeroFolder(self): paths = [gc.Path("/foo", 0), gc.Path("/foo", 3)] newest = gc.largest_export_versions(2) n = newest(paths) self.assertEqual(n, [gc.Path("/foo", 0), gc.Path("/foo", 3)])
def testLargestExportVersions(self): paths = [gc.Path("/foo", 8), gc.Path("/foo", 9), gc.Path("/foo", 10)] newest = gc.largest_export_versions(2) n = newest(paths) self.assertEqual(n, [gc.Path("/foo", 9), gc.Path("/foo", 10)])