def rotate_concurrent(self, *locations, **kw): """ Rotate the backups in the given locations concurrently. :param locations: One or more values accepted by :func:`coerce_location()`. :param kw: Any keyword arguments are passed on to :func:`rotate_backups()`. This function uses :func:`rotate_backups()` to prepare rotation commands for the given locations and then it removes backups in parallel, one backup per mount point at a time. The idea behind this approach is that parallel rotation is most useful when the files to be removed are on different disks and so multiple devices can be utilized at the same time. Because mount points are per system :func:`rotate_concurrent()` will also parallelize over backups located on multiple remote systems. """ timer = Timer() pool = CommandPool(concurrency=10) logger.info("Scanning %s ..", pluralize(len(locations), "backup location")) for location in locations: for cmd in self.rotate_backups(location, prepare=True, **kw): pool.add(cmd) if pool.num_commands > 0: backups = pluralize(pool.num_commands, "backup") logger.info("Preparing to rotate %s (in parallel) ..", backups) pool.run() logger.info("Successfully rotated %s in %s.", backups, timer)
class CommandAgent: def __init__(self, concurrency=8, show_result=False): self.pool = CommandPool(concurrency=concurrency) self.cmd_records = {} self.show_result = show_result self.logger = logging.getLogger('.'.join( [__name__, self.__class__.__name__])) def __enter__(self): return self def __exit__(self, exception_type, exception_value, traceback): self.pool.run() if not self.show_result: return i = 0 for (identifier, cmd) in self.cmd_records.items(): i = i + 1 self.logger.debug("[{}] {}".format( i, cmd.ssh_alias if type(cmd) is RemoteCommand else identifier)) self.logger.debug(cmd.output) def submit(self, cmd_id, cmd, *args, **kwargs): ec = None if type(cmd) is ExternalCommand or type(cmd) is RemoteCommand: ec = cmd else: ec = ExternalCommand(cmd, *args, **kwargs) ec. async = True self.cmd_records[cmd_id] = ec self.pool.add(ec) def submit_command(self, cmd, *args, **kwargs): self.submit(hash(cmd), ExternalCommand(cmd, *args, **kwargs)) def submit_remote_command(self, host, cmd, *args, cid=None, **kwargs): if 'strict_host_key_checking' not in kwargs: kwargs['strict_host_key_checking'] = False if 'ignore_known_hosts' not in kwargs: kwargs['ignore_known_hosts'] = True host_used = host if isinstance( host, str) else host.get_ip() if isinstance(host, base.Node) else None if host_used is None: raise Exception("unknown host type: {}".format(type(host))) if cid is None: self.submit(hash(host_used + cmd), RemoteCommand(host_used, cmd, *args, **kwargs)) else: self.submit(cid, RemoteCommand(host_used, cmd, *args, **kwargs)) def submit_remote_commands(self, nodes, cmd, *args, cids=None, **kwargs): if cids is not None: for node, cid in zip(nodes, cids): self.submit_remote_command(node, cmd, *args, cid=cid, **kwargs) else: for node in nodes: self.submit_remote_command(node, cmd, *args, **kwargs)
def rotate_concurrent(self, *locations, **kw): """ Rotate the backups in the given locations concurrently. :param locations: One or more values accepted by :func:`coerce_location()`. :param kw: Any keyword arguments are passed on to :func:`rotate_backups()`. This function uses :func:`rotate_backups()` to prepare rotation commands for the given locations and then it removes backups in parallel, one backup per mount point at a time. The idea behind this approach is that parallel rotation is most useful when the files to be removed are on different disks and so multiple devices can be utilized at the same time. Because mount points are per system :func:`rotate_concurrent()` will also parallelize over backups located on multiple remote systems. """ timer = Timer() pool = CommandPool(concurrency=10) logger.info("Scanning %s ..", pluralize(len(locations), "backup location")) for location in locations: for cmd in self.rotate_backups(location, prepare=True, **kw): pool.add(cmd) if pool.num_commands > 0: backups = pluralize(pool.num_commands, "backup") logger.info("Preparing to rotate %s (in parallel) ..", backups) pool.run() logger.info("Successfully rotated %s in %s.", backups, timer)
class CommandAgent: def __init__(self, concurrency=8, show_result=True): self.pool = CommandPool(concurrency=concurrency) self.cmd_records = {} self.show_result = show_result self.logger = logging.getLogger('.'.join( [__name__, self.__class__.__name__])) def __enter__(self): return self def __exit__(self, exception_type, exception_value, traceback): self.pool.run() if not self.show_result: return i = 0 for (identifier, cmd) in self.cmd_records.items(): i = i + 1 print("[{}] {}".format( i, cmd.ssh_alias if type(cmd) is RemoteCommand else identifier)) print(cmd.output) def submit(self, cmd_id, cmd, *args, **kwargs): ec = None if type(cmd) is ExternalCommand or type(cmd) is RemoteCommand: ec = cmd else: ec = ExternalCommand(cmd, *args, **kwargs) ec. async = True self.cmd_records[cmd_id] = ec self.pool.add(ec) def submit_command(self, cmd, *args, **kwargs): self.submit(hash(cmd), ExternalCommand(cmd, *args, **kwargs)) def submit_remote_command(self, host, cmd, *args, **kwargs): if 'strict_host_key_checking' not in kwargs: kwargs['strict_host_key_checking'] = False if 'ignore_known_hosts' not in kwargs: kwargs['ignore_known_hosts'] = True self.submit(hash(host + cmd), RemoteCommand(host, cmd, *args, **kwargs)) def submit_remote_commands(self, nodes, cmd, *args, **kwargs): for node in nodes: self.submit_remote_command(node, cmd, *args, **kwargs) def run(self): self.pool.run() def results(self): return {k: v.output for (k, v) in self.cmd_records.items()} def status(self): return {k: v.succeeded for (k, v) in self.cmd_records.items()}
def __init__(self, concurrency=8, show_result=False): self.pool = CommandPool(concurrency=concurrency) self.cmd_records = {} self.show_result = show_result self.logger = logging.getLogger('.'.join( [__name__, self.__class__.__name__]))