def rotate_concurrent(self, *locations, **kw):
"""
Rotate the backups in the given locations concurrently.
:param locations: One or more values accepted by :func:`coerce_location()`.
:param kw: Any keyword arguments are passed on to :func:`rotate_backups()`.
This function uses :func:`rotate_backups()` to prepare rotation
commands for the given locations and then it removes backups in
parallel, one backup per mount point at a time.
The idea behind this approach is that parallel rotation is most useful
when the files to be removed are on different disks and so multiple
devices can be utilized at the same time.
Because mount points are per system :func:`rotate_concurrent()` will
also parallelize over backups located on multiple remote systems.
"""
timer = Timer()
pool = CommandPool(concurrency=10)
logger.info("Scanning %s ..", pluralize(len(locations), "backup location"))
for location in locations:
for cmd in self.rotate_backups(location, prepare=True, **kw):
pool.add(cmd)
if pool.num_commands > 0:
backups = pluralize(pool.num_commands, "backup")
logger.info("Preparing to rotate %s (in parallel) ..", backups)
pool.run()
logger.info("Successfully rotated %s in %s.", backups, timer)
class CommandAgent:
def __init__(self, concurrency=8, show_result=False):
self.pool = CommandPool(concurrency=concurrency)
self.cmd_records = {}
self.show_result = show_result
self.logger = logging.getLogger('.'.join(
[__name__, self.__class__.__name__]))
def __enter__(self):
return self
def __exit__(self, exception_type, exception_value, traceback):
self.pool.run()
if not self.show_result:
return
i = 0
for (identifier, cmd) in self.cmd_records.items():
i = i + 1
self.logger.debug("[{}] {}".format(
i,
cmd.ssh_alias if type(cmd) is RemoteCommand else identifier))
self.logger.debug(cmd.output)
def submit(self, cmd_id, cmd, *args, **kwargs):
ec = None
if type(cmd) is ExternalCommand or type(cmd) is RemoteCommand:
ec = cmd
else:
ec = ExternalCommand(cmd, *args, **kwargs)
ec. async = True
self.cmd_records[cmd_id] = ec
self.pool.add(ec)
def submit_command(self, cmd, *args, **kwargs):
self.submit(hash(cmd), ExternalCommand(cmd, *args, **kwargs))
def submit_remote_command(self, host, cmd, *args, cid=None, **kwargs):
if 'strict_host_key_checking' not in kwargs:
kwargs['strict_host_key_checking'] = False
if 'ignore_known_hosts' not in kwargs:
kwargs['ignore_known_hosts'] = True
host_used = host if isinstance(
host,
str) else host.get_ip() if isinstance(host, base.Node) else None
if host_used is None:
raise Exception("unknown host type: {}".format(type(host)))
if cid is None:
self.submit(hash(host_used + cmd),
RemoteCommand(host_used, cmd, *args, **kwargs))
else:
self.submit(cid, RemoteCommand(host_used, cmd, *args, **kwargs))
def submit_remote_commands(self, nodes, cmd, *args, cids=None, **kwargs):
if cids is not None:
for node, cid in zip(nodes, cids):
self.submit_remote_command(node, cmd, *args, cid=cid, **kwargs)
else:
for node in nodes:
self.submit_remote_command(node, cmd, *args, **kwargs)
def rotate_concurrent(self, *locations, **kw):
"""
Rotate the backups in the given locations concurrently.
:param locations: One or more values accepted by :func:`coerce_location()`.
:param kw: Any keyword arguments are passed on to :func:`rotate_backups()`.
This function uses :func:`rotate_backups()` to prepare rotation
commands for the given locations and then it removes backups in
parallel, one backup per mount point at a time.
The idea behind this approach is that parallel rotation is most useful
when the files to be removed are on different disks and so multiple
devices can be utilized at the same time.
Because mount points are per system :func:`rotate_concurrent()` will
also parallelize over backups located on multiple remote systems.
"""
timer = Timer()
pool = CommandPool(concurrency=10)
logger.info("Scanning %s ..",
pluralize(len(locations), "backup location"))
for location in locations:
for cmd in self.rotate_backups(location, prepare=True, **kw):
pool.add(cmd)
if pool.num_commands > 0:
backups = pluralize(pool.num_commands, "backup")
logger.info("Preparing to rotate %s (in parallel) ..", backups)
pool.run()
logger.info("Successfully rotated %s in %s.", backups, timer)
class CommandAgent:
def __init__(self, concurrency=8, show_result=True):
self.pool = CommandPool(concurrency=concurrency)
self.cmd_records = {}
self.show_result = show_result
self.logger = logging.getLogger('.'.join(
[__name__, self.__class__.__name__]))
def __enter__(self):
return self
def __exit__(self, exception_type, exception_value, traceback):
self.pool.run()
if not self.show_result:
return
i = 0
for (identifier, cmd) in self.cmd_records.items():
i = i + 1
print("[{}] {}".format(
i,
cmd.ssh_alias if type(cmd) is RemoteCommand else identifier))
print(cmd.output)
def submit(self, cmd_id, cmd, *args, **kwargs):
ec = None
if type(cmd) is ExternalCommand or type(cmd) is RemoteCommand:
ec = cmd
else:
ec = ExternalCommand(cmd, *args, **kwargs)
ec. async = True
self.cmd_records[cmd_id] = ec
self.pool.add(ec)
def submit_command(self, cmd, *args, **kwargs):
self.submit(hash(cmd), ExternalCommand(cmd, *args, **kwargs))
def submit_remote_command(self, host, cmd, *args, **kwargs):
if 'strict_host_key_checking' not in kwargs:
kwargs['strict_host_key_checking'] = False
if 'ignore_known_hosts' not in kwargs:
kwargs['ignore_known_hosts'] = True
self.submit(hash(host + cmd), RemoteCommand(host, cmd, *args,
**kwargs))
def submit_remote_commands(self, nodes, cmd, *args, **kwargs):
for node in nodes:
self.submit_remote_command(node, cmd, *args, **kwargs)
def run(self):
self.pool.run()
def results(self):
return {k: v.output for (k, v) in self.cmd_records.items()}
def status(self):
return {k: v.succeeded for (k, v) in self.cmd_records.items()}
def __init__(self, concurrency=8, show_result=False):
self.pool = CommandPool(concurrency=concurrency)
self.cmd_records = {}
self.show_result = show_result
self.logger = logging.getLogger('.'.join(
[__name__, self.__class__.__name__]))