def get_size(self):
# "stat --format %s" returns the size of the file in bytes
cmd_args = self._connection_tokens + [
"stat", "--format", "%s", self.filename
]
cmd = get_command_from_tokens(cmd_args)
log.debug("Retrieving the size of the patch: %s", cmd)
return int(cmd())
def exists(self):
cmd_args = self._connection_tokens + ["[", "-e", self.filename, "]"]
cmd = get_command_from_tokens(cmd_args)
log.debug("Testing if target exists: %s", cmd)
retcode = cmd.run(retcode=(0, 1))[0]
log.debug("Target %s", "exists" if
(retcode == 0) else "does not exist")
return retcode == 0
def get_store_command(self):
if self._connection_tokens:
remote_cmd_string = "cat > {0}".format(shlex.quote(self.filename))
store_cmd_args = self._connection_tokens + [remote_cmd_string]
store_cmd = get_command_from_tokens(store_cmd_args)
else:
store_cmd_args = ["cat"]
store_cmd = get_command_from_tokens(store_cmd_args) > self.filename
log.debug("Storing patch file: %s", store_cmd)
return store_cmd
def _create_snapshot(self, snapshot_name, snapshot_size):
assert self._snapshot_name is None
log.info("Creating LVM snapshot: %s/%s", self._group, snapshot_name)
cmd = self._caller["lvcreate", "--snapshot", "--name", snapshot_name,
"--size", snapshot_size, self._get_path()]
log.debug("LVM snapshot create command: %s", cmd)
try:
cmd()
except plumbum.commands.processes.ProcessExecutionError as exc:
raise TaskProcessingError("Failed to create LVM snapshot: {0}".format(exc))
self._snapshot_name = snapshot_name
def bdsync_run(source_filename, target_filename, connection_command,
local_bdsync, remote_bdsync, bdsync_args, target_patch_dir,
create_if_missing, apply_in_place, bandwidth_limit):
""" Run bdsync in one or two phases. With one phase changes are applied in-place.
With two phases there are separate stages of patch generation / transfer followed
by the application of the patch.
"""
source = SyncSource(source_filename, local_bdsync, bdsync_args)
target = SyncTarget(target_filename,
remote_bdsync if connection_command else local_bdsync,
bdsync_args, connection_command)
# preparations
if not target.exists():
if create_if_missing:
log.warning("Creating missing target file: %s", target.filename)
target.create_empty()
else:
raise NotFoundError(
"The target does not exist (while 'create_target_if_missing' "
"is disabled)")
generate_patch_cmd = source.get_generate_patch_command(target)
if bandwidth_limit:
generate_patch_cmd = generate_patch_cmd | get_command_from_tokens(
["pv", "--rate-limit",
str(bandwidth_limit), "--quiet"])
if apply_in_place:
start_time = time.time()
operation = generate_patch_cmd | target.get_apply_patch_command()
log.debug("Applying changes in-place: %s", operation)
operation()
change_apply_time = datetime.timedelta(seconds=(time.time() -
start_time))
log.info("Change Apply Time: %s", change_apply_time)
else:
patch = SyncPatch(target_patch_dir, connection_command)
# generate and store the patch
start_time = time.time()
generate_patch_command = generate_patch_cmd | patch.get_store_command()
log.debug("Generating Patch: %s", generate_patch_command)
generate_patch_command()
patch_generate_time = datetime.timedelta(seconds=(time.time() -
start_time))
log.info("Patch Generate Time: %s", patch_generate_time)
log.info("Patch Size: %s", sizeof_fmt(patch.get_size()))
# apply the new patch
start_time = time.time()
apply_patch_command = target.get_apply_patch_command(patch)
apply_patch_command()
patch_apply_time = datetime.timedelta(seconds=(time.time() -
start_time))
log.info("Patch Apply Time: %s", patch_apply_time)
patch.cleanup()
def _parse_volume_path(self, volume_path):
lvm_info_args = ("lvdisplay", "--columns", "--noheading", "--separator", ":",
"-o", "vg_name,lv_name", volume_path)
cmd = self._caller[lvm_info_args]
log.debug("Trying to parse LVM volume information: %s", cmd)
# remove left alignment and the linebreak
output = cmd().strip()
if output:
vg_name, lv_name = output.split(":")
log.debug("Parsed LVM volume information: %s/%s", vg_name, lv_name)
return vg_name, lv_name
else:
raise NotFoundError("Failed to find find given LVM volume: %s" % volume_path)
def __init__(self, filename):
log.debug("Reading config file: %s", filename)
self.config = configparser.ConfigParser()
try:
self.config.read([filename])
except configparser.Error as error:
raise TaskSettingsError(
"Failed to parse configuration ({0}): {1}".format(
filename, error))
self._tasks = {
key: TaskConfiguration(self.config[key])
for key in self.config.sections()
}
def remove_snapshot(self):
# the name should not be None or empty
assert self._snapshot_name
lv_path = "%s/%s" % (self._group, self._snapshot_name)
log.debug("Verifying status of snapshot volume before removing: %s", lv_path)
cmd = self._caller["lvdisplay", "--columns", "--noheading",
"--select", "origin=%s" % self._volume, lv_path]
if self._snapshot_name not in cmd():
log.error("Refusing to remove LVM snapshot due to its unclear state: %s", lv_path)
else:
log.info("Removing LVM snapshot: %s", lv_path)
cmd = self._caller["lvremove", "--force", lv_path]
log.debug("LVM snapshot remove command: %s", cmd)
cmd()
self._snapshot_name = None
def main():
try:
bdsync_manager.utils.verify_requirements()
except bdsync_manager.RequirementsError as error:
log.error(str(error))
return EXITCODE_MISSING_DEPENDENCY
log.debug("Parsing arguments")
args = parse_arguments()
try:
settings = bdsync_manager.config.Configuration(args.config_file.name)
except bdsync_manager.TaskSettingsError as error:
log.error(error)
return EXITCODE_CONFIGURATION_ERROR
if args.tasks:
tasks = []
for task in args.tasks:
if task in settings.tasks:
tasks.append(task)
else:
log.warning("Skipping unknown task: %s",
_get_safe_string(task))
else:
tasks = settings.tasks.keys()
if not tasks:
log.warning(
"There is nothing to be done (no tasks found in config file).")
processing_error = False
# late import: avoid import problems before dependency checks (see above)
from bdsync_manager.task import Task
for task_name in tasks:
task = Task(settings.tasks[task_name])
bdsync_manager.utils.set_log_format(
"[Task {0}] %(levelname)s: %(message)s".format(task_name))
try:
task.run()
except bdsync_manager.TaskProcessingError as error:
log.error(str(error))
processing_error = True
except KeyboardInterrupt:
log.error("Terminated via user input")
return EXITCODE_CANCELLED
if processing_error:
return EXITCODE_TASK_PROCESSING_ERROR
else:
return EXITCODE_SUCCESS
def cleanup(self):
# remove patch file
cmd_args = self._connection_tokens + ["rm", self.filename]
cmd = get_command_from_tokens(cmd_args)
log.debug("Removing temporary patch file: %s", cmd)
cmd()
def __init__(self, target_patch_dir, connection_command=None):
self._connection_tokens = shlex.split(connection_command or "")
self.filename = get_tempfile(target_patch_dir, self._connection_tokens)
log.debug("Using temporary patch file: %s", self.filename)
def create_empty(self):
cmd_args = self._connection_tokens + ["touch", self.filename]
cmd = get_command_from_tokens(cmd_args)
log.debug("Creating target: %s", cmd)
cmd()
