def _create_temp_structure(temp_directory, structure):
for folder, files in structure.items():
current_directory = os.path.join(temp_directory, folder)
os.makedirs(current_directory)
for name, content in files.items():
path = os.path.join(temp_directory, folder, name)
with salt.utils.files.fopen(path, "w+") as fh:
fh.write(content)
def _create_temp_structure(self, temp_directory, structure):
for folder, files in structure.items():
current_directory = os.path.join(temp_directory, folder)
os.makedirs(current_directory)
for name, content in files.items():
path = os.path.join(temp_directory, folder, name)
with open(path, "w+") as fh:
fh.write(content)
def run_chunked(self):
"""
Make the salt client call in the new fasion chunked multi-call way
"""
files, empty_dirs = self._list_files()
dest = self.opts["dest"]
gzip = self.opts["gzip"]
tgt = self.opts["tgt"]
timeout = self.opts["timeout"]
selected_target_option = self.opts.get("selected_target_option")
dest_is_dir = (bool(empty_dirs) or len(files) > 1
or bool(re.search(r"[\\/]$", dest)))
reader = (salt.utils.gzip_util.compress_file
if gzip else salt.utils.itertools.read_file)
_res = salt.utils.minions.CkMinions(self.opts).check_minions(
tgt, tgt_type=selected_target_option or "glob")
minions = _res["minions"]
def _get_remote_path(fn_):
if fn_ in self.opts["src"]:
# This was a filename explicitly passed on the CLI
return (os.path.join(dest, os.path.basename(fn_))
if dest_is_dir else dest)
else:
for path in self.opts["src"]:
relpath = os.path.relpath(fn_, path + os.sep)
if relpath.startswith(parent):
# File is not within this dir
continue
return os.path.join(dest, os.path.basename(path), relpath)
else: # pylint: disable=useless-else-on-loop
# Should not happen
log.error("Failed to find remote path for %s", fn_)
return None
ret = {}
parent = ".." + os.sep
with salt.client.get_local_client(self.opts["conf_file"]) as local:
for fn_, mode in files.items():
remote_path = _get_remote_path(fn_)
index = 1
failed = {}
for chunk in reader(
fn_, chunk_size=self.opts["salt_cp_chunk_size"]):
chunk = base64.b64encode(
salt.utils.stringutils.to_bytes(chunk))
append = index > 1
log.debug(
"Copying %s to %starget '%s' as %s%s",
fn_,
"{} ".format(selected_target_option)
if selected_target_option else "",
tgt,
remote_path,
" (chunk #{})".format(index) if append else "",
)
args = [
tgt,
"cp.recv_chunked",
[remote_path, chunk, append, gzip, mode],
timeout,
]
if selected_target_option is not None:
args.append(selected_target_option)
result = local.cmd(*args)
if not result:
# Publish failed
msg = ("Publish failed.{} It may be necessary to "
"decrease salt_cp_chunk_size (current value: "
"{})".format(
" File partially transferred."
if index > 1 else "",
self.opts["salt_cp_chunk_size"],
))
for minion in minions:
ret.setdefault(minion, {})[remote_path] = msg
break
for minion_id, minion_ret in result.items():
ret.setdefault(minion_id, {})[remote_path] = minion_ret
# Catch first error message for a given minion, we will
# rewrite the results after we're done iterating through
# the chunks.
if minion_ret is not True and minion_id not in failed:
failed[minion_id] = minion_ret
index += 1
for minion_id, msg in failed.items():
ret[minion_id][remote_path] = msg
for dirname in empty_dirs:
remote_path = _get_remote_path(dirname)
log.debug(
"Creating empty dir %s on %starget '%s'",
dirname,
"{} ".format(selected_target_option) # pylint: disable=str-format-in-logging
if selected_target_option else "",
tgt,
)
args = [tgt, "cp.recv_chunked", [remote_path, None], timeout]
if selected_target_option is not None:
args.append(selected_target_option)
for minion_id, minion_ret in local.cmd(*args).items():
ret.setdefault(minion_id, {})[remote_path] = minion_ret
return ret
def _validate_folder_structure_and_contents(self, target_directory, desired_structure):
for folder, files in desired_structure.items():
for name, content in files.items():
path = os.path.join(target_directory, folder, name)
with open(path) as fh:
assert fh.read().strip() == content