def _download_file(command_template, remote_loc, local_path):
with atomic_output_file(local_path, make_parents=True) as temp_target:
popenargs = shell_expand_to_popen(command_template,
dict_merge(os.environ, {"REMOTE": remote_loc, "LOCAL": temp_target}))
log.info("downloading: %s", " ".join(popenargs))
# TODO: Find a way to support force here.
subprocess.check_call(popenargs, stdout=SHELL_OUTPUT, stderr=SHELL_OUTPUT, stdin=DEV_NULL)
def _decompress_dir(archive_path, target_path, force=False):
if os.path.exists(target_path):
if force:
log.info("deleting previous dir: %s", target_path)
_rmtree_fast(target_path)
else:
raise AppError("Target already exists: %r" % target_path)
with atomic_output_file(target_path) as temp_dir:
make_all_dirs(temp_dir)
ARCHIVER.unarchive(archive_path, temp_dir)
def _decompress_dir(archive_path, target_path, force=False):
if os.path.exists(target_path):
if force:
log.info("deleting previous dir: %s", target_path)
rmtree_or_file(target_path)
else:
raise AppError("Target already exists: %r" % target_path)
with atomic_output_file(target_path) as temp_dir:
make_all_dirs(temp_dir)
ARCHIVER.unarchive(archive_path, temp_dir)
def _compress_dir(local_dir, archive_path, force=False):
if os.path.exists(archive_path):
if force:
log.info("deleting previous archive: %s", archive_path)
os.unlink(archive_path)
else:
raise AppError("Archive already in cache (has version changed?): %r" % archive_path)
with atomic_output_file(archive_path) as temp_archive:
make_parent_dirs(temp_archive)
ARCHIVER.archive(local_dir, temp_archive)
def merge_files(target_filename,
*yaml_filenames,
strict_base=True,
make_parents=True,
output_format='yaml'):
"""
Merge and write out a unified config file to target location, with settings from later files overriding
those in earlier files, and output in consistent key order (according to last occurrence).
"""
with atomic_output_file(target_filename,
make_parents=make_parents) as temp_out:
with open(temp_out, "w") as out:
target_path = Path(target_filename)
if target_path.suffix == '.json':
output_format = 'json'
elif target_path.suffix == '.yml':
output_format = 'yaml'
merge_files_to_stream(out,
*yaml_filenames,
strict_base=strict_base,
output_format=output_format)