def remote_head(self, remote_url: str) -> Mapping[str, str]:
logger.debug(f"http.head.fetch: {remote_url}")
header_path = make_temp_path("curl-head")
SUDO.parent_ensure(header_path)
curl = Curl()
with_auth_url(curl, remote_url)
with_proxy_config(curl, remote_url)
curl.with_file_head(header_path)
curl.execute_unit_sert()
header_dict = parse_header_file(header_path)
SUDO.files_delete(header_path)
return header_dict
def synchronize_header(source: str, target: str) -> None:
"transfer header metadata between local paths"
header_dict = dict() # never empty
try:
header_dict = SUDO.xattr_load(source)
except:
pass
if not header_dict:
header_dict = produce_header(source)
try:
SUDO.xattr_save(target, header_dict)
except:
SUDO.files_sync_time(source, target)
def perform_command(self) -> None:
"invoke command on the host"
self.logger.debug("perform_command")
for command in self.command_list:
self.logger.info(f"Command: {command}")
result = SUDO.execute_flow(command)
result.assert_return()
def smart_extract(self, archive_path: str, extract_path: str) -> None:
"make extract from archive, when needed"
if SUDO.folder_check(extract_path):
logger.debug(f"smart.extract.present: {extract_path}")
archive_head = discover_header(archive_path)
extract_head = discover_header(extract_path)
head_comp = compare_header(archive_head, extract_head)
logger.debug(f"smart.extract.{head_comp}: {extract_path}")
if head_comp == HeadComp.same:
pass
elif head_comp == HeadComp.different:
logger.debug(
f"smart.extract.{head_comp}: archive={archive_head}")
logger.debug(
f"smart.extract.{head_comp}: extract={extract_head}")
self.extract(archive_path, extract_path)
elif head_comp == HeadComp.undetermined:
logger.debug(
f"smart.extract.{head_comp}: archive={archive_head}")
logger.debug(
f"smart.extract.{head_comp}: extract={extract_head}")
self.extract(archive_path, extract_path)
else:
assert False, f"wrong head_comp: {head_comp}"
else:
logger.debug(f"smart.extract.missing: {extract_path}")
self.extract(archive_path, extract_path)
def image_type_form(extract_path: str) -> ImageType:
for entry in image_type_list:
has_root = False
if entry.root_path is None:
has_root = True
else:
root_path = extract_path + entry.root_path
has_root = SUDO.folder_check(root_path)
has_meta = False
if entry.meta_path is None:
has_meta = True
else:
meta_path = extract_path + entry.meta_path
has_meta = SUDO.file_check(meta_path)
if has_root and has_meta:
return entry
raise Exception(f"missing type for {extract_path}")
def invoke_template(self, effect: token.TemplateEffect) -> None:
source_path = self.source_path(effect)
target_path = self.target_path(effect)
source_text = SUDO.file_load(source_path)
template = jinja2.Template(source_text)
target_text = template.render(effect.context)
SUDO.files_sync_base(source_path, target_path) # copy attr
SUDO.file_save(target_path, target_text) # change content
SUDO.files_sync_time(source_path, target_path) # copy time
def discover_header(local_url: str) -> Mapping[str, str]:
"obtain meta description of local resource"
local = urlparse(local_url)
header_dict = dict() # never empty
try:
header_dict = SUDO.xattr_load(local.path)
except:
pass
if not header_dict:
header_dict = produce_header(local.path)
return header_dict
def interface_list() -> List[str]:
"""
Discover available host interfaces
"""
cmd_ip = system_command('ip')
command = f"{cmd_ip} -o addr show up primary scope global".split()
result = SUDO.execute_unit(command)
result.assert_return()
line_list = result.stdout.splitlines()
pattern = re.compile(r"^\d+[:]\s+(\S+)\s+(.+)$")
select = lambda line: pattern.search(line).group(1)
face_list = list(map(select, line_list))
return face_list
def invoke_push(self, effect: token.PushEffect) -> None:
#
assert self.image_url
#
source_image = self.build_image
target_image = image_store_from_url(self.image_url)
perform_image_move(source_image, target_image)
#
image_type = ImageTypeNspawn
image_meta = ImageMeta(
image_url=self.image_url,
overlay_url_list=self.overlay_url_list,
profile_command=self.profile_bucket.command,
profile_entry_list=self.profile_bucket.render_entry_list(),
)
target_image = target_image.with_image_type(
image_type).with_image_meta(image_meta)
#
runtime_root = self.runtime_path('/')
extract_root = target_image.extract_path()
SUDO.files_sync_full(runtime_root, extract_root)
#
perform_image_put_steps(target_image)
def extract(self, archive_path: str, extract_path: str) -> None:
SUDO.file_assert(archive_path)
extract_temp = make_temp_path("zip-extract-temp")
SUDO.folder_ensure(extract_temp)
zipper = CmdUnZip()
zipper.with_archive(archive_path)
zipper.with_extract(extract_temp)
zipper.execute_unit_sert()
SUDO.files_move(extract_temp, extract_path)
def archive(self, archive_path: str, extract_path: str) -> None:
SUDO.folder_assert(extract_path)
archive_temp = make_temp_path("zip-archive-temp")
SUDO.parent_ensure(archive_temp)
zipper = CmdZip()
zipper.with_archive(archive_temp)
zipper.with_extract(extract_path)
zipper.execute_unit_sert()
SUDO.files_move(archive_temp, archive_path)
def extract(self, archive_path: str, extract_path: str) -> None:
SUDO.file_assert(archive_path)
extract_temp = make_temp_path("zip-extract-temp")
SUDO.folder_ensure(extract_temp)
tar = Tar()
tar.with_archive(archive_path)
tar.with_extract(extract_temp)
tar.with_packer(archive_path)
tar.with_make_unpack() # keep last
tar.execute_unit_sert()
SUDO.files_move(extract_temp, extract_path)
def get(self, local_url: str, remote_url: str) -> None:
"fetch remote resource and persist headers in xattr"
logger.debug(f"http.get.fetch: {remote_url}")
local, remote = self.parse(local_url, remote_url)
header_path = make_temp_path("curl-head")
source_path = make_temp_path("curl-gets")
target_path = local.path
SUDO.parent_ensure(header_path)
curl = Curl()
with_auth_url(curl, remote_url)
with_proxy_config(curl, remote_url)
curl.with_file_get(source_path)
curl.with_dump_header(header_path)
curl.execute_unit_sert()
header_dict = parse_header_file(header_path)
SUDO.files_delete(header_path)
SUDO.files_move(source_path, target_path)
# TODO attr
SUDO.xattr_save(target_path, header_dict)
def archive(self, archive_path: str, extract_path: str) -> None:
logger.debug(f"pack.archive.tar: {extract_path}")
SUDO.folder_assert(extract_path)
archive_temp = make_temp_path("tar-archive-temp")
SUDO.parent_ensure(archive_temp)
tar = Tar()
tar.with_archive(archive_temp)
tar.with_extract(extract_path)
tar.with_packer(archive_path)
tar.with_make_pack() # keep last
tar.execute_unit_sert()
SUDO.files_move(archive_temp, archive_path)
synchronize_header(extract_path, archive_path)
def invoke_fetch(self, effect: token.FetchEffect) -> None:
remote_url = effect.url
package_name = os.path.basename(remote_url)
fetch_archive = make_temp_path('fetch-archive') + '-' + package_name
fetch_extract = make_temp_path('fetch-extract') + '-' + package_name
# download
transport = transport_provider(remote_url)
transport.get(fetch_archive, remote_url)
# extract
packer = packer_provider(remote_url)
packer.extract(fetch_archive, fetch_extract)
# move
source_path = fetch_extract + '/' + effect.source
target_path = self.runtime_path(effect.target)
SUDO.files_move(source_path, target_path)
# clean
SUDO.files_delete(fetch_archive)
SUDO.files_delete(fetch_extract)
def smart_get(self, local_url:str, remote_url:str) -> None:
"fetch resource when local-vs-remote headers have changed"
local, remote = self.parse(local_url, remote_url)
if SUDO.file_check(local.path):
logger.debug(f"smart.get.present: {remote_url}")
local_head = self.local_head(local_url)
remote_head = self.remote_head(remote_url)
head_comp = compare_header(local_head, remote_head)
logger.debug(f"smart.get.{head_comp}: {remote_url}")
if head_comp == HeadComp.same:
pass
elif head_comp == HeadComp.different:
logger.debug(f"smart.get.{head_comp}: remote={remote_head}")
logger.debug(f"smart.get.{head_comp}: local={local_head}")
self.get(local_url, remote_url)
elif head_comp == HeadComp.undetermined:
logger.debug(f"smart.get.{head_comp}: remote={remote_head}")
logger.debug(f"smart.get.{head_comp}: local={local_head}")
self.get(local_url, remote_url)
else:
assert False, f"wrong head_comp: {head_comp}"
else:
logger.debug(f"smart.get.missing: {remote_url}")
self.get(local_url, remote_url)
def has_extract(self) -> bool:
return SUDO.folder_check(self.extract_path())
def perform_erase(service: str) -> None:
image_path = service_image(service)
SUDO.files_delete(image_path)
def put(self, local_url: str, remote_url: str) -> None:
logger.debug(f"file.put.upload: {remote_url}")
local, remote = self.parse(local_url, remote_url)
SUDO.files_sync_full(local.path, remote.path)
synchronize_header(local.path, remote.path)
def perform_machine_delete(machine_result:MachineResult) -> None:
"delete machine systemd service file"
service_file = machine_result.machine_store.service_file()
SUDO.files_delete(service_file)
SYSTEM_CTL.daemon_reload()
def perform_machine_runtime_erase(machine_result:MachineResult) -> None:
"remove machine runtime folder, i.e,:"
"/var/lib/nspawn/runtime/<machine>"
base_dir = machine_result.machine_store.base_dir()
SUDO.files_delete(base_dir)
def perform_exec_delete(machine_result:MachineResult) -> None:
"delete machine runtime environment: folders and mounts"
for command in machine_result.resource_delete_list():
SUDO.execute_unit_sert(command)
def perform_machine_create(machine_result:MachineResult) -> None:
"create machine systemd service file"
service_text = produce_service_text(machine_result)
service_file = machine_result.machine_store.service_file()
SUDO.file_save(service_file, service_text)
SYSTEM_CTL.daemon_reload()
def invoke_copy(self, effect: token.CopyEffect) -> None:
source_path = self.source_path(effect)
target_path = self.target_path(effect)
SUDO.files_sync_base(source_path, target_path)
def extract(self, archive_path: str, extract_path: str) -> None:
logger.debug(f"pack.extract.tar: {extract_path}")
SUDO.file_assert(archive_path)
extract_temp = make_temp_path("tar-extract-temp")
SUDO.folder_ensure(extract_temp)
tar = Tar()
tar.with_archive(archive_path)
tar.with_extract(extract_temp)
tar.with_packer(archive_path)
tar.with_make_unpack() # keep last
tar.execute_unit_sert()
if SUDO.folder_check(extract_path):
# user rsync to preserve active machines
SUDO.files_sync_full(extract_temp, extract_path)
SUDO.files_delete(extract_temp)
else:
# user move for efficiency
SUDO.files_move(extract_temp, extract_path)
synchronize_header(archive_path, extract_path)
def has_archive(self) -> bool:
return SUDO.file_check(self.archive_path())
def image_meta_save(file: str, image_meta: ImageMeta) -> None:
text = image_meta_encode(image_meta)
SUDO.file_save(file, text)
def image_meta_load(file: str) -> ImageMeta:
text = SUDO.file_load(file)
return image_meta_decode(text)
def perform_image_erase_path(image_store: ImageStore) -> None:
SUDO.files_delete(image_store.archive_path())
SUDO.files_delete(image_store.extract_path())
def perform_image_move(source: ImageStore, target: ImageStore) -> None:
if source.has_archive():
SUDO.files_move(source.archive_path(), target.archive_path())
if source.has_extract():
SUDO.files_move(source.extract_path(), target.extract_path())
