def create(cls, layer):
create_base_zfs()
filesystem_id = generate_random_filesystem_id()
zfs_filesystem = '%s/%s' % (oci_config['driver']['zfs']['base'],
filesystem_id)
mountpoint = pathlib.Path(oci_config['global']['path'], 'filesystems',
filesystem_id)
if layer is None:
log.debug('Creating filesystem (%s)' % zfs_filesystem)
zfs_filesystem = zfs_create(zfs_filesystem, mountpoint=mountpoint)
if zfs_filesystem != zfs_filesystem:
raise OCIError('Could not create zfs filesystem (%s)' %
zfs_filesystem)
else:
origin = layer.filesystem
log.debug('Cloning filesystem (%s) from (%s)' %
(zfs_filesystem, origin.zfs_snapshot))
zfs_filesystem = zfs_clone(zfs_filesystem,
origin.zfs_snapshot,
mountpoint=mountpoint)
if zfs_filesystem != zfs_filesystem:
raise OCIError(
'Could not clone zfs filesystem (%s) from zfs snapshot (%s)'
% (zfs_filesystem, origin.zfs_snapshot))
return Filesystem(filesystem_id, layer, None)
def unmount(self, container_id):
if self.container_id is None:
raise OCIError('Filesystem (%s) is not mounted' % self.id)
if self.container_id != container_id:
raise OCIError(
'Filesystem (%s) mounted for container (%s), not (%s)' %
(self.id, self.container_id, container_id))
self.container_id = None
mountpoint = pathlib.Path(oci_config['global']['path'], 'filesystems',
self.id)
zfs_set(self.zfs_filesystem, mountpoint=mountpoint)
def create(self):
driver_path = pathlib.Path(oci_config['global']['path'])
driver_file_path = driver_path.joinpath('driver.json')
log.debug('Start loading driver file (%s)' % driver_file_path)
if driver_file_path.is_file():
raise OCIError('Driver file (%s) already exist' % driver_file_path)
if self.filesystems is not None:
raise OCIError('Driver filesystems is already initialized' %
driver_file_path)
self.filesystems = {}
self.layers = {}
self.save()
def compress(uncompressed_file_path,
compressed_file_path=None,
method='gz',
keep_original=False,
force=True,
parallel=True):
log.debug('Start compressing file (%s)' % uncompressed_file_path)
commands = {
'xz': ['/usr/bin/xz'],
'gz': ['/usr/bin/gzip'],
'bz2': ['/usr/bin/bzip2'],
'lz': ['/usr/bin/lzma']
}
parallel_commands = {
'gz': ['/usr/bin/pigz'],
'xz': ['/usr/bin/xz', '-T', '0'],
}
if parallel:
cmd = parallel_commands.get(method, None)
else:
cmd = commands.get(method, None)
if cmd is None:
raise OCIError('method (%s) not supported' % method)
cmd.append('--stdout')
cmd.append(str(uncompressed_file_path))
if compressed_file_path is None:
compressed_file_path = uncompressed_file_path.with_suffix(
uncompressed_file_path.suffix + '.' + method)
if compressed_file_path.is_file():
if force:
rm(compressed_file_path)
else:
raise OCIError(
'Target file (%s) allready exist, can not compress' %
str(compressed_file_path))
result = None
with open(compressed_file_path, 'wb') as compressed_file:
log.debug('Start running command: "' + ' '.join(cmd) +
' > %s"' % compressed_file_path)
rc = subprocess.call(cmd, stdout=compressed_file)
log.debug('Finish running command: "' + ' '.join(cmd) +
' > %s"' % compressed_file_path)
if rc == 0:
if not keep_original:
rm(uncompressed_file_path)
result = compressed_file_path
log.debug('Finish compressing file (%s)' % uncompressed_file_path)
return result
def load_layers(self, path=None):
log.debug('Start loading image (%s) layers' % self.id)
if self.config is None:
raise OCIError('Image (%s) has no config' % self.id)
if self.manifest is None:
raise OCIError('Image (%s) has no manifest' % self.id)
root_fs = self.config.get('RootFS')
self.layers = []
for layer_descriptor in self.manifest.get('Layers'):
layer_id = layer_descriptor.get('Digest').encoded()
log.debug('Loading image (%s) layer (%s)' % (self.id, layer_id))
layer = Driver().get_layer(layer_id)
self.layers.append(layer)
log.debug('Finish loading image (%s) layers' % self.id)
def destroy(self):
path = self.path
if zfs_destroy(self.zfs_filesystem, recursive=True) != 0:
raise OCIError('Could not destroy zfs filesystem (%s)' %
self.zfs_filesystem)
if path is not None:
rm(path)
def create(self):
runtime_path = pathlib.Path(oci_config['global']['path'])
runtime_file_path = runtime_path.joinpath('runtime.json')
if runtime_file_path.is_file():
raise OCIError('Runtime file (%s) already exists' %
runtime_file_path)
self.containers = {}
self.save()
def create(self):
distribution_path = pathlib.Path(oci_config['global']['path'])
distribution_file_path = distribution_path.joinpath(
'distribution.json')
log.debug('Start creating distribution file (%s)' %
distribution_file_path)
if self.images is not None:
raise OCIError(
'Distribution repositories is already initialized, can not create'
)
if distribution_file_path.is_file():
raise OCIError('Distribution file (%s) already exists' %
distribution_file_path)
self.images = {}
self.save()
log.debug('Finish creating distribution file (%s)' %
distribution_file_path)
def destroy(self):
if self.id is None:
raise OCIError('Can not destroy image (%s)' % self.id)
if len(Driver().get_child_filesystems(self.top_layer())) != 0:
raise ImageInUseException()
for tag in self.tags:
self.remove_tag(tag)
self.tags = None
self.destroy_manifest()
def mount(self, container_id, path):
if self.container_id is not None:
if self.container_id == container_id:
log.warning(
'Filesystem (%s) already mounted for container (%s)' %
(self.id, container_id))
if self.path != path:
raise OCIError('Filesystem (%s) path (%s) should be (%s)' %
(self.id, self.path, path))
else:
raise OCIError(
'Filesystem (%s) already mounted for container (%s)' %
(self.id, self.container_id))
else:
self.container_id = container_id
previous_path = self.path
zfs_set(self.zfs_filesystem, mountpoint=path)
rm(previous_path)
def uncompress(compressed_file_path,
uncompressed_file_path=None,
method='gz',
keep_original=False,
force=True):
log.debug('Start uncompressing file (%s)' % compressed_file_path)
commands = {
'xz': ['/usr/bin/xzcat'],
'gz': ['/usr/bin/gzcat'],
'bz2': ['/usr/bin/bzcat'],
'lz': ['/usr/bin/lzcat']
}
cmd = commands.get(method, None)
if cmd is None:
raise OCIError('method (%s) not supported' % method)
cmd.append(str(compressed_file_path))
if uncompressed_file_path is None:
if compressed_file_path.suffix == 'method':
uncompressed_file_path = compressed_file_path.parent.joinpath(
compressed_file_path.stem)
else:
raise NotImplementedError()
if uncompressed_file_path.is_file():
if force:
rm(uncompressed_file_path)
else:
raise OCIError(
'Target file (%s) allready exist, can not uncompress' %
str(uncompressed_file_path))
result = None
with open(uncompressed_file_path, 'wb') as uncompressed_file:
log.debug('Start running command: "' + ' '.join(cmd) +
' > %s"' % uncompressed_file_path)
rc = subprocess.call(cmd, stdout=uncompressed_file)
log.debug('Finish running command: "' + ' '.join(cmd) +
' > %s"' % uncompressed_file_path)
if rc == 0:
if not keep_original:
rm(compressed_file_path)
result = uncompressed_file_path
log.debug('Finish uncompressing file (%s)' % compressed_file_path)
return result
def create(cls, image, name, command=None, workdir=None):
log.debug('Start creating container named (%s) from image (%s)' % (name, image.id))
create_time = datetime.utcnow()
os = image.config.get('OS')
if os != operating_system():
raise OCIError('Image (%s) operating system (%s) is not supported' % (image.id, os))
arch = image.config.get('Architecture')
if arch != architecture():
raise OCIError('Image (%s) operating system (%s) is not supported' % (image.id, os))
while True:
container_id = generate_random_sha256()
runc_id = container_id[:12]
if check_free_runc_id(runc_id):
break
image_config_config = image.config.get('Config')
args = command or image_config_config.get('Cmd') or ['/bin/sh']
env = image_config_config.get('Env') or []
cwd = workdir or image_config_config.get('WorkingDir') or '/'
layer = image.top_layer()
container_path = pathlib.Path(oci_config['global']['path'], 'containers', container_id)
if not container_path.is_dir():
container_path.mkdir(parents=True)
rootfs_path = container_path.joinpath('rootfs')
root_path = rootfs_path
solaris = None
if os == 'SunOS':
solaris = Solaris(anet=[SolarisAnet()])
root_path = rootfs_path.joinpath('root')
filesystem = Driver().create_filesystem(layer)
Driver().mount_filesystem(filesystem, container_id, root_path)
config = Spec(
platform=Platform(os=os, arch=arch),
hostname=runc_id,
process=Process(terminal=True, user=User(uid=0, gid=0), args=args, env=env, cwd=cwd),
root=Root(path=str(rootfs_path), readonly=False),
solaris=solaris
)
config_file_path = container_path.joinpath('config.json')
config.save(config_file_path)
runc_create(runc_id, container_path)
log.debug('Finish creating container named (%s) from image (%s)' % (name, image.id))
return cls(container_id, name, create_time)
def commit(self, changeset_file_path):
zfs_snapshot('diff', self.zfs_filesystem)
self.save_changeset(changeset_file_path)
diff_id = sha256sum(changeset_file_path)
if diff_id is None:
raise OCIError('Could not get hash of file (%s)' %
str(changeset_file_path))
previous_path = self.path
zfs_set(self.zfs_filesystem, mountpoint='none')
rm(previous_path)
return diff_id
def create(cls, config, layers):
image_config = config.get('Config')
config.add('Created', datetime.utcnow())
if len(layers) == 0:
raise OCIError('Images must have at least one layer')
manifest = create_manifest(config, layers)
manifest_desciptor = save_manifest(manifest)
image_id = manifest_desciptor.get('Digest').encoded()
for layer in layers:
Driver().add_image_reference(layer, image_id)
return cls(image_id, [])
def load(self):
driver_path = pathlib.Path(oci_config['global']['path'])
driver_file_path = driver_path.joinpath('driver.json')
log.debug('Start loading driver file (%s)' % driver_file_path)
if not driver_file_path.is_file():
raise OCIError('Driver file (%s) does not exist' %
driver_file_path)
if self.filesystems is not None:
raise OCIError('Driver filesystems is already initialized' %
driver_file_path)
with driver_file_path.open() as driver_file:
driver_json = json.load(driver_file)
if oci_config['driver']['type'] != driver_json['type']:
# Convertion between drivers
raise NotImplementedError()
self.filesystems = {}
self.layers = {}
for filesystem_json in driver_json.get('filesystems', []):
self.load_filesystem(filesystem_json)
log.debug('Finish loading driver file (%s)' % driver_file_path)
def remove_layers(self):
log.debug('Start removing image (%s) layers' % self.id)
if self.layers is None:
raise OCIError('Image (%s) has no layers' % self.id)
for layer in reversed(self.layers):
try:
Driver().remove_image_reference(layer, self.id)
Driver().remove_layer(layer)
except LayerInUseException as e:
log.debug(e.args[0])
self.layers = None
log.debug('Finish removing image (%s) layers' % self.id)
def load(self, path=None):
if self.id is None:
raise OCIError('Can not load image without id')
log.debug('Start loading image (%s) manifest' % self.id)
manifests_path = path or pathlib.Path(oci_config['global']['path'],
'manifests')
manifest_file_path = manifests_path.joinpath(self.id)
self.manifest = Manifest.from_file(manifest_file_path)
log.debug('Finish loading image (%s) manifest' % self.id)
if path is not None:
self.copy_manifest(manifest_file_path)
self.load_config(path)
def create(cls, filesystem, compressed=True):
filesystem_id = filesystem.id
log.debug('Start creating layer from filesystem (%s)' % filesystem.id)
with tempfile.TemporaryDirectory() as temp_dir_name:
size = filesystem.size()
changeset_file_path = pathlib.Path(temp_dir_name, 'changeset.tar')
diff_id = filesystem.commit(changeset_file_path)
try:
from .driver import Driver
layer = Driver().get_layer_by_diff_id(diff_id)
Driver().remove_filesystem(filesystem)
except LayerUnknownException:
layer_id = diff_id
media_type = MediaTypeImageLayer
if compressed:
changeset_file_path = compress(changeset_file_path,
keep_original=True)
if changeset_file_path is None:
raise OCIError('Could not compress layer file (%s)' %
str(changeset_file_path))
media_type = MediaTypeImageLayerGzip
layer_id = sha256sum(changeset_file_path)
if layer_id is None:
raise OCIError('Could not get hash of file %s' %
str(changeset_file_path))
layers_path = pathlib.Path(oci_config['global']['path'],
'layers')
layer_file_path = layers_path.joinpath(layer_id)
if not layer_file_path.is_file():
if not layers_path.is_dir():
layers_path.mkdir(parents=True)
cp(changeset_file_path, layer_file_path)
descriptor = Descriptor(
digest=id_to_digest(layer_id),
size=layer_file_path.stat().st_size,
media_type=media_type,
)
layer = cls(descriptor, diff_id, filesystem, size, [])
log.debug('Finish creating layer from filesystem (%s)' % filesystem_id)
return layer
def __init__(self, options):
log.debug('Start importing (%s)' % options.file)
layer = None
with tempfile.TemporaryDirectory() as temp_dir_name:
if options.file == '-':
input_file = os.fdopen(sys.stdin.fileno(), 'rb')
else:
try:
input_file = urlopen(options.file)
except ValueError:
input_file = open(options.file, 'rb')
rootfs_tar_path = pathlib.Path(temp_dir_name, 'rootfs.tar')
with rootfs_tar_path.open('wb') as output_file:
log.debug('Start copying (%s) to (%s)', options.file,
str(rootfs_tar_path))
shutil.copyfileobj(input_file, output_file)
log.debug('Finish copying (%s) to (%s)', options.file,
str(rootfs_tar_path))
if rootfs_tar_path.is_file():
filesystem = Driver().create_filesystem()
untar(filesystem.path, tar_file=input_file)
layer = Driver().create_layer(filesystem)
if layer is None:
log.error('Could not create layer')
exit(-1)
try:
distribution = Distribution()
history = '/bin/sh -c #(nop) IMPORTED file:%s in / ' % options.file
image = Distribution().create_image(layer=layer, history=history)
if options.runc_config is not None:
config_file_path = pathlib.Path(options.runc_config)
if not config_file_path.is_file():
OCIError('Runc config file (%s) does not exist' %
str(config_file_path))
spec = Spec.from_file(config_file_path)
process = spec.get('Process')
command = process.get('Args')
if command is not None:
image.set_command(command)
environment = process.get('Env')
if environment is not None:
image.set_environment(environment)
working_dir = process.get('Cwd')
if working_dir is not None:
image.set_working_dir(working_dir)
if options.tag is not None:
for tag in options.tag:
Distribution().add_tag(self.image, tag)
except Exception as e:
log.error(e.args[0])
exit(-1)
log.debug('Finish importing (%s)' % options.file)
def load(self):
if self.images is not None:
raise OCIError('Distribution already loaded')
distribution_path = pathlib.Path(oci_config['global']['path'])
distribution_file_path = distribution_path.joinpath(
'distribution.json')
log.debug('Start loading distribution file (%s)' %
distribution_file_path)
if not distribution_file_path.is_file():
raise OCIError('Distribution file (%s) does not exist' %
distribution_file_path)
with distribution_file_path.open() as distribution_file:
self.images = {}
images_json = json.load(distribution_file)
for image_json in images_json['images']:
image_id = image_json['id']
tags = image_json['tags'] or []
from .image import Image
image = Image(image_id, tags)
self.images[image_id] = image
log.debug('Finish loading distribution file (%s)' %
distribution_file_path)
def destroy_config(self):
log.debug('Start removing image (%s) config' % self.id)
if self.config is None:
raise OCIError('Image (%s) has no config' % self.id)
config_descriptor = self.manifest.get('Config')
config_digest = config_descriptor.get('Digest')
config_id = digest_to_id(config_digest)
configs_path = pathlib.Path(oci_config['global']['path'], 'configs')
config_file_path = configs_path.joinpath(config_id)
rm(config_file_path)
self.config = None
log.info('Deleted config: %s' % config_digest)
log.debug('Finish removing image (%s) config' % self.id)
def create_base_zfs():
filesystems_path = pathlib.Path(oci_config['global']['path'],
'filesystems')
if not filesystems_path.is_dir():
filesystems_path.mkdir(parents=True)
base_zfs = oci_config['driver']['zfs']['base']
compression = oci_config['driver']['zfs']['compression']
if not zfs_is_filesystem(base_zfs):
log.info('Creating base zfs (%s)' % base_zfs)
base_zfs = zfs_create(base_zfs,
mountpoint=filesystems_path,
compression=compression)
if base_zfs is None:
raise OCIError('Could not create base zfs (%s)' % base_zfs)
def destroy_manifest(self):
log.debug('Start removing image (%s) manifest' % self.id)
if self.manifest is None:
raise OCIError('Image (%s) has no manifest' % self.id)
self.remove_layers()
self.destroy_config()
manifests_path = pathlib.Path(oci_config['global']['path'],
'manifests')
manifest_digest = self.digest
manifest_file_path = manifests_path.joinpath(self.id)
rm(manifest_file_path)
self.manifest = None
log.info('Deleted manifest: %s' % manifest_digest)
log.debug('Finish removing image (%s) manifest' % self.id)
def load_config(self, path=None):
log.debug('Start loading image (%s) config' % self.id)
if self.manifest is None:
raise OCIError('Image (%s) has no manifest' % self.id)
config_descriptor = self.manifest.get('Config')
config_id = config_descriptor.get('Digest').encoded()
log.debug('Loading image (%s) config (%s)' % (self.id, config_id))
configs_path = path or pathlib.Path(oci_config['global']['path'],
'configs')
config_file_path = configs_path.joinpath(config_id)
self.config = Config.from_file(config_file_path)
log.debug('Finish loading image (%s) config' % self.id)
if path is not None:
self.copy_config(config_file_path)
self.load_layers(path)
def untar(dir_path, tar_file_path=None, tar_file=None):
if tar_file_path is not None:
tar_file_path_str = str(tar_file_path)
stdin = None
elif tar_file is not None:
tar_file_path_str = '-'
stdin = tar_file
else:
raise OCIError(
'untar error either tar_file_path or tar_file is needed')
log.debug('Start inserting into tar file (%s)' % tar_file_path_str)
cmd = ['/usr/gnu/bin/tar', '-x', '-f', tar_file_path_str]
log.debug('Running command: "cd ' + str(dir_path) + ';' + ' '.join(cmd) +
'"')
result = subprocess.call(cmd, cwd=dir_path, stdin=stdin)
log.debug('Finish inserting into tar file (%s)' % tar_file_path_str)
return result
def save(self):
distribution_path = pathlib.Path(oci_config['global']['path'])
distribution_file_path = distribution_path.joinpath(
'distribution.json')
log.debug('Start saving distribution file (%s)' %
distribution_file_path)
if self.images is None:
raise OCIError('Distribution is not initialized, can not save')
if not distribution_path.is_dir():
distribution_path.mkdir(parents=True)
images_json = []
for image in self.images.values():
images_json.append({'id': image.id, 'tags': image.tags})
distribution_json = {'images': images_json}
with distribution_file_path.open('w') as distribution_file:
json.dump(distribution_json,
distribution_file,
separators=(',', ':'))
log.debug('Finish saving distribution file (%s)' %
distribution_file_path)
def save(self):
driver_path = pathlib.Path(oci_config['global']['path'])
driver_file_path = driver_path.joinpath('driver.json')
log.debug('Start saving driver file (%s)' % driver_file_path)
if self.filesystems is None:
raise OCIError(
'Driver filesystems is not initialized, can not save')
filesystems_json = [
self.filesystem_to_json(filesystem)
for filesystem in self.filesystems.values()
if filesystem.layer is None
]
if not driver_path.is_dir():
driver_path.mkdir(filesystems=True)
with driver_file_path.open('w') as driver_file:
driver_json = {
'type': oci_config['driver']['type'],
'filesystems': filesystems_json
}
json.dump(driver_json, driver_file, separators=(',', ':'))
log.debug('Finish saving driver file (%s)' % driver_file_path)
def remove_tag(self, tag):
if tag not in self.tags:
raise OCIError('Image (%s) does not have tag (%s)' %
(self.id, tag))
log.info('Untagged: ' + tag)
self.tags.remove(tag)
def add_tag(self, tag):
if tag in self.tags:
raise OCIError('Image (%s) already has tag (%s)' % (self.id, tag))
self.tags.append(tag)
def get_filesystem_class():
driver_type = oci_config['driver']['type']
if driver_type == 'zfs':
from .zfs_filesystem import ZFSFilesystem
return ZFSFilesystem
raise OCIError('Unsupported driver type (%s)' % driver_type)