Exemplo n.º 1
0
 def create(cls, layer):
     create_base_zfs()
     filesystem_id = generate_random_filesystem_id()
     zfs_filesystem = '%s/%s' % (oci_config['driver']['zfs']['base'],
                                 filesystem_id)
     mountpoint = pathlib.Path(oci_config['global']['path'], 'filesystems',
                               filesystem_id)
     if layer is None:
         log.debug('Creating filesystem (%s)' % zfs_filesystem)
         zfs_filesystem = zfs_create(zfs_filesystem, mountpoint=mountpoint)
         if zfs_filesystem != zfs_filesystem:
             raise OCIError('Could not create zfs filesystem (%s)' %
                            zfs_filesystem)
     else:
         origin = layer.filesystem
         log.debug('Cloning filesystem (%s) from (%s)' %
                   (zfs_filesystem, origin.zfs_snapshot))
         zfs_filesystem = zfs_clone(zfs_filesystem,
                                    origin.zfs_snapshot,
                                    mountpoint=mountpoint)
         if zfs_filesystem != zfs_filesystem:
             raise OCIError(
                 'Could not clone zfs filesystem (%s) from zfs snapshot (%s)'
                 % (zfs_filesystem, origin.zfs_snapshot))
     return Filesystem(filesystem_id, layer, None)
Exemplo n.º 2
0
 def unmount(self, container_id):
     if self.container_id is None:
         raise OCIError('Filesystem (%s) is not mounted' % self.id)
     if self.container_id != container_id:
         raise OCIError(
             'Filesystem (%s) mounted for container (%s), not (%s)' %
             (self.id, self.container_id, container_id))
     self.container_id = None
     mountpoint = pathlib.Path(oci_config['global']['path'], 'filesystems',
                               self.id)
     zfs_set(self.zfs_filesystem, mountpoint=mountpoint)
Exemplo n.º 3
0
 def create(self):
     driver_path = pathlib.Path(oci_config['global']['path'])
     driver_file_path = driver_path.joinpath('driver.json')
     log.debug('Start loading driver file (%s)' % driver_file_path)
     if driver_file_path.is_file():
         raise OCIError('Driver file (%s) already exist' % driver_file_path)
     if self.filesystems is not None:
         raise OCIError('Driver filesystems is already initialized' %
                        driver_file_path)
     self.filesystems = {}
     self.layers = {}
     self.save()
Exemplo n.º 4
0
def compress(uncompressed_file_path,
             compressed_file_path=None,
             method='gz',
             keep_original=False,
             force=True,
             parallel=True):
    log.debug('Start compressing file (%s)' % uncompressed_file_path)
    commands = {
        'xz': ['/usr/bin/xz'],
        'gz': ['/usr/bin/gzip'],
        'bz2': ['/usr/bin/bzip2'],
        'lz': ['/usr/bin/lzma']
    }

    parallel_commands = {
        'gz': ['/usr/bin/pigz'],
        'xz': ['/usr/bin/xz', '-T', '0'],
    }
    if parallel:
        cmd = parallel_commands.get(method, None)
    else:
        cmd = commands.get(method, None)
    if cmd is None:
        raise OCIError('method (%s) not supported' % method)
    cmd.append('--stdout')

    cmd.append(str(uncompressed_file_path))
    if compressed_file_path is None:
        compressed_file_path = uncompressed_file_path.with_suffix(
            uncompressed_file_path.suffix + '.' + method)

    if compressed_file_path.is_file():
        if force:
            rm(compressed_file_path)
        else:
            raise OCIError(
                'Target file (%s) allready exist, can not compress' %
                str(compressed_file_path))
    result = None
    with open(compressed_file_path, 'wb') as compressed_file:
        log.debug('Start running command: "' + ' '.join(cmd) +
                  ' > %s"' % compressed_file_path)
        rc = subprocess.call(cmd, stdout=compressed_file)
        log.debug('Finish running command: "' + ' '.join(cmd) +
                  ' > %s"' % compressed_file_path)
        if rc == 0:
            if not keep_original:
                rm(uncompressed_file_path)
            result = compressed_file_path
    log.debug('Finish compressing file (%s)' % uncompressed_file_path)
    return result
Exemplo n.º 5
0
 def load_layers(self, path=None):
     log.debug('Start loading image (%s) layers' % self.id)
     if self.config is None:
         raise OCIError('Image (%s) has no config' % self.id)
     if self.manifest is None:
         raise OCIError('Image (%s) has no manifest' % self.id)
     root_fs = self.config.get('RootFS')
     self.layers = []
     for layer_descriptor in self.manifest.get('Layers'):
         layer_id = layer_descriptor.get('Digest').encoded()
         log.debug('Loading image (%s) layer (%s)' % (self.id, layer_id))
         layer = Driver().get_layer(layer_id)
         self.layers.append(layer)
     log.debug('Finish loading image (%s) layers' % self.id)
Exemplo n.º 6
0
 def destroy(self):
     path = self.path
     if zfs_destroy(self.zfs_filesystem, recursive=True) != 0:
         raise OCIError('Could not destroy zfs filesystem (%s)' %
                        self.zfs_filesystem)
     if path is not None:
         rm(path)
Exemplo n.º 7
0
 def create(self):
     runtime_path = pathlib.Path(oci_config['global']['path'])
     runtime_file_path = runtime_path.joinpath('runtime.json')
     if runtime_file_path.is_file():
         raise OCIError('Runtime file (%s) already exists' %
                        runtime_file_path)
     self.containers = {}
     self.save()
Exemplo n.º 8
0
 def create(self):
     distribution_path = pathlib.Path(oci_config['global']['path'])
     distribution_file_path = distribution_path.joinpath(
         'distribution.json')
     log.debug('Start creating distribution file (%s)' %
               distribution_file_path)
     if self.images is not None:
         raise OCIError(
             'Distribution repositories is already initialized, can not create'
         )
     if distribution_file_path.is_file():
         raise OCIError('Distribution file (%s) already exists' %
                        distribution_file_path)
     self.images = {}
     self.save()
     log.debug('Finish creating distribution file (%s)' %
               distribution_file_path)
Exemplo n.º 9
0
 def destroy(self):
     if self.id is None:
         raise OCIError('Can not destroy image (%s)' % self.id)
     if len(Driver().get_child_filesystems(self.top_layer())) != 0:
         raise ImageInUseException()
     for tag in self.tags:
         self.remove_tag(tag)
     self.tags = None
     self.destroy_manifest()
Exemplo n.º 10
0
 def mount(self, container_id, path):
     if self.container_id is not None:
         if self.container_id == container_id:
             log.warning(
                 'Filesystem (%s) already mounted for container (%s)' %
                 (self.id, container_id))
             if self.path != path:
                 raise OCIError('Filesystem (%s) path (%s) should be (%s)' %
                                (self.id, self.path, path))
         else:
             raise OCIError(
                 'Filesystem (%s) already mounted for container (%s)' %
                 (self.id, self.container_id))
     else:
         self.container_id = container_id
         previous_path = self.path
         zfs_set(self.zfs_filesystem, mountpoint=path)
         rm(previous_path)
Exemplo n.º 11
0
def uncompress(compressed_file_path,
               uncompressed_file_path=None,
               method='gz',
               keep_original=False,
               force=True):
    log.debug('Start uncompressing file (%s)' % compressed_file_path)
    commands = {
        'xz': ['/usr/bin/xzcat'],
        'gz': ['/usr/bin/gzcat'],
        'bz2': ['/usr/bin/bzcat'],
        'lz': ['/usr/bin/lzcat']
    }

    cmd = commands.get(method, None)
    if cmd is None:
        raise OCIError('method (%s) not supported' % method)
    cmd.append(str(compressed_file_path))

    if uncompressed_file_path is None:
        if compressed_file_path.suffix == 'method':
            uncompressed_file_path = compressed_file_path.parent.joinpath(
                compressed_file_path.stem)
        else:
            raise NotImplementedError()
    if uncompressed_file_path.is_file():
        if force:
            rm(uncompressed_file_path)
        else:
            raise OCIError(
                'Target file (%s) allready exist, can not uncompress' %
                str(uncompressed_file_path))
    result = None
    with open(uncompressed_file_path, 'wb') as uncompressed_file:
        log.debug('Start running command: "' + ' '.join(cmd) +
                  ' > %s"' % uncompressed_file_path)
        rc = subprocess.call(cmd, stdout=uncompressed_file)
        log.debug('Finish running command: "' + ' '.join(cmd) +
                  ' > %s"' % uncompressed_file_path)
        if rc == 0:
            if not keep_original:
                rm(compressed_file_path)
            result = uncompressed_file_path
    log.debug('Finish uncompressing file (%s)' % compressed_file_path)
    return result
Exemplo n.º 12
0
 def create(cls, image, name, command=None, workdir=None):
     log.debug('Start creating container named (%s) from image (%s)' % (name, image.id))
     create_time = datetime.utcnow()
     os = image.config.get('OS')
     if os != operating_system():
         raise OCIError('Image (%s) operating system (%s) is not supported' % (image.id, os))
     arch = image.config.get('Architecture')
     if arch != architecture():
         raise OCIError('Image (%s) operating system (%s) is not supported' % (image.id, os))
     while True:
         container_id = generate_random_sha256()
         runc_id = container_id[:12]
         if check_free_runc_id(runc_id):
             break
     image_config_config = image.config.get('Config')
     args = command or image_config_config.get('Cmd') or ['/bin/sh']
     env = image_config_config.get('Env') or []
     cwd = workdir or image_config_config.get('WorkingDir') or '/'
     layer = image.top_layer()
     container_path = pathlib.Path(oci_config['global']['path'], 'containers', container_id)
     if not container_path.is_dir():
         container_path.mkdir(parents=True)
     rootfs_path = container_path.joinpath('rootfs')
     root_path = rootfs_path
     solaris = None
     if os == 'SunOS':
         solaris = Solaris(anet=[SolarisAnet()])
         root_path = rootfs_path.joinpath('root')
     filesystem = Driver().create_filesystem(layer)
     Driver().mount_filesystem(filesystem, container_id, root_path)
     config = Spec(
         platform=Platform(os=os, arch=arch),
         hostname=runc_id,
         process=Process(terminal=True, user=User(uid=0, gid=0), args=args, env=env, cwd=cwd),
         root=Root(path=str(rootfs_path), readonly=False),
         solaris=solaris
     )
     config_file_path = container_path.joinpath('config.json')
     config.save(config_file_path)
     runc_create(runc_id, container_path)
     log.debug('Finish creating container named (%s) from image (%s)' % (name, image.id))
     return cls(container_id, name, create_time)
Exemplo n.º 13
0
 def commit(self, changeset_file_path):
     zfs_snapshot('diff', self.zfs_filesystem)
     self.save_changeset(changeset_file_path)
     diff_id = sha256sum(changeset_file_path)
     if diff_id is None:
         raise OCIError('Could not get hash of file (%s)' %
                        str(changeset_file_path))
     previous_path = self.path
     zfs_set(self.zfs_filesystem, mountpoint='none')
     rm(previous_path)
     return diff_id
Exemplo n.º 14
0
 def create(cls, config, layers):
     image_config = config.get('Config')
     config.add('Created', datetime.utcnow())
     if len(layers) == 0:
         raise OCIError('Images must have at least one layer')
     manifest = create_manifest(config, layers)
     manifest_desciptor = save_manifest(manifest)
     image_id = manifest_desciptor.get('Digest').encoded()
     for layer in layers:
         Driver().add_image_reference(layer, image_id)
     return cls(image_id, [])
Exemplo n.º 15
0
 def load(self):
     driver_path = pathlib.Path(oci_config['global']['path'])
     driver_file_path = driver_path.joinpath('driver.json')
     log.debug('Start loading driver file (%s)' % driver_file_path)
     if not driver_file_path.is_file():
         raise OCIError('Driver file (%s) does not exist' %
                        driver_file_path)
     if self.filesystems is not None:
         raise OCIError('Driver filesystems is already initialized' %
                        driver_file_path)
     with driver_file_path.open() as driver_file:
         driver_json = json.load(driver_file)
         if oci_config['driver']['type'] != driver_json['type']:
             # Convertion between drivers
             raise NotImplementedError()
         self.filesystems = {}
         self.layers = {}
         for filesystem_json in driver_json.get('filesystems', []):
             self.load_filesystem(filesystem_json)
     log.debug('Finish loading driver file (%s)' % driver_file_path)
Exemplo n.º 16
0
 def remove_layers(self):
     log.debug('Start removing image (%s) layers' % self.id)
     if self.layers is None:
         raise OCIError('Image (%s) has no layers' % self.id)
     for layer in reversed(self.layers):
         try:
             Driver().remove_image_reference(layer, self.id)
             Driver().remove_layer(layer)
         except LayerInUseException as e:
             log.debug(e.args[0])
     self.layers = None
     log.debug('Finish removing image (%s) layers' % self.id)
Exemplo n.º 17
0
 def load(self, path=None):
     if self.id is None:
         raise OCIError('Can not load image without id')
     log.debug('Start loading image (%s) manifest' % self.id)
     manifests_path = path or pathlib.Path(oci_config['global']['path'],
                                           'manifests')
     manifest_file_path = manifests_path.joinpath(self.id)
     self.manifest = Manifest.from_file(manifest_file_path)
     log.debug('Finish loading image (%s) manifest' % self.id)
     if path is not None:
         self.copy_manifest(manifest_file_path)
     self.load_config(path)
Exemplo n.º 18
0
 def create(cls, filesystem, compressed=True):
     filesystem_id = filesystem.id
     log.debug('Start creating layer from filesystem (%s)' % filesystem.id)
     with tempfile.TemporaryDirectory() as temp_dir_name:
         size = filesystem.size()
         changeset_file_path = pathlib.Path(temp_dir_name, 'changeset.tar')
         diff_id = filesystem.commit(changeset_file_path)
         try:
             from .driver import Driver
             layer = Driver().get_layer_by_diff_id(diff_id)
             Driver().remove_filesystem(filesystem)
         except LayerUnknownException:
             layer_id = diff_id
             media_type = MediaTypeImageLayer
             if compressed:
                 changeset_file_path = compress(changeset_file_path,
                                                keep_original=True)
                 if changeset_file_path is None:
                     raise OCIError('Could not compress layer file (%s)' %
                                    str(changeset_file_path))
                 media_type = MediaTypeImageLayerGzip
                 layer_id = sha256sum(changeset_file_path)
                 if layer_id is None:
                     raise OCIError('Could not get hash of file %s' %
                                    str(changeset_file_path))
             layers_path = pathlib.Path(oci_config['global']['path'],
                                        'layers')
             layer_file_path = layers_path.joinpath(layer_id)
             if not layer_file_path.is_file():
                 if not layers_path.is_dir():
                     layers_path.mkdir(parents=True)
                 cp(changeset_file_path, layer_file_path)
             descriptor = Descriptor(
                 digest=id_to_digest(layer_id),
                 size=layer_file_path.stat().st_size,
                 media_type=media_type,
             )
             layer = cls(descriptor, diff_id, filesystem, size, [])
     log.debug('Finish creating layer from filesystem (%s)' % filesystem_id)
     return layer
Exemplo n.º 19
0
 def __init__(self, options):
     log.debug('Start importing (%s)' % options.file)
     layer = None
     with tempfile.TemporaryDirectory() as temp_dir_name:
         if options.file == '-':
             input_file = os.fdopen(sys.stdin.fileno(), 'rb')
         else:
             try:
                 input_file = urlopen(options.file)
             except ValueError:
                 input_file = open(options.file, 'rb')
         rootfs_tar_path = pathlib.Path(temp_dir_name, 'rootfs.tar')
         with rootfs_tar_path.open('wb') as output_file:
             log.debug('Start copying (%s) to (%s)', options.file,
                       str(rootfs_tar_path))
             shutil.copyfileobj(input_file, output_file)
             log.debug('Finish copying (%s) to (%s)', options.file,
                       str(rootfs_tar_path))
         if rootfs_tar_path.is_file():
             filesystem = Driver().create_filesystem()
             untar(filesystem.path, tar_file=input_file)
             layer = Driver().create_layer(filesystem)
     if layer is None:
         log.error('Could not create layer')
         exit(-1)
     try:
         distribution = Distribution()
         history = '/bin/sh -c #(nop) IMPORTED file:%s in / ' % options.file
         image = Distribution().create_image(layer=layer, history=history)
         if options.runc_config is not None:
             config_file_path = pathlib.Path(options.runc_config)
             if not config_file_path.is_file():
                 OCIError('Runc config file (%s) does not exist' %
                          str(config_file_path))
             spec = Spec.from_file(config_file_path)
             process = spec.get('Process')
             command = process.get('Args')
             if command is not None:
                 image.set_command(command)
             environment = process.get('Env')
             if environment is not None:
                 image.set_environment(environment)
             working_dir = process.get('Cwd')
             if working_dir is not None:
                 image.set_working_dir(working_dir)
                 if options.tag is not None:
                     for tag in options.tag:
                         Distribution().add_tag(self.image, tag)
     except Exception as e:
         log.error(e.args[0])
         exit(-1)
     log.debug('Finish importing (%s)' % options.file)
Exemplo n.º 20
0
 def load(self):
     if self.images is not None:
         raise OCIError('Distribution already loaded')
     distribution_path = pathlib.Path(oci_config['global']['path'])
     distribution_file_path = distribution_path.joinpath(
         'distribution.json')
     log.debug('Start loading distribution file (%s)' %
               distribution_file_path)
     if not distribution_file_path.is_file():
         raise OCIError('Distribution file (%s) does not exist' %
                        distribution_file_path)
     with distribution_file_path.open() as distribution_file:
         self.images = {}
         images_json = json.load(distribution_file)
         for image_json in images_json['images']:
             image_id = image_json['id']
             tags = image_json['tags'] or []
             from .image import Image
             image = Image(image_id, tags)
             self.images[image_id] = image
     log.debug('Finish loading distribution file (%s)' %
               distribution_file_path)
Exemplo n.º 21
0
 def destroy_config(self):
     log.debug('Start removing image (%s) config' % self.id)
     if self.config is None:
         raise OCIError('Image (%s) has no config' % self.id)
     config_descriptor = self.manifest.get('Config')
     config_digest = config_descriptor.get('Digest')
     config_id = digest_to_id(config_digest)
     configs_path = pathlib.Path(oci_config['global']['path'], 'configs')
     config_file_path = configs_path.joinpath(config_id)
     rm(config_file_path)
     self.config = None
     log.info('Deleted config: %s' % config_digest)
     log.debug('Finish removing image (%s) config' % self.id)
Exemplo n.º 22
0
def create_base_zfs():
    filesystems_path = pathlib.Path(oci_config['global']['path'],
                                    'filesystems')
    if not filesystems_path.is_dir():
        filesystems_path.mkdir(parents=True)
    base_zfs = oci_config['driver']['zfs']['base']
    compression = oci_config['driver']['zfs']['compression']
    if not zfs_is_filesystem(base_zfs):
        log.info('Creating base zfs (%s)' % base_zfs)
        base_zfs = zfs_create(base_zfs,
                              mountpoint=filesystems_path,
                              compression=compression)
        if base_zfs is None:
            raise OCIError('Could not create base zfs (%s)' % base_zfs)
Exemplo n.º 23
0
 def destroy_manifest(self):
     log.debug('Start removing image (%s) manifest' % self.id)
     if self.manifest is None:
         raise OCIError('Image (%s) has no manifest' % self.id)
     self.remove_layers()
     self.destroy_config()
     manifests_path = pathlib.Path(oci_config['global']['path'],
                                   'manifests')
     manifest_digest = self.digest
     manifest_file_path = manifests_path.joinpath(self.id)
     rm(manifest_file_path)
     self.manifest = None
     log.info('Deleted manifest: %s' % manifest_digest)
     log.debug('Finish removing image (%s) manifest' % self.id)
Exemplo n.º 24
0
 def load_config(self, path=None):
     log.debug('Start loading image (%s) config' % self.id)
     if self.manifest is None:
         raise OCIError('Image (%s) has no manifest' % self.id)
     config_descriptor = self.manifest.get('Config')
     config_id = config_descriptor.get('Digest').encoded()
     log.debug('Loading image (%s) config (%s)' % (self.id, config_id))
     configs_path = path or pathlib.Path(oci_config['global']['path'],
                                         'configs')
     config_file_path = configs_path.joinpath(config_id)
     self.config = Config.from_file(config_file_path)
     log.debug('Finish loading image (%s) config' % self.id)
     if path is not None:
         self.copy_config(config_file_path)
     self.load_layers(path)
Exemplo n.º 25
0
def untar(dir_path, tar_file_path=None, tar_file=None):
    if tar_file_path is not None:
        tar_file_path_str = str(tar_file_path)
        stdin = None
    elif tar_file is not None:
        tar_file_path_str = '-'
        stdin = tar_file
    else:
        raise OCIError(
            'untar error either tar_file_path or tar_file is needed')
    log.debug('Start inserting into tar file (%s)' % tar_file_path_str)
    cmd = ['/usr/gnu/bin/tar', '-x', '-f', tar_file_path_str]
    log.debug('Running command: "cd ' + str(dir_path) + ';' + ' '.join(cmd) +
              '"')
    result = subprocess.call(cmd, cwd=dir_path, stdin=stdin)
    log.debug('Finish inserting into tar file (%s)' % tar_file_path_str)
    return result
Exemplo n.º 26
0
 def save(self):
     distribution_path = pathlib.Path(oci_config['global']['path'])
     distribution_file_path = distribution_path.joinpath(
         'distribution.json')
     log.debug('Start saving distribution file (%s)' %
               distribution_file_path)
     if self.images is None:
         raise OCIError('Distribution is not initialized, can not save')
     if not distribution_path.is_dir():
         distribution_path.mkdir(parents=True)
     images_json = []
     for image in self.images.values():
         images_json.append({'id': image.id, 'tags': image.tags})
     distribution_json = {'images': images_json}
     with distribution_file_path.open('w') as distribution_file:
         json.dump(distribution_json,
                   distribution_file,
                   separators=(',', ':'))
     log.debug('Finish saving distribution file (%s)' %
               distribution_file_path)
Exemplo n.º 27
0
 def save(self):
     driver_path = pathlib.Path(oci_config['global']['path'])
     driver_file_path = driver_path.joinpath('driver.json')
     log.debug('Start saving driver file (%s)' % driver_file_path)
     if self.filesystems is None:
         raise OCIError(
             'Driver filesystems is not initialized, can not save')
     filesystems_json = [
         self.filesystem_to_json(filesystem)
         for filesystem in self.filesystems.values()
         if filesystem.layer is None
     ]
     if not driver_path.is_dir():
         driver_path.mkdir(filesystems=True)
     with driver_file_path.open('w') as driver_file:
         driver_json = {
             'type': oci_config['driver']['type'],
             'filesystems': filesystems_json
         }
         json.dump(driver_json, driver_file, separators=(',', ':'))
     log.debug('Finish saving driver file (%s)' % driver_file_path)
Exemplo n.º 28
0
 def remove_tag(self, tag):
     if tag not in self.tags:
         raise OCIError('Image (%s) does not have tag (%s)' %
                        (self.id, tag))
     log.info('Untagged: ' + tag)
     self.tags.remove(tag)
Exemplo n.º 29
0
 def add_tag(self, tag):
     if tag in self.tags:
         raise OCIError('Image (%s) already has tag (%s)' % (self.id, tag))
     self.tags.append(tag)
Exemplo n.º 30
0
def get_filesystem_class():
    driver_type = oci_config['driver']['type']
    if driver_type == 'zfs':
        from .zfs_filesystem import ZFSFilesystem
        return ZFSFilesystem
    raise OCIError('Unsupported driver type (%s)' % driver_type)