Пример #1
0
        def virtual_mount_spec_impl(virtual_source, repository):
            TestPlugin.assert_plugin_args(virtual_source=virtual_source,
                                          repository=repository)

            primary_mount = Mount(virtual_source.connection.environment,
                                  TEST_MOUNT_PATH, TEST_SHARED_PATH)
            another_mount = Mount(virtual_source.connection.environment,
                                  TEST_MOUNT_PATH, TEST_SHARED_PATH)
            ownership_spec = OwnershipSpecification(TEST_UID, TEST_GID)

            return MountSpecification([primary_mount, another_mount],
                                      ownership_spec)
Пример #2
0
def virtual_mount_specification(virtual_source, repository):
    mount_path = virtual_source.parameters.mount_path

    if check_stale_mountpoint(virtual_source.connection, mount_path):
        cleanup_process = CouchbaseOperation(
            Resource.ObjectBuilder.set_virtual_source(
                virtual_source).set_repository(repository).build())
        cleanup_process.stop_couchbase()
        clean_stale_mountpoint(virtual_source.connection, mount_path)

    check_server_is_used(virtual_source.connection, mount_path)

    mounts = [Mount(virtual_source.connection.environment, mount_path)]
    logger.debug("Mounting path {}".format(mount_path))
    logger.debug("Setting ownership to uid {} and gid {}".format(
        repository.uid, repository.gid))
    ownership_spec = OwnershipSpecification(repository.uid, repository.gid)

    logger.debug("in mounting: {}".format(
        str(virtual_source.parameters.node_list)))

    if virtual_source.parameters.node_list is not None and len(
            virtual_source.parameters.node_list) > 0:
        # more nodes
        for m in virtual_source.parameters.node_list:
            logger.debug("in loop: {}".format(str(m)))
            node_host = RemoteHost(name='foo',
                                   reference=m["environment"].replace(
                                       '_ENVIRONMENT', ''),
                                   binary_path="",
                                   scratch_path="")
            e = RemoteEnvironment("foo", m["environment"], node_host)
            mount = Mount(e, mount_path)
            mounts.append(mount)

            user = RemoteUser(name="unused", reference=m['environmentUser'])
            environment = RemoteEnvironment(name="unused",
                                            reference=m['environment'],
                                            host=node_host)
            clean_node_conn = RemoteConnection(environment=environment,
                                               user=user)

            if check_stale_mountpoint(clean_node_conn, mount_path):
                clean_node = CouchbaseOperation(
                    Resource.ObjectBuilder.set_virtual_source(
                        virtual_source).set_repository(repository).build(),
                    clean_node_conn)
                clean_node.stop_couchbase()
                clean_stale_mountpoint(clean_node_conn, mount_path)

            check_server_is_used(clean_node_conn, mount_path)

    return MountSpecification(mounts, ownership_spec)
Пример #3
0
 def test_init_mount_invalid_reference_type(reference):
     with pytest.raises(IncorrectTypeError) as err_info:
         Mount(reference, 'mount_path', 'shared_path')
     assert err_info.value.message == (
         "Mount's parameter 'remote_environment' was type '{}' but "
         "should be of any one of the following types: '['dlpx.virtualization.common._common_classes.RemoteEnvironment', 'basestring']'.".format(type(reference).__name__)
     )
Пример #4
0
def virtual_mount_specification(virtual_source, repository):
    logger.debug("virtual_mount_specification")
    mount_path = virtual_source.parameters.m_path
    logger.debug("Mount Path:" + mount_path)
    environment = virtual_source.connection.environment
    mounts = [Mount(environment, mount_path)]
    return MountSpecification(mounts)
 def test_init_mount_bad_remote_env():
     with pytest.raises(IncorrectTypeError) as err_info:
         Mount('bad string', 'mount_path', 'shared_path')
     assert err_info.value.message == (
         "Mount's parameter 'remote_environment' was type 'str' but"
         " should be of"
         " class 'dlpx.virtualization.common._common_classes.RemoteEnvironment'."
     )
Пример #6
0
    def _internal_worker(self, request):
        """Staged Worker Wrapper for staged plugins.

        Executed as part of validated sync. This plugin
        operation is run to sync staging source as part
        of the validated sync operation.

        Run worker operation for a staged source.

        Args:
           request (StagedWorkerRequest): Worker arguments.

        Returns:
           StagedWorkerResponse: A response containing StagedWorkerResult
           if successful or PluginErrorResult in case of an error.
        """
        # Reasoning for method imports are in this file's docstring.
        from generated.definitions import RepositoryDefinition
        from generated.definitions import LinkedSourceDefinition
        from generated.definitions import SourceConfigDefinition

        #
        # While linked.worker() is not a required operation, this should
        # not be called if it wasn't implemented.
        #
        if not self.worker_impl:
            raise OperationNotDefinedError(Op.LINKED_WORKER)

        staged_source_definition = LinkedSourceDefinition.from_dict(
            json.loads(request.staged_source.linked_source.parameters.json))
        mount = Mount(
            remote_environment=(RemoteEnvironment.from_proto(
                request.staged_source.staged_mount.remote_environment)),
            mount_path=request.staged_source.staged_mount.mount_path,
            shared_path=request.staged_source.staged_mount.shared_path)
        staged_source = StagedSource(
            guid=request.staged_source.linked_source.guid,
            source_connection=RemoteConnection.from_proto(
                request.staged_source.source_connection),
            parameters=staged_source_definition,
            mount=mount,
            staged_connection=RemoteConnection.from_proto(
                request.staged_source.staged_connection))

        repository = RepositoryDefinition.from_dict(
            json.loads(request.repository.parameters.json))
        source_config = SourceConfigDefinition.from_dict(
            json.loads(request.source_config.parameters.json))

        self.worker_impl(staged_source=staged_source,
                         repository=repository,
                         source_config=source_config)

        staged_worker_response = platform_pb2.StagedWorkerResponse()
        staged_worker_response.return_value.CopyFrom(
            platform_pb2.StagedWorkerResult())

        return staged_worker_response
Пример #7
0
        def staged_mount_spec_impl(staged_source, repository):
            TestPlugin.assert_plugin_args(staged_source=staged_source,
                                                repository=repository)

            mount = Mount(
                staged_source.source_connection.environment, TEST_MOUNT_PATH)
            ownership_spec = OwnershipSpecification(TEST_UID, TEST_GID)

            return MountSpecification([mount], ownership_spec)
Пример #8
0
def staged_mount_specification(staged_source, repository):
    common.add_debug_heading_block("Start Staged Mount Specification")
    helpers._record_hook("staging mount specification",
                         staged_source.staged_connection)
    logger.debug("mount_path={}".format(staged_source.parameters.mount_path))
    mount = Mount(staged_source.staged_connection.environment,
                  staged_source.parameters.mount_path)
    common.add_debug_heading_block("End Staged Mount Specification")
    return MountSpecification([mount])
Пример #9
0
        def staged_mount_spec_impl(staged_source, repository):
            TestPlugin.assert_plugin_args(staged_source=staged_source,
                                          repository=repository)
            # setting the shared_path should fail in the wrapper
            mount = Mount(staged_source.source_connection.environment,
                          TEST_MOUNT_PATH, TEST_SHARED_PATH)
            ownership_spec = OwnershipSpecification(TEST_UID, TEST_GID)

            return MountSpecification([mount], ownership_spec)
    def test_init_mount_spec_bad_owner_spec(remote_environment):
        mount = Mount(remote_environment, 'mount_path', 'shared_path')

        with pytest.raises(IncorrectTypeError) as err_info:
            MountSpecification([mount], 'string')
        assert err_info.value.message == (
            "MountSpecification's parameter 'ownership_specification' was"
            " type 'str' but should be of class 'dlpx.virtualization"
            ".platform._plugin_classes.OwnershipSpecification'"
            " if defined.")
Пример #11
0
def linked_mount_specification(staged_source, repository):
    logger.debug("linked_mount_specification")
    try:
        mount_path = staged_source.parameters.mount_path
        logger.debug("Mount Path:" + mount_path)
        environment = staged_source.staged_connection.environment
        mounts = [Mount(environment, mount_path)]
    except Exception as err:
        logger.debug("ERROR: Error creating NFS Mount" + err.message)
        raise
    return MountSpecification(mounts)
Пример #12
0
def mount_specification(repository, virtual_source):
    helpers._record_hook("virtual mount spec", virtual_source.connection)
    virtual_source.mongo_install_path = repository.mongo_install_path
    virtual_source.mongo_shell_path = repository.mongo_shell_path

    nodes = []
    nodes.append(virtual_source.connection.environment.reference)
    for node in virtual_source.parameters.additional_nodes:
        nodes.append(node['environment'])
    totalnodes = len(nodes)
    logger.info("Total Nodes : {}".format(totalnodes))
    logger.info("Nodes = {}".format(nodes))
    logger.info("Mount Path = {}".format(virtual_source.parameters.mount_path))

    nodemount_list = []
    for node in nodes:
        nodemount = Mount(node, virtual_source.parameters.mount_path)
        nodemount_list.append(nodemount)
    return MountSpecification(nodemount_list)
Пример #13
0
def linked_mount_specification(staged_source, repository):
    mount_path = staged_source.parameters.mount_path

    if check_stale_mountpoint(staged_source.staged_connection, mount_path):
        cleanup_process = CouchbaseOperation(
            Resource.ObjectBuilder.set_staged_source(
                staged_source).set_repository(repository).build())
        cleanup_process.stop_couchbase()
        clean_stale_mountpoint(staged_source.staged_connection, mount_path)

    check_server_is_used(staged_source.staged_connection, mount_path)

    environment = staged_source.staged_connection.environment
    linked.check_mount_path(staged_source, repository)
    logger.debug("Mounting path {}".format(mount_path))
    mounts = [Mount(environment, mount_path)]
    logger.debug("Setting ownership to uid {} and gid {}".format(
        repository.uid, repository.gid))
    ownership_spec = OwnershipSpecification(repository.uid, repository.gid)
    return MountSpecification(mounts, ownership_spec)
Пример #14
0
    def _internal_staged_pre_snapshot(self, request):
        """Pre Snapshot Wrapper for staged plugins.

        Executed before creating a snapshot. This plugin
        operation is run prior to creating a snapshot for a staged source.

        Run pre-snapshot operation for a staged source.

        Args:
            request (StagedPreSnapshotRequest): Pre Snapshot arguments.

        Returns:
            StagedPreSnapshotResponse: A response containing
                StagedPreSnapshotResult if successful or PluginErrorResult
                in case of an error.
        """
        # Reasoning for method imports are in this file's docstring.
        from generated.definitions import RepositoryDefinition
        from generated.definitions import LinkedSourceDefinition
        from generated.definitions import SourceConfigDefinition
        from generated.definitions import SnapshotParametersDefinition

        #
        # While linked.pre_snapshot() is not a required operation, this should
        # not be called if it wasn't implemented.
        #
        if not self.pre_snapshot_impl:
            raise OperationNotDefinedError(Op.LINKED_PRE_SNAPSHOT)

        linked_source = request.staged_source.linked_source
        staged_source_definition = (LinkedSourceDefinition.from_dict(
            json.loads(linked_source.parameters.json)))
        staged_mount = request.staged_source.staged_mount
        mount = Mount(remote_environment=RemoteEnvironment.from_proto(
            staged_mount.remote_environment),
                      mount_path=staged_mount.mount_path,
                      shared_path=staged_mount.shared_path)
        staged_source = StagedSource(
            guid=linked_source.guid,
            source_connection=RemoteConnection.from_proto(
                request.staged_source.source_connection),
            parameters=staged_source_definition,
            mount=mount,
            staged_connection=RemoteConnection.from_proto(
                request.staged_source.staged_connection))

        repository = RepositoryDefinition.from_dict(
            json.loads(request.repository.parameters.json))
        source_config = SourceConfigDefinition.from_dict(
            json.loads(request.source_config.parameters.json))
        snapshot_parameters = SnapshotParametersDefinition.from_dict(
            json.loads(request.snapshot_parameters.parameters.json))

        self.pre_snapshot_impl(staged_source=staged_source,
                               repository=repository,
                               source_config=source_config,
                               snapshot_parameters=snapshot_parameters)

        response = platform_pb2.StagedPreSnapshotResponse()
        response.return_value.CopyFrom(platform_pb2.StagedPreSnapshotResult())

        return response
Пример #15
0
def linked_mount_specification(staged_source, repository):
    mount_path = staged_source.parameters.mount_path
    environment = staged_source.staged_connection.environment
    linked.check_mount_path(staged_source, repository)
    mounts = [Mount(environment, mount_path)]
    return MountSpecification(mounts)
Пример #16
0
def virtual_mount_specification(virtual_source, repository):
    mount_path = virtual_source.parameters.mount_path
    mounts = [Mount(virtual_source.connection.environment, mount_path)]
    return MountSpecification(mounts)
Пример #17
0
 def _from_protobuf_single_subset_mount(single_subset_mount):
     return Mount(remote_environment=RemoteEnvironment.from_proto(
         single_subset_mount.remote_environment),
                  mount_path=single_subset_mount.mount_path,
                  shared_path=single_subset_mount.shared_path)
 def test_init_mount_incorrect_format_reference_string(reference_string):
     with pytest.raises(IncorrectReferenceFormatError) as err_info:
         Mount(reference_string, 'mount_path', 'shared_path')
     assert err_info.value.message == (
         "Reference '{}' is not a correctly formatted host"
         " environment reference.".format(reference_string))
 def test_init_mount_reference_string_success(reference_string):
     mount = Mount(reference_string, 'mount_path', 'shared_path')
     assert (mount.remote_environment.reference == reference_string and
             mount.remote_environment.host.reference == 'dummy reference')
 def test_init_mount_spec(remote_environment):
     mount = Mount(remote_environment, 'mount_path', 'shared_path')
     MountSpecification([mount], OwnershipSpecification(10, 10))
 def test_init_mount_bad_shared_path(remote_environment):
     with pytest.raises(IncorrectTypeError) as err_info:
         Mount(remote_environment, 'mount_path', 10000)
     assert err_info.value.message == (
         "Mount's parameter 'shared_path' was type 'int' but should"
         " be of type 'basestring' if defined.")
Пример #22
0
    def _internal_staged_post_snapshot(self, request):
        """Post Snapshot Wrapper for staged plugins.

        Executed after creating a snapshot. This plugin
        operation is run after creating a snapshot for a staged source.

        Run post-snapshot operation for a staged source.

        Args:
           request (StagedPostSnapshotRequest): Post Snapshot arguments.

        Returns:
            StagedPostSnapshotResponse: A response containing the return value
                StagedPostSnapshotResult which has the snapshot metadata on
                success. In case of errors, response object will contain
                PluginErrorResult.
        """
        # Reasoning for method imports are in this file's docstring.
        from generated.definitions import RepositoryDefinition
        from generated.definitions import LinkedSourceDefinition
        from generated.definitions import SourceConfigDefinition
        from generated.definitions import SnapshotDefinition
        from generated.definitions import SnapshotParametersDefinition

        def to_protobuf(snapshot):
            parameters = common_pb2.PluginDefinedObject()
            parameters.json = json.dumps(snapshot.to_dict())
            snapshot_protobuf = common_pb2.Snapshot()
            snapshot_protobuf.parameters.CopyFrom(parameters)
            return snapshot_protobuf

        if not self.post_snapshot_impl:
            raise OperationNotDefinedError(Op.LINKED_POST_SNAPSHOT)

        staged_source_definition = LinkedSourceDefinition.from_dict(
            json.loads(request.staged_source.linked_source.parameters.json))
        mount = Mount(
            remote_environment=RemoteEnvironment.from_proto(
                request.staged_source.staged_mount.remote_environment),
            mount_path=request.staged_source.staged_mount.mount_path,
            shared_path=request.staged_source.staged_mount.shared_path)
        staged_source = StagedSource(
            guid=request.staged_source.linked_source.guid,
            source_connection=RemoteConnection.from_proto(
                request.staged_source.source_connection),
            parameters=staged_source_definition,
            mount=mount,
            staged_connection=RemoteConnection.from_proto(
                request.staged_source.staged_connection))

        repository = RepositoryDefinition.from_dict(
            json.loads(request.repository.parameters.json))
        source_config = SourceConfigDefinition.from_dict(
            json.loads(request.source_config.parameters.json))
        snapshot_parameters = SnapshotParametersDefinition.from_dict(
            json.loads(request.snapshot_parameters.parameters.json))

        snapshot = self.post_snapshot_impl(
            staged_source=staged_source,
            repository=repository,
            source_config=source_config,
            snapshot_parameters=snapshot_parameters)

        # Validate that this is a SnapshotDefinition object
        if not isinstance(snapshot, SnapshotDefinition):
            raise IncorrectReturnTypeError(Op.LINKED_POST_SNAPSHOT,
                                           type(snapshot), SnapshotDefinition)

        response = platform_pb2.StagedPostSnapshotResponse()
        response.return_value.snapshot.CopyFrom(to_protobuf(snapshot))

        return response
Пример #23
0
def mount(environment):
    return Mount(environment, constants.mount_path, None)
Пример #24
0
    def _internal_status(self, request):
        """Staged Status Wrapper for staged plugins.

        Executed as part of several operations to get the status
        of a staged source - active or inactive.

        Run status operation for a staged source.

        Args:
           request (StagedStatusRequest): Post Snapshot arguments.

        Returns:
           StagedStatusResponse: A response containing the return value -
           StagedStatusResult which has active or inactive status. In
           case of errors, response object will contain PluginErrorResult.
        """
        # Reasoning for method imports are in this file's docstring.
        from generated.definitions import RepositoryDefinition
        from generated.definitions import LinkedSourceDefinition
        from generated.definitions import SourceConfigDefinition

        #
        # While linked.status() is not a required operation, this should
        # not be called if it wasn't implemented.
        #
        if not self.status_impl:
            raise OperationNotDefinedError(Op.LINKED_STATUS)

        staged_source_definition = LinkedSourceDefinition.from_dict(
            json.loads(request.staged_source.linked_source.parameters.json))
        mount = Mount(
            remote_environment=(RemoteEnvironment.from_proto(
                request.staged_source.staged_mount.remote_environment)),
            mount_path=request.staged_source.staged_mount.mount_path,
            shared_path=request.staged_source.staged_mount.shared_path)
        staged_source = StagedSource(
            guid=request.staged_source.linked_source.guid,
            source_connection=RemoteConnection.from_proto(
                request.staged_source.source_connection),
            parameters=staged_source_definition,
            mount=mount,
            staged_connection=RemoteConnection.from_proto(
                request.staged_source.staged_connection))

        repository = RepositoryDefinition.from_dict(
            json.loads(request.repository.parameters.json))
        source_config = SourceConfigDefinition.from_dict(
            json.loads(request.source_config.parameters.json))

        status = self.status_impl(staged_source=staged_source,
                                  repository=repository,
                                  source_config=source_config)

        # Validate that this is a Status object.
        if not isinstance(status, Status):
            raise IncorrectReturnTypeError(Op.LINKED_STATUS, type(status),
                                           Status)

        staged_status_response = platform_pb2.StagedStatusResponse()
        staged_status_response.return_value.status = status.value

        return staged_status_response
 def test_init_mount_success(remote_environment):
     Mount(remote_environment, 'mount_path', 'shared_path')
Пример #26
0
    def _internal_mount_specification(self, request):
        """Staged Mount/Ownership Spec Wrapper for staged plugins.

        Executed before creating a snapshot during sync or before
        enable/disable. This plugin operation is run before mounting datasets
        on staging to set the mount path and/or ownership.

        Run mount/ownership spec operation for a staged source.

        Args:
           request (StagedMountSpecRequest): Mount Spec arguments.

        Returns:
           StagedMountSpecResponse: A response containing the return value -
           StagedMountSpecResult which has the mount/ownership metadata on
           success. In case of errors, response object will contain
           PluginErrorResult.
        """
        # Reasoning for method imports are in this file's docstring.
        from generated.definitions import RepositoryDefinition
        from generated.definitions import LinkedSourceDefinition

        def to_protobuf_single_mount(single_mount):
            if single_mount.shared_path:
                raise PluginRuntimeError(
                    'Shared path is not supported for linked sources.')

            single_mount_protobuf = common_pb2.SingleEntireMount()
            single_mount_protobuf.mount_path = single_mount.mount_path
            single_mount_protobuf.remote_environment.CopyFrom(
                single_mount.remote_environment.to_proto())
            return single_mount_protobuf

        def to_protobuf_ownership_spec(ownership_spec):
            ownership_spec_protobuf = common_pb2.OwnershipSpec()
            ownership_spec_protobuf.uid = ownership_spec.uid
            ownership_spec_protobuf.gid = ownership_spec.gid
            return ownership_spec_protobuf

        if not self.mount_specification_impl:
            raise OperationNotDefinedError(Op.LINKED_MOUNT_SPEC)

        staged_source_definition = LinkedSourceDefinition.from_dict(
            json.loads(request.staged_source.linked_source.parameters.json))
        mount = Mount(
            remote_environment=(RemoteEnvironment.from_proto(
                request.staged_source.staged_mount.remote_environment)),
            mount_path=request.staged_source.staged_mount.mount_path,
            shared_path=request.staged_source.staged_mount.shared_path)
        staged_source = StagedSource(
            guid=request.staged_source.linked_source.guid,
            source_connection=RemoteConnection.from_proto(
                request.staged_source.source_connection),
            parameters=staged_source_definition,
            mount=mount,
            staged_connection=RemoteConnection.from_proto(
                request.staged_source.staged_connection))

        repository = RepositoryDefinition.from_dict(
            json.loads(request.repository.parameters.json))

        mount_spec = self.mount_specification_impl(staged_source=staged_source,
                                                   repository=repository)

        # Validate that this is a MountSpecification object.
        if not isinstance(mount_spec, MountSpecification):
            raise IncorrectReturnTypeError(Op.LINKED_MOUNT_SPEC,
                                           type(mount_spec),
                                           MountSpecification)

        # Only one mount is supported for linked sources.
        mount_len = len(mount_spec.mounts)
        if mount_len != 1:
            raise PluginRuntimeError(
                'Exactly one mount must be provided for staging sources.'
                ' Found {}'.format(mount_len))

        staged_mount = to_protobuf_single_mount(mount_spec.mounts[0])

        staged_mount_spec_response = platform_pb2.StagedMountSpecResponse()
        staged_mount_spec_response.return_value.staged_mount.CopyFrom(
            staged_mount)

        # Ownership spec is optional for linked sources.
        if mount_spec.ownership_specification:
            ownership_spec = to_protobuf_ownership_spec(
                mount_spec.ownership_specification)
            staged_mount_spec_response.return_value.ownership_spec.CopyFrom(
                ownership_spec)

        return staged_mount_spec_response
 def test_init_mount_bad_remote_env():
     with pytest.raises(IncorrectReferenceFormatError) as err_info:
         Mount('bad string', 'mount_path', 'shared_path')
     assert err_info.value.message == (
         "Reference 'bad string' is not a correctly formatted host "
         "environment reference.")