def _internal_worker(self, request): """Staged Worker Wrapper for staged plugins. Executed as part of validated sync. This plugin operation is run to sync staging source as part of the validated sync operation. Run worker operation for a staged source. Args: request (StagedWorkerRequest): Worker arguments. Returns: StagedWorkerResponse: A response containing StagedWorkerResult if successful or PluginErrorResult in case of an error. """ # Reasoning for method imports are in this file's docstring. from generated.definitions import RepositoryDefinition from generated.definitions import LinkedSourceDefinition from generated.definitions import SourceConfigDefinition # # While linked.worker() is not a required operation, this should # not be called if it wasn't implemented. # if not self.worker_impl: raise OperationNotDefinedError(Op.LINKED_WORKER) staged_source_definition = LinkedSourceDefinition.from_dict( json.loads(request.staged_source.linked_source.parameters.json)) mount = Mount( remote_environment=(RemoteEnvironment.from_proto( request.staged_source.staged_mount.remote_environment)), mount_path=request.staged_source.staged_mount.mount_path, shared_path=request.staged_source.staged_mount.shared_path) staged_source = StagedSource( guid=request.staged_source.linked_source.guid, source_connection=RemoteConnection.from_proto( request.staged_source.source_connection), parameters=staged_source_definition, mount=mount, staged_connection=RemoteConnection.from_proto( request.staged_source.staged_connection)) repository = RepositoryDefinition.from_dict( json.loads(request.repository.parameters.json)) source_config = SourceConfigDefinition.from_dict( json.loads(request.source_config.parameters.json)) self.worker_impl(staged_source=staged_source, repository=repository, source_config=source_config) staged_worker_response = platform_pb2.StagedWorkerResponse() staged_worker_response.return_value.CopyFrom( platform_pb2.StagedWorkerResult()) return staged_worker_response
def _internal_mount_specification(self, request): """Staged Mount/Ownership Spec Wrapper for staged plugins. Executed before creating a snapshot during sync or before enable/disable. This plugin operation is run before mounting datasets on staging to set the mount path and/or ownership. Run mount/ownership spec operation for a staged source. Args: request (StagedMountSpecRequest): Mount Spec arguments. Returns: StagedMountSpecResponse: A response containing the return value - StagedMountSpecResult which has the mount/ownership metadata on success. In case of errors, response object will contain PluginErrorResult. """ # Reasoning for method imports are in this file's docstring. from generated.definitions import RepositoryDefinition from generated.definitions import LinkedSourceDefinition def to_protobuf_single_mount(single_mount): if single_mount.shared_path: raise PluginRuntimeError( 'Shared path is not supported for linked sources.') single_mount_protobuf = common_pb2.SingleEntireMount() single_mount_protobuf.mount_path = single_mount.mount_path single_mount_protobuf.remote_environment.CopyFrom( single_mount.remote_environment.to_proto()) return single_mount_protobuf def to_protobuf_ownership_spec(ownership_spec): ownership_spec_protobuf = common_pb2.OwnershipSpec() ownership_spec_protobuf.uid = ownership_spec.uid ownership_spec_protobuf.gid = ownership_spec.gid return ownership_spec_protobuf if not self.mount_specification_impl: raise OperationNotDefinedError(Op.LINKED_MOUNT_SPEC) staged_source_definition = LinkedSourceDefinition.from_dict( json.loads(request.staged_source.linked_source.parameters.json)) mount = Mount( remote_environment=(RemoteEnvironment.from_proto( request.staged_source.staged_mount.remote_environment)), mount_path=request.staged_source.staged_mount.mount_path, shared_path=request.staged_source.staged_mount.shared_path) staged_source = StagedSource( guid=request.staged_source.linked_source.guid, source_connection=RemoteConnection.from_proto( request.staged_source.source_connection), parameters=staged_source_definition, mount=mount, staged_connection=RemoteConnection.from_proto( request.staged_source.staged_connection)) repository = RepositoryDefinition.from_dict( json.loads(request.repository.parameters.json)) mount_spec = self.mount_specification_impl(staged_source=staged_source, repository=repository) # Validate that this is a MountSpecification object. if not isinstance(mount_spec, MountSpecification): raise IncorrectReturnTypeError(Op.LINKED_MOUNT_SPEC, type(mount_spec), MountSpecification) # Only one mount is supported for linked sources. mount_len = len(mount_spec.mounts) if mount_len != 1: raise PluginRuntimeError( 'Exactly one mount must be provided for staging sources.' ' Found {}'.format(mount_len)) staged_mount = to_protobuf_single_mount(mount_spec.mounts[0]) staged_mount_spec_response = platform_pb2.StagedMountSpecResponse() staged_mount_spec_response.return_value.staged_mount.CopyFrom( staged_mount) # Ownership spec is optional for linked sources. if mount_spec.ownership_specification: ownership_spec = to_protobuf_ownership_spec( mount_spec.ownership_specification) staged_mount_spec_response.return_value.ownership_spec.CopyFrom( ownership_spec) return staged_mount_spec_response
def _internal_status(self, request): """Staged Status Wrapper for staged plugins. Executed as part of several operations to get the status of a staged source - active or inactive. Run status operation for a staged source. Args: request (StagedStatusRequest): Post Snapshot arguments. Returns: StagedStatusResponse: A response containing the return value - StagedStatusResult which has active or inactive status. In case of errors, response object will contain PluginErrorResult. """ # Reasoning for method imports are in this file's docstring. from generated.definitions import RepositoryDefinition from generated.definitions import LinkedSourceDefinition from generated.definitions import SourceConfigDefinition # # While linked.status() is not a required operation, this should # not be called if it wasn't implemented. # if not self.status_impl: raise OperationNotDefinedError(Op.LINKED_STATUS) staged_source_definition = LinkedSourceDefinition.from_dict( json.loads(request.staged_source.linked_source.parameters.json)) mount = Mount( remote_environment=(RemoteEnvironment.from_proto( request.staged_source.staged_mount.remote_environment)), mount_path=request.staged_source.staged_mount.mount_path, shared_path=request.staged_source.staged_mount.shared_path) staged_source = StagedSource( guid=request.staged_source.linked_source.guid, source_connection=RemoteConnection.from_proto( request.staged_source.source_connection), parameters=staged_source_definition, mount=mount, staged_connection=RemoteConnection.from_proto( request.staged_source.staged_connection)) repository = RepositoryDefinition.from_dict( json.loads(request.repository.parameters.json)) source_config = SourceConfigDefinition.from_dict( json.loads(request.source_config.parameters.json)) status = self.status_impl(staged_source=staged_source, repository=repository, source_config=source_config) # Validate that this is a Status object. if not isinstance(status, Status): raise IncorrectReturnTypeError(Op.LINKED_STATUS, type(status), Status) staged_status_response = platform_pb2.StagedStatusResponse() staged_status_response.return_value.status = status.value return staged_status_response
def _internal_staged_post_snapshot(self, request): """Post Snapshot Wrapper for staged plugins. Executed after creating a snapshot. This plugin operation is run after creating a snapshot for a staged source. Run post-snapshot operation for a staged source. Args: request (StagedPostSnapshotRequest): Post Snapshot arguments. Returns: StagedPostSnapshotResponse: A response containing the return value StagedPostSnapshotResult which has the snapshot metadata on success. In case of errors, response object will contain PluginErrorResult. """ # Reasoning for method imports are in this file's docstring. from generated.definitions import RepositoryDefinition from generated.definitions import LinkedSourceDefinition from generated.definitions import SourceConfigDefinition from generated.definitions import SnapshotDefinition from generated.definitions import SnapshotParametersDefinition def to_protobuf(snapshot): parameters = common_pb2.PluginDefinedObject() parameters.json = json.dumps(snapshot.to_dict()) snapshot_protobuf = common_pb2.Snapshot() snapshot_protobuf.parameters.CopyFrom(parameters) return snapshot_protobuf if not self.post_snapshot_impl: raise OperationNotDefinedError(Op.LINKED_POST_SNAPSHOT) staged_source_definition = LinkedSourceDefinition.from_dict( json.loads(request.staged_source.linked_source.parameters.json)) mount = Mount( remote_environment=RemoteEnvironment.from_proto( request.staged_source.staged_mount.remote_environment), mount_path=request.staged_source.staged_mount.mount_path, shared_path=request.staged_source.staged_mount.shared_path) staged_source = StagedSource( guid=request.staged_source.linked_source.guid, source_connection=RemoteConnection.from_proto( request.staged_source.source_connection), parameters=staged_source_definition, mount=mount, staged_connection=RemoteConnection.from_proto( request.staged_source.staged_connection)) repository = RepositoryDefinition.from_dict( json.loads(request.repository.parameters.json)) source_config = SourceConfigDefinition.from_dict( json.loads(request.source_config.parameters.json)) snapshot_parameters = SnapshotParametersDefinition.from_dict( json.loads(request.snapshot_parameters.parameters.json)) snapshot = self.post_snapshot_impl( staged_source=staged_source, repository=repository, source_config=source_config, snapshot_parameters=snapshot_parameters) # Validate that this is a SnapshotDefinition object if not isinstance(snapshot, SnapshotDefinition): raise IncorrectReturnTypeError(Op.LINKED_POST_SNAPSHOT, type(snapshot), SnapshotDefinition) response = platform_pb2.StagedPostSnapshotResponse() response.return_value.snapshot.CopyFrom(to_protobuf(snapshot)) return response
def _internal_staged_pre_snapshot(self, request): """Pre Snapshot Wrapper for staged plugins. Executed before creating a snapshot. This plugin operation is run prior to creating a snapshot for a staged source. Run pre-snapshot operation for a staged source. Args: request (StagedPreSnapshotRequest): Pre Snapshot arguments. Returns: StagedPreSnapshotResponse: A response containing StagedPreSnapshotResult if successful or PluginErrorResult in case of an error. """ # Reasoning for method imports are in this file's docstring. from generated.definitions import RepositoryDefinition from generated.definitions import LinkedSourceDefinition from generated.definitions import SourceConfigDefinition from generated.definitions import SnapshotParametersDefinition # # While linked.pre_snapshot() is not a required operation, this should # not be called if it wasn't implemented. # if not self.pre_snapshot_impl: raise OperationNotDefinedError(Op.LINKED_PRE_SNAPSHOT) linked_source = request.staged_source.linked_source staged_source_definition = (LinkedSourceDefinition.from_dict( json.loads(linked_source.parameters.json))) staged_mount = request.staged_source.staged_mount mount = Mount(remote_environment=RemoteEnvironment.from_proto( staged_mount.remote_environment), mount_path=staged_mount.mount_path, shared_path=staged_mount.shared_path) staged_source = StagedSource( guid=linked_source.guid, source_connection=RemoteConnection.from_proto( request.staged_source.source_connection), parameters=staged_source_definition, mount=mount, staged_connection=RemoteConnection.from_proto( request.staged_source.staged_connection)) repository = RepositoryDefinition.from_dict( json.loads(request.repository.parameters.json)) source_config = SourceConfigDefinition.from_dict( json.loads(request.source_config.parameters.json)) snapshot_parameters = SnapshotParametersDefinition.from_dict( json.loads(request.snapshot_parameters.parameters.json)) self.pre_snapshot_impl(staged_source=staged_source, repository=repository, source_config=source_config, snapshot_parameters=snapshot_parameters) response = platform_pb2.StagedPreSnapshotResponse() response.return_value.CopyFrom(platform_pb2.StagedPreSnapshotResult()) return response
def staged_source(source_connection, staged_connection, staged_parameters_xdcr, mount): staged_source = StagedSource(constants.UID, source_connection, staged_parameters_xdcr, mount, staged_connection) return staged_source