def _assert_nuclio_v3io_mount(self, local_path, remote_path): args, _ = nuclio.deploy.deploy_config.call_args deploy_spec = args[0]["spec"] container, path = split_path(remote_path) expected_volume = { "volume": { "flexVolume": { "driver": "v3io/fuse", "options": { "accessKey": self.v3io_access_key, "container": container, "subPath": path, }, }, "name": "v3io", }, "volumeMount": { "mountPath": local_path, "name": "v3io", "subPath": "" }, } assert (deepdiff.DeepDiff(deploy_spec["volumes"], [expected_volume], ignore_order=True) == {}) env_config = deploy_spec["env"] expected_env = { "V3IO_ACCESS_KEY": self.v3io_access_key, "V3IO_USERNAME": self.v3io_user, "V3IO_API": None, "MLRUN_NAMESPACE": self.namespace, } self._assert_pod_env(env_config, expected_env)
def _get_expected_struct_for_v3io_trigger(self, parameters): container, path = split_path(parameters["stream_path"]) # Remove leading / in the path path = path[1:] # TODO - Not sure what happens to the "shards" parameter. Seems to be dropped along the way? return { "kind": "v3ioStream", "name": parameters["name"], "password": self.v3io_access_key, "attributes": { "containerName": container, "streamPath": path, "consumerGroup": parameters["group"], "seekTo": parameters["seek_to"], }, }
def __init__(self, state, resource): import v3io self._v3io_client = v3io.dataplane.Client() uri = stream_uri(state._root, resource.uri, state.resource) self._container, self._stream_path = split_path(uri)
def deploy_pipline(pipeline): print('deploy:') for name, resource in pipeline.resources.items(): print(name, resource.get_inputs()) if resource.skip_deploy: continue kind = resource.kind if kind == 'function': if not resource.uri: continue function = import_function(url=resource.uri) function.metadata.project = pipeline.project function.metadata.name = name if resource.spec: for attribute in [ "volumes", "volume_mounts", "env", "resources", "image_pull_policy", "replicas", ]: value = resource.spec.get(attribute, None) if value and hasattr(function.spec, attribute): setattr(function.spec, attribute, value) function.set_env('V3IO_API', os.environ.get("V3IO_API", 'v3io-webapi:8081')) function.set_env('V3IO_ACCESS_KEY', os.environ.get("V3IO_ACCESS_KEY", '')) function.set_env('RESOURCE_NAME', name) function.set_env('PIPELINE_SPEC_ENV', pipeline.to_json()) if resource.spec and resource.spec.get('with_v3io', ''): function.apply(mount_v3io()) for source, source_state, target_state in resource.get_inputs(): source_resource = pipeline.resources[source] if source_resource.kind == 'stream': uri = stream_uri(pipeline, source_resource.uri, source) function.set_env('FROM_STATE', target_state) add_stream_trigger(function, uri) print(name, resource) print(function.to_yaml()) elif kind == 'stream': import v3io v3io_client = v3io.dataplane.Client() container, stream_path = split_path( stream_uri(pipeline, resource.uri, name)) print(f'stream path: {container}{stream_path}') response = v3io_client.create_stream( container=container, path=stream_path, shard_count=resource.spec.get('shards', 1), retention_period_hours=resource.spec.get( 'retention_hours', 24), raise_for_status=v3io.dataplane.RaiseForStatus.never, ) print(response.status_code, response.body) if not (response.status_code == 400 and "ResourceInUse" in str(response.body)): response.raise_for_status([409, 204])