def _assert_nuclio_v3io_mount(self, local_path, remote_path):
args, _ = nuclio.deploy.deploy_config.call_args
deploy_spec = args[0]["spec"]
container, path = split_path(remote_path)
expected_volume = {
"volume": {
"flexVolume": {
"driver": "v3io/fuse",
"options": {
"accessKey": self.v3io_access_key,
"container": container,
"subPath": path,
},
},
"name": "v3io",
},
"volumeMount": {
"mountPath": local_path,
"name": "v3io",
"subPath": ""
},
}
assert (deepdiff.DeepDiff(deploy_spec["volumes"], [expected_volume],
ignore_order=True) == {})
env_config = deploy_spec["env"]
expected_env = {
"V3IO_ACCESS_KEY": self.v3io_access_key,
"V3IO_USERNAME": self.v3io_user,
"V3IO_API": None,
"MLRUN_NAMESPACE": self.namespace,
}
self._assert_pod_env(env_config, expected_env)
def _get_expected_struct_for_v3io_trigger(self, parameters):
container, path = split_path(parameters["stream_path"])
# Remove leading / in the path
path = path[1:]
# TODO - Not sure what happens to the "shards" parameter. Seems to be dropped along the way?
return {
"kind": "v3ioStream",
"name": parameters["name"],
"password": self.v3io_access_key,
"attributes": {
"containerName": container,
"streamPath": path,
"consumerGroup": parameters["group"],
"seekTo": parameters["seek_to"],
},
}
def __init__(self, state, resource):
import v3io
self._v3io_client = v3io.dataplane.Client()
uri = stream_uri(state._root, resource.uri, state.resource)
self._container, self._stream_path = split_path(uri)
def deploy_pipline(pipeline):
print('deploy:')
for name, resource in pipeline.resources.items():
print(name, resource.get_inputs())
if resource.skip_deploy:
continue
kind = resource.kind
if kind == 'function':
if not resource.uri:
continue
function = import_function(url=resource.uri)
function.metadata.project = pipeline.project
function.metadata.name = name
if resource.spec:
for attribute in [
"volumes",
"volume_mounts",
"env",
"resources",
"image_pull_policy",
"replicas",
]:
value = resource.spec.get(attribute, None)
if value and hasattr(function.spec, attribute):
setattr(function.spec, attribute, value)
function.set_env('V3IO_API',
os.environ.get("V3IO_API", 'v3io-webapi:8081'))
function.set_env('V3IO_ACCESS_KEY',
os.environ.get("V3IO_ACCESS_KEY", ''))
function.set_env('RESOURCE_NAME', name)
function.set_env('PIPELINE_SPEC_ENV', pipeline.to_json())
if resource.spec and resource.spec.get('with_v3io', ''):
function.apply(mount_v3io())
for source, source_state, target_state in resource.get_inputs():
source_resource = pipeline.resources[source]
if source_resource.kind == 'stream':
uri = stream_uri(pipeline, source_resource.uri, source)
function.set_env('FROM_STATE', target_state)
add_stream_trigger(function, uri)
print(name, resource)
print(function.to_yaml())
elif kind == 'stream':
import v3io
v3io_client = v3io.dataplane.Client()
container, stream_path = split_path(
stream_uri(pipeline, resource.uri, name))
print(f'stream path: {container}{stream_path}')
response = v3io_client.create_stream(
container=container,
path=stream_path,
shard_count=resource.spec.get('shards', 1),
retention_period_hours=resource.spec.get(
'retention_hours', 24),
raise_for_status=v3io.dataplane.RaiseForStatus.never,
)
print(response.status_code, response.body)
if not (response.status_code == 400
and "ResourceInUse" in str(response.body)):
response.raise_for_status([409, 204])
