def test_initiate_upload(fs: V3ioFS, tmp_obj):
fs.touch(tmp_obj.path)
assert fs.exists(tmp_obj.path)
v3f = V3ioFile(fs, tmp_obj.path, 'wb')
v3f._initiate_upload()
assert not fs.exists(tmp_obj.path)
# should not fail even if the file does not exist
v3f._initiate_upload()
def test_ls(fs: V3ioFS, new_file, path_cls):
new_file(fs._client, f'{test_dir}/test-file') # Make sure dir exists
path = path_cls(f'/{test_container}/{test_dir}/')
out = fs.ls(path)
assert len(out) > 0, 'nothing found'
assert all(isinstance(p, dict) for p in out), 'not dict'
out = fs.ls(path, detail=False)
assert len(out) > 0, 'nothing found'
assert all(isinstance(p, str) for p in out), 'not string'
def test_glob(fs: V3ioFS, tmp_obj):
assert fs.glob("bigdata/v3io-fs-test") == ["/bigdata/v3io-fs-test"]
outfiles = fs.glob("bigdata/v3io-fs-test/")
outfiles = [f for f in outfiles if 'iguazio' not in f]
assert outfiles == [
"/bigdata/v3io-fs-test/a",
"/bigdata/v3io-fs-test/b",
"/bigdata/v3io-fs-test/test-file",
]
outfiles2 = fs.glob("bigdata/v3io-fs-test/*")
outfiles2 = [f for f in outfiles if 'iguazio' not in f]
assert outfiles2 == [
"/bigdata/v3io-fs-test/a",
"/bigdata/v3io-fs-test/b",
"/bigdata/v3io-fs-test/test-file",
]
def tmp_obj():
user, ts = getuser(), datetime.now().isoformat()
client = V3ioFS(v3io_api=host, v3io_access_key=access_key)._client
path = f'{test_dir}/{user}-test-{ts}'
body = f'test data for {user} at {ts}'.encode()
resp = client.put_object(test_container, path, body=body)
assert resp.status_code == HTTPStatus.OK, 'create failed'
body = datetime.now().isoformat().encode('utf-8')
# Add 2nd-level object
path2 = f'{test_dir}/test-file'
resp = client.put_object(test_container, path2, body=body)
assert resp.status_code == HTTPStatus.OK, 'create path2 failed'
path3 = f'{test_dir}/a/file.txt'
resp = client.put_object(test_container, path3, body=body)
assert resp.status_code == HTTPStatus.OK, 'create path3 failed'
path4 = f'{test_dir}/b/file2.txt'
resp = client.put_object(test_container, path4, body=body)
assert resp.status_code == HTTPStatus.OK, 'create path4 failed'
path5 = f'{test_dir}/a/file2.txt'
resp = client.put_object(test_container, path5, body=body)
assert resp.status_code == HTTPStatus.OK, 'create path5 failed'
yield Obj(f'/{test_container}/{path}', body)
client.delete_object(test_container, path)
def tree():
fs = V3ioFS()
create_tree(fs, tree_root, tree_data)
yield Tree(tree_root, tree_data)
fs.rm(tree_root, recursive=True)
fs._client.close()
def tmp_file():
client = V3ioFS(v3io_api=host, v3io_access_key=access_key)._client
path = f'{test_dir}/test_file.txt'
body = 'In god we trust; all others must bring data.'.encode()
resp = client.put_object(test_container, path, body=body)
assert resp.status_code == HTTPStatus.OK, 'create failed'
yield Obj(f'/{test_container}/{path}', body)
client.delete_object(test_container, path)
def test_upload_chunk(fs: V3ioFS, tmp_obj):
v3f = V3ioFile(fs, tmp_obj.path, 'ab')
chunk = b'::chunk of data'
v3f.buffer.write(chunk)
v3f._upload_chunk()
expected = tmp_obj.data + chunk
with fs.open(tmp_obj.path, 'rb') as fp:
data = fp.read()
assert expected == data, 'bad data'
def fs():
fs = V3ioFS()
yield fs
fs._client.close()
from .model.base import DataTargetBase, DataSource
from .retrieval import LocalFeatureMerger, init_feature_vector_graph
from .ingestion import init_featureset_graph, deploy_ingestion_function
from .model import FeatureVector, FeatureSet, OnlineVectorService, OfflineVectorResponse
from .targets import get_default_targets
from ..runtimes.function_reference import FunctionReference
from ..utils import get_caller_globals
from ..features import infer_schema_from_df, InferOptions, get_df_stats, get_df_preview
_v3iofs = None
try:
# add v3io:// path prefix support to pandas & dask, todo: move to datastores
from v3iofs import V3ioFS
_v3iofs = V3ioFS()
except Exception:
pass
def _features_to_vector(features):
if isinstance(features, str):
vector = get_feature_vector_by_uri(features)
elif isinstance(features, list):
vector = FeatureVector(features=features)
elif isinstance(features, FeatureVector):
vector = features
else:
raise mlrun.errors.MLRunInvalidArgumentError(
"illegal features value/type")
return vector
def test_touch(fs: V3ioFS, tmp_obj):
path = tmp_obj.path
fs.touch(path)
container, path = split_container(path)
resp = fs._client.get_object(container, path)
assert resp.body == b'', 'not truncated'
def test_rm(fs: V3ioFS, tmp_obj):
path = tmp_obj.path
fs.rm(path)
out = fs.ls(dirname(path), detail=False)
assert path not in out, 'not deleted'
def test_ls(fs: V3ioFS, tmp_obj):
path = f'/{test_container}/{test_dir}/'
out0 = fs.ls(path)
assert len(out0) > 0, 'nothing found'
out1 = fs.ls(path, detail=False)
assert len(out1) > 0, 'nothing found'
assert all(isinstance(p, str) for p in out1), 'not string'
out1 = [f for f in out1 if 'iguazio' not in f]
assert set(out1) == set([
'/bigdata/v3io-fs-test/test-file', '/bigdata/v3io-fs-test/a',
"/bigdata/v3io-fs-test/b"
])
out2 = fs.ls('bigdata/v3io-fs-test/a', detail=False)
assert set(out2) == set([
'/bigdata/v3io-fs-test/a/file.txt', '/bigdata/v3io-fs-test/a/file2.txt'
])
path = f'/{test_container}/{test_dir}/test-file'
out3 = fs.ls(path, detail=True)
assert len(out3) > 0, 'nothing found'
assert out3 == [{
'name': '/bigdata/v3io-fs-test/test-file',
'size': 26,
'type': 'file'
}]
out4 = fs.ls('bigdata/v3io-fs-test/a', detail=True)
assert out4 == [
{
'name': '/bigdata/v3io-fs-test/a/file.txt',
'size': 26,
'type': 'file'
},
{
'name': '/bigdata/v3io-fs-test/a/file2.txt',
'size': 26,
'type': 'file'
},
]
out5 = fs.ls('/bigdata/v3io-fs-test', detail=True)
out5 = [f for f in out5 if 'iguazio' not in f['name']]
assert len(out5) > 0, 'nothing found'
assert out5 == [
{
'name': '/bigdata/v3io-fs-test/a',
'size': 0,
'type': 'directory'
},
{
'name': '/bigdata/v3io-fs-test/b',
'size': 0,
'type': 'directory'
},
{
'name': '/bigdata/v3io-fs-test/test-file',
'size': 26,
'type': 'file'
},
]