async def test_dsm_datcore(postgres_service_url, dsm_fixture,
datcore_structured_testbucket):
if not has_datcore_tokens():
return
utils.create_tables(url=postgres_service_url)
dsm = dsm_fixture
user_id = "0"
data = await dsm.list_files(user_id=user_id,
location=DATCORE_STR,
uuid_filter=BUCKET_NAME)
# the fixture creates 3 files
assert len(data) == 3
# delete the first one
fmd_to_delete = data[0].fmd
print("Deleting", fmd_to_delete.bucket_name, fmd_to_delete.object_name)
is_deleted = await dsm.delete_file(user_id, DATCORE_STR,
fmd_to_delete.file_id)
assert is_deleted
import time
time.sleep(1) # FIXME: takes some time to delete!!
data = await dsm.list_files(user_id=user_id,
location=DATCORE_STR,
uuid_filter=BUCKET_NAME)
assert len(data) == 2
async def test_datcore_list_files_raw(loop):
if not utils.has_datcore_tokens():
return
api_token = os.environ.get("BF_API_KEY", "none")
api_secret = os.environ.get("BF_API_SECRET", "none")
pool = ThreadPoolExecutor(2)
dcw = DatcoreWrapper(api_token, api_secret, loop, pool)
f = await dcw.list_files_raw()
assert len(f)
async def test_datcore_ping(loop):
if not utils.has_datcore_tokens():
return
api_token = os.environ.get("BF_API_KEY", "none")
api_secret = os.environ.get("BF_API_SECRET", "none")
pool = ThreadPoolExecutor(2)
dcw = DatcoreWrapper(api_token, api_secret, loop, pool)
responsive = await dcw.ping()
assert responsive
async def test_dsm_list_datasets_datcore(dsm_fixture,
datcore_structured_testbucket):
if not has_datcore_tokens():
return
datasets = await dsm_fixture.list_datasets(user_id=USER_ID,
location=DATCORE_STR)
assert len(datasets)
assert any(BUCKET_NAME in d.display_name for d in datasets)
async def test_dsm_s3_to_datcore(
postgres_service_url,
s3_client,
mock_files_factory,
dsm_fixture,
datcore_structured_testbucket,
):
if not has_datcore_tokens():
return
utils.create_tables(url=postgres_service_url)
tmp_file = mock_files_factory(1)[0]
fmd = _create_file_meta_for_s3(postgres_service_url, s3_client, tmp_file)
dsm = dsm_fixture
up_url = await dsm.upload_link(fmd.user_id, fmd.file_uuid)
with io.open(tmp_file, "rb") as fp:
d = fp.read()
req = urllib.request.Request(up_url, data=d, method="PUT")
with urllib.request.urlopen(req) as _f:
pass
# given the fmd, upload to datcore
tmp_file2 = tmp_file + ".fordatcore"
user_id = USER_ID
down_url = await dsm.download_link_s3(fmd.file_uuid)
urllib.request.urlretrieve(down_url, tmp_file2)
assert filecmp.cmp(tmp_file2, tmp_file)
# now we have the file locally, upload the file
await dsm.upload_file_to_datcore(
user_id=user_id,
local_file_path=tmp_file2,
destination_id=datcore_structured_testbucket["dataset_id"],
)
# and into a deeper strucutre
await dsm.upload_file_to_datcore(
user_id=user_id,
local_file_path=tmp_file2,
destination_id=datcore_structured_testbucket["coll2_id"],
)
# FIXME: upload takes some time
import time
time.sleep(1)
data = await dsm.list_files(user_id=user_id,
location=DATCORE_STR,
uuid_filter=BUCKET_NAME)
# there should now be 5 files
assert len(data) == 5
async def test_datcore_nested_download_link(loop):
if not utils.has_datcore_tokens():
return
api_token = os.environ.get("BF_API_KEY", "none")
api_secret = os.environ.get("BF_API_SECRET", "none")
pool = ThreadPoolExecutor(2)
dcw = DatcoreWrapper(api_token, api_secret, loop, pool)
destination = str(Path("Shared Data/ISAN/UCDavis use case 0D/inputs/"))
filename = "initial_WTstates.txt"
f = await dcw.download_link(destination, filename)
assert f
async def test_locations(client):
user_id = USER_ID
resp = await client.get("/v0/locations?user_id={}".format(user_id))
payload = await resp.json()
assert resp.status == 200, str(payload)
data, error = tuple(payload.get(k) for k in ("data", "error"))
_locs = 2 if has_datcore_tokens() else 1
assert len(data) == _locs
assert not error
async def test_dsm_datcore_to_S3(
postgres_service_url,
s3_client,
dsm_fixture,
mock_files_factory,
datcore_structured_testbucket,
):
if not has_datcore_tokens():
return
utils.create_tables(url=postgres_service_url)
# create temporary file
tmp_file = mock_files_factory(1)[0]
dest_fmd = _create_file_meta_for_s3(postgres_service_url, s3_client,
tmp_file)
user_id = dest_fmd.user_id
dest_uuid = dest_fmd.file_uuid
dsm = dsm_fixture
s3_data = await dsm.list_files(user_id=user_id, location=SIMCORE_S3_STR)
assert len(s3_data) == 0
dc_data = await dsm.list_files(user_id=user_id,
location=DATCORE_STR,
uuid_filter=BUCKET_NAME)
assert len(dc_data) == 3
src_fmd = dc_data[0]
await dsm.copy_file(
user_id=user_id,
dest_location=SIMCORE_S3_STR,
dest_uuid=dest_uuid,
source_location=DATCORE_STR,
source_uuid=datcore_structured_testbucket["file_id1"],
)
s3_data = await dsm.list_files(user_id=user_id, location=SIMCORE_S3_STR)
assert len(s3_data) == 1
# now download the original file
tmp_file1 = tmp_file + ".fromdatcore"
down_url_dc, filename = await dsm.download_link_datcore(
user_id, datcore_structured_testbucket["file_id1"])
urllib.request.urlretrieve(down_url_dc, tmp_file1)
# and the one on s3
tmp_file2 = tmp_file + ".fromS3"
down_url_s3 = await dsm.download_link_s3(dest_uuid)
urllib.request.urlretrieve(down_url_s3, tmp_file2)
assert filecmp.cmp(tmp_file1, tmp_file2)
async def test_copy_datcore(
postgres_service_url,
s3_client,
dsm_fixture,
mock_files_factory,
datcore_structured_testbucket,
):
if not has_datcore_tokens():
return
utils.create_tables(url=postgres_service_url)
# the fixture should provide 3 files
dsm = dsm_fixture
user_id = USER_ID
data = await dsm.list_files(user_id=user_id,
location=DATCORE_STR,
uuid_filter=BUCKET_NAME)
assert len(data) == 3
# create temporary file and upload to s3
tmp_file = mock_files_factory(1)[0]
fmd = _create_file_meta_for_s3(postgres_service_url, s3_client, tmp_file)
up_url = await dsm.upload_link(fmd.user_id, fmd.file_uuid)
with io.open(tmp_file, "rb") as fp:
d = fp.read()
req = urllib.request.Request(up_url, data=d, method="PUT")
with urllib.request.urlopen(req) as _f:
pass
# now copy to datcore
dat_core_uuid = os.path.join(BUCKET_NAME, fmd.file_name)
await dsm.copy_file(
user_id=user_id,
dest_location=DATCORE_STR,
dest_uuid=datcore_structured_testbucket["coll2_id"],
source_location=SIMCORE_S3_STR,
source_uuid=fmd.file_uuid,
)
data = await dsm.list_files(user_id=user_id,
location=DATCORE_STR,
uuid_filter=BUCKET_NAME)
# there should now be 4 files
assert len(data) == 4
async def test_copy(client, dsm_mockup_db, datcore_structured_testbucket):
if not has_datcore_tokens():
return
# copy N files
N = 2
counter = 0
for d in dsm_mockup_db.keys():
fmd = dsm_mockup_db[d]
source_uuid = fmd.file_uuid
datcore_id = datcore_structured_testbucket["coll1_id"]
resp = await client.put(
"/v0/locations/1/files/{}?user_id={}&extra_location={}&extra_source={}"
.format(
quote(datcore_id, safe=""),
fmd.user_id,
SIMCORE_S3_ID,
quote(source_uuid, safe=""),
))
payload = await resp.json()
assert resp.status == 200, str(payload)
data, error = tuple(payload.get(k) for k in ("data", "error"))
assert not error
assert data
counter = counter + 1
if counter == N:
break
# list files for every user
user_id = USER_ID
resp = await client.get(
"/v0/locations/1/files/metadata?user_id={}&uuid_filter={}".format(
user_id, BUCKET_NAME))
payload = await resp.json()
assert resp.status == 200, str(payload)
data, error = tuple(payload.get(k) for k in ("data", "error"))
assert not error
assert len(data) > N
async def test_dsm_datcore_to_local(postgres_service_url, dsm_fixture,
mock_files_factory,
datcore_structured_testbucket):
if not has_datcore_tokens():
return
utils.create_tables(url=postgres_service_url)
dsm = dsm_fixture
user_id = USER_ID
data = await dsm.list_files(user_id=user_id,
location=DATCORE_STR,
uuid_filter=BUCKET_NAME)
assert len(data)
url, filename = await dsm.download_link_datcore(
user_id, datcore_structured_testbucket["file_id1"])
tmp_file = mock_files_factory(1)[0]
tmp_file2 = tmp_file + ".fromdatcore"
urllib.request.urlretrieve(url, tmp_file2)
assert filecmp.cmp(tmp_file2, tmp_file)
import tempfile
import typing
from pathlib import Path
from blackfynn import Blackfynn
from blackfynn.models import Collection
import utils
from simcore_service_storage.datcore import DatcoreClient
from simcore_service_storage.models import FileMetaData
dir_path = os.path.dirname(os.path.realpath(__file__))
api_token = os.environ.get("BF_API_KEY")
api_secret = os.environ.get("BF_API_SECRET")
if utils.has_datcore_tokens():
client = DatcoreClient(api_token=api_token, api_secret=api_secret)
api_secret = os.environ.get("BF_API_SECRET", "none")
destination = str(Path("MaG/level1/level2"))
fd, path = tempfile.mkstemp()
try:
with os.fdopen(fd, "w") as tmp:
# do stuff with temp file
tmp.write("stuff")
f = client.upload_file(destination, path)
f = client.delete_file(destination, Path(path).name)
finally:
os.remove(path)
async def test_deep_copy_project_simcore_s3(dsm_fixture, s3_client,
postgres_service_url,
datcore_structured_testbucket):
if not has_datcore_tokens():
return
dsm = dsm_fixture
utils.create_full_tables(url=postgres_service_url)
path_in_datcore = datcore_structured_testbucket["file_id3"]
file_name_in_datcore = Path(
datcore_structured_testbucket["filename3"]).name
user_id = USER_ID
source_project = {
"uuid": "de2578c5-431e-4d5e-b80e-401c8066782f",
"name": "ISAN: 2D Plot",
"description": "2D RawGraphs viewer with one input",
"thumbnail": "",
"prjOwner": "*****@*****.**",
"creationDate": "2019-05-24T10:36:57.813Z",
"lastChangeDate": "2019-05-24T11:36:12.015Z",
"workbench": {
"de2578c5-431e-48eb-a9d2-aaad6b72400a": {
"key": "simcore/services/frontend/file-picker",
"version": "1.0.0",
"label": "File Picker",
"inputs": {},
"inputNodes": [],
"outputs": {
"outFile": {
"store": 1,
"path":
"N:package:ab8c214d-a596-401f-a90c-9c50e3c048b0",
}
},
"progress": 100,
"thumbnail": "",
"position": {
"x": 100,
"y": 100
},
},
"de2578c5-431e-4c63-a705-03a2c339646c": {
"key": "simcore/services/dynamic/raw-graphs",
"version": "2.8.0",
"label": "2D plot",
"inputs": {
"input_1": {
"nodeUuid": "de2578c5-431e-48eb-a9d2-aaad6b72400a",
"output": "outFile",
}
},
"inputNodes": ["de2578c5-431e-48eb-a9d2-aaad6b72400a"],
"outputs": {},
"progress": 0,
"thumbnail": "",
"position": {
"x": 400,
"y": 100
},
},
},
}
bucket_name = BUCKET_NAME
s3_client.create_bucket(bucket_name, delete_contents_if_exists=True)
source_project["workbench"]["de2578c5-431e-48eb-a9d2-aaad6b72400a"][
"outputs"]["outFile"]["path"] = path_in_datcore
destination_project = copy.deepcopy(source_project)
source_project_id = source_project["uuid"]
destination_project["uuid"] = source_project_id.replace(
"template", "deep-copy")
destination_project["workbench"] = {}
node_mapping = {}
for node_id, node in source_project["workbench"].items():
object_name = str(
Path(source_project_id) / Path(node_id) / Path(node_id + ".dat"))
f = utils.data_dir() / Path("notebooks.zip")
s3_client.upload_file(bucket_name, object_name, f)
key = node_id.replace("template", "deep-copy")
destination_project["workbench"][key] = node
node_mapping[node_id] = key
status = await dsm.deep_copy_project_simcore_s3(user_id, source_project,
destination_project,
node_mapping)
new_path = destination_project["workbench"][
"deep-copy-uuid-48eb-a9d2-aaad6b72400a"]["outputs"]["outFile"]["path"]
assert new_path != path_in_datcore
assert Path(new_path).name == file_name_in_datcore
files = await dsm.list_files(user_id=user_id, location=SIMCORE_S3_STR)
assert len(files) == 3
# one of the files in s3 should be the dowloaded one from datcore
assert any(f.fmd.file_name == Path(
datcore_structured_testbucket["filename3"]).name for f in files)
response = await dsm.delete_project_simcore_s3(user_id,
destination_project["uuid"])
files = await dsm.list_files(user_id=user_id, location=SIMCORE_S3_STR)
assert len(files) == 0
def test_datcore_fixture(datcore_structured_testbucket):
if not has_datcore_tokens():
return
print(datcore_structured_testbucket)
