def test_scan_user_mytardis_exp_dataset(set_user_mytardis_exp_dataset_config):
from mydata.commands.scan import scan_cmd
from mydata.conf import settings
with requests_mock.Mocker() as mocker:
mock_testfacility_user_response(mocker, settings.general.mytardis_url)
mock_testusers_response(mocker, settings,
["testuser1", "testuser2", "testuser3"])
mock_test_facility_response(mocker, settings.general.mytardis_url)
mock_test_instrument_response(mocker, settings.general.mytardis_url)
runner = CliRunner()
result = runner.invoke(scan_cmd, [])
assert result.exit_code == 0
assert result.output == "%s\n" % textwrap.dedent("""
Scanning tests/testdata/testdata-user-mytardis-exp-dataset/ using the "Username / "MyTardis" / Experiment / Dataset" folder structure...
Found user folder: testuser1
Found user folder: testuser2
Found user folder: testuser3
Found 2 dataset folders in tests/testdata/testdata-user-mytardis-exp-dataset/
Datasets will be collected into 2 experiments.
""")
def test_scan_email_exp_dataset_folders(set_email_exp_dataset_config):
"""Test ability to scan the Email / Experiment / Dataset folder structure.
"""
from mydata.conf import settings
from mydata.tasks.folders import scan_folders
users = []
exps = []
folders = []
def found_user(user):
users.append(user)
def found_exp(exp_folder_name):
exps.append(exp_folder_name)
found_group = None
def found_dataset(folder):
folders.append(folder)
with requests_mock.Mocker() as mocker:
mock_testfacility_user_response(mocker, settings.general.mytardis_url)
mock_testusers_response(mocker, settings, ["testuser1", "testuser2"])
mock_test_facility_response(mocker, settings.general.mytardis_url)
mock_test_instrument_response(mocker, settings.general.mytardis_url)
scan_folders(found_user, found_group, found_exp, found_dataset)
assert sorted([user.username
for user in users]) == ["testuser1", "testuser2"]
assert sorted(exps) == ["Exp1", "Exp2"]
assert sorted([folder.name for folder in folders]) == ["Birds", "Flowers"]
assert sum([folder.num_files for folder in folders]) == 5
def test_scan_email_dataset(set_email_dataset_config):
from mydata.commands.scan import scan_cmd
from mydata.conf import settings
with requests_mock.Mocker() as mocker:
mock_testfacility_user_response(mocker, settings.general.mytardis_url)
mock_testusers_response(mocker, settings, ["testuser1", "testuser2"])
mock_test_facility_response(mocker, settings.general.mytardis_url)
mock_test_instrument_response(mocker, settings.general.mytardis_url)
runner = CliRunner()
result = runner.invoke(scan_cmd, [])
assert result.exit_code == 0
assert result.output == "%s\n" % textwrap.dedent("""
Scanning tests/testdata/testdata-email-dataset/ using the "Email / Dataset" folder structure...
Found user folder: [email protected]
Found user folder: [email protected]
Found 2 dataset folders in tests/testdata/testdata-email-dataset/
""")
def test_upload_email_dataset_structure(set_email_dataset_config,
mock_key_pair):
"""Test uploading files from within the "Email / Dataset" folder structure
"""
from mydata.commands.upload import upload_cmd
from mydata.conf import settings
with requests_mock.Mocker() as mocker:
mock_uploader_creation_response(mocker, settings)
settings.uploader.ssh_key_pair = mock_key_pair
mock_get_urr(mocker,
settings,
mock_key_pair.fingerprint,
approved=False)
mock_testfacility_user_response(mocker, settings.general.mytardis_url)
mock_test_facility_response(mocker, settings.general.mytardis_url)
mock_test_instrument_response(mocker, settings.general.mytardis_url)
mock_testusers_response(mocker, settings, ["testuser1", "testuser2"])
for user in ("testuser1", "testuser2"):
name = "Test User1" if user == "testuser1" else "Test User2"
title = "Test Instrument - %s" % name
user_folder_name = "*****@*****.**" % user
mock_exp_creation(mocker, settings, title, user_folder_name)
mock_birds_flowers_dataset_creation(mocker, settings)
mock_birds_flowers_datafile_lookups(mocker, api_prefix="mydata_")
runner = CliRunner()
result = runner.invoke(upload_cmd, ["-vv"], input="y\n")
if result.exception:
raise result.exception
assert result.exit_code == 0
assert result.output == textwrap.dedent("""
Using MyData configuration in: %s
Scanning tests/testdata/testdata-email-dataset/ using the "Email / Dataset" folder structure...
Checking for approved upload method...
Using Multipart POST upload method.
Uploads via staging haven't yet been approved. Do you want to continue? [y/N]: y
Found user folder: [email protected]
Found user folder: [email protected]
Found 2 dataset folders in tests/testdata/testdata-email-dataset/
Data in Birds/ is being archived to http://127.0.0.1:9000/dataset/1002
Data in Flowers/ is being archived to http://127.0.0.1:9000/dataset/1001
4 of 5 files have been uploaded to MyTardis.
2 of 5 files have been verified by MyTardis.
1 of 5 files were found unverified without any DataFileObjects! Contact server admins!
2 of 5 files were newly uploaded in this session.
0 of 5 file lookups were found in the local cache.
File records on server without any DataFileObjects:
Dataset ID: 1001, Filename: Pond_Water_Hyacinth_Flowers.jpg
Failed lookups:
Black-beaked-sea-bird-close-up.jpg [500 Server Error: None for url: %s/api/v1/mydata_dataset_file/?format=json&dataset__id=1002&filename=Black-beaked-sea-bird-close-up.jpg&directory=]
Unverified lookups:
Pond_Water_Hyacinth_Flowers.jpg
Not found on MyTardis server:
1024px-Australian_Birds_@_Jurong_Bird_Park_(4374195521).jpg
Files uploaded:
1024px-Australian_Birds_@_Jurong_Bird_Park_(4374195521).jpg [Completed]
Pond_Water_Hyacinth_Flowers.jpg [Completed]
""" % (settings.config_path, settings.mytardis_url))
def test_upload_username_dataset_scp(set_username_dataset_config,
mock_scp_server, mock_key_pair,
mock_staging_path):
"""Test ability to scan the Username / Dataset folder structure
and upload using the SCP protocol.
"""
from mydata.commands.upload import upload_cmd
from mydata.conf import settings
upload_uploader_info(settings, mock_key_pair)
data_dir = settings.general.data_directory
filenames = {
"Flowers":
os.listdir(os.path.join(data_dir, "testuser1", "Flowers")),
"Birds":
os.listdir(os.path.join(data_dir, "testuser2", "Birds")),
"Dataset with spaces":
os.listdir(os.path.join(data_dir, "testuser1", "Dataset with spaces")),
"InvalidUserDataset1":
os.listdir(
os.path.join(data_dir, "INVALID_USER", "InvalidUserDataset1")),
}
folders = []
for folder_name in filenames:
folder = Mock()
folder.dataset = Mock(id=1)
folder.name = folder_name
folder.num_files = len(filenames[folder_name])
folder.local_files = []
for filename in filenames[folder_name]:
folder.local_files.append(LocalFile(filename, "", False))
folders.append(folder)
with requests_mock.Mocker() as mocker:
mock_testfacility_user_response(mocker, settings.general.mytardis_url)
mock_testusers_response(mocker, settings, ["testuser1", "testuser2"])
mock_invalid_user_response(mocker, settings)
mock_birds_flowers_dataset_creation(mocker, settings)
for folder in folders:
exp_id = 1
instrument_id = 1
mock_dataset_creation(mocker, settings, exp_id, instrument_id,
folder.name)
mock_responses_for_upload_folders(folders, mocker, settings,
mock_staging_path, mock_scp_server)
runner = CliRunner()
result = runner.invoke(upload_cmd, ["-vv"])
if result.exception:
raise result.exception
assert result.exit_code == 0
assert result.output == textwrap.dedent("""
Using MyData configuration in: %s
Scanning tests/testdata/testdata-username-dataset/ using the "Username / Dataset" folder structure...
Checking for approved upload method...
Using SCP upload method.
Found user folder: INVALID_USER [USER "INVALID_USER" WAS NOT FOUND ON THE MYTARDIS SERVER]
Found user folder: testuser1
Found user folder: testuser2
Found 4 dataset folders in tests/testdata/testdata-username-dataset/
Data in Birds/ is being archived to http://127.0.0.1:9000/dataset/1
Data in Dataset with spaces/ is being archived to http://127.0.0.1:9000/dataset/1
Data in Flowers/ is being archived to http://127.0.0.1:9000/dataset/1
Data in InvalidUserDataset1/ is being archived to http://127.0.0.1:9000/dataset/1
12 of 12 files have been uploaded to MyTardis.
0 of 12 files have been verified by MyTardis.
12 of 12 files were newly uploaded in this session.
0 of 12 file lookups were found in the local cache.
Not found on MyTardis server:
1024px-Australian_Birds_@_Jurong_Bird_Park_(4374195521).jpg
1024px-Colourful_flowers.JPG
Black-beaked-sea-bird-close-up.jpg
Flowers_growing_on_the_campus_of_Cebu_City_National_Science_High_School.jpg
InvalidUserFile1.txt
Pond_Water_Hyacinth_Flowers.jpg
existing_unverified_full_size_file.txt
existing_unverified_incomplete_file.txt
existing_verified_file.txt
hello.txt
missing_mydata_replica_api_endpoint.txt
zero_sized_file.txt
Files uploaded:
1024px-Australian_Birds_@_Jurong_Bird_Park_(4374195521).jpg [Completed]
1024px-Colourful_flowers.JPG [Completed]
Black-beaked-sea-bird-close-up.jpg [Completed]
Flowers_growing_on_the_campus_of_Cebu_City_National_Science_High_School.jpg [Completed]
InvalidUserFile1.txt [Completed]
Pond_Water_Hyacinth_Flowers.jpg [Completed]
existing_unverified_full_size_file.txt [Completed]
existing_unverified_incomplete_file.txt [Completed]
existing_verified_file.txt [Completed]
hello.txt [Completed]
missing_mydata_replica_api_endpoint.txt [Completed]
zero_sized_file.txt [Completed]
""" % settings.config_path)
def test_post_uploads(set_username_dataset_config):
"""
Test POST uploads
"""
from mydata.conf import settings
from mydata.tasks.folders import scan_folders
from mydata.tasks.uploads import upload_folder
from mydata.models.lookup import LookupStatus
from mydata.models.upload import UploadStatus, UploadMethod
users = []
folders = []
def found_user(user):
users.append(user)
found_exp = None
found_group = None
def found_dataset(folder):
folders.append(folder)
with requests_mock.Mocker() as mocker:
mock_testfacility_user_response(mocker, settings.general.mytardis_url)
mock_testusers_response(mocker, settings, ["testuser1", "testuser2"])
mock_invalid_user_response(mocker, settings)
mock_test_facility_response(mocker, settings.general.mytardis_url)
mock_test_instrument_response(mocker, settings.general.mytardis_url)
scan_folders(found_user, found_group, found_exp, found_dataset)
assert sorted([user.username for user in users]) == [
"INVALID_USER",
"testuser1",
"testuser2",
]
assert sorted([folder.name for folder in folders]) == [
"Birds",
"Dataset with spaces",
"Flowers",
"InvalidUserDataset1",
]
assert sum([folder.num_files for folder in folders]) == 12
with requests_mock.Mocker() as mocker:
mock_test_facility_response(mocker, settings.general.mytardis_url)
mock_test_instrument_response(mocker, settings.general.mytardis_url)
for username, name in [
("testuser1", "Test User1"),
("testuser2", "Test User2"),
("INVALID_USER", "INVALID_USER (USER NOT FOUND IN MYTARDIS)"),
]:
title = "Test Instrument - %s" % name
user_folder_name = username
mock_exp_creation(mocker, settings, title, user_folder_name)
for folder in folders:
get_dataset_url = (
"%s/api/v1/dataset/?format=json&experiments__id=1"
"&description=%s&instrument__id=1") % (
settings.general.mytardis_url, quote(folder.name))
mocker.get(get_dataset_url, text=EMPTY_LIST_RESPONSE)
get_df_url_template = Template((
"%s/api/v1/mydata_dataset_file/?format=json&dataset__id=1&filename=$filename&directory="
) % settings.general.mytardis_url)
for dfi in range(0, folder.num_files):
datafile_path = folder.get_datafile_path(dfi)
datafile_name = os.path.basename(datafile_path)
get_datafile_url = get_df_url_template.substitute(
filename=quote(datafile_name))
mocker.get(get_datafile_url, text=EMPTY_LIST_RESPONSE)
post_dataset_url = "%s/api/v1/dataset/" % settings.general.mytardis_url
post_datafile_url = ("%s/api/v1/mydata_dataset_file/" %
settings.general.mytardis_url)
mocker.post(post_datafile_url, status_code=201)
lookups = []
uploads = []
def lookup_callback(lookup):
msg = "File: %s had unexpected lookup result: %s" % (
lookup.filename,
lookup.message,
)
assert lookup.status == LookupStatus.NOT_FOUND, msg
lookups.append(lookup)
def upload_callback(upload):
msg = "File: %s had unexpected upload result: %s" % (
upload.filename,
upload.message,
)
assert upload.status == UploadStatus.COMPLETED, msg
uploads.append(upload)
for folder in folders:
mock_dataset_response = created_dataset_response(1, folder.name)
mocker.post(post_dataset_url, text=mock_dataset_response)
upload_folder(folder, lookup_callback, upload_callback,
UploadMethod.MULTIPART_POST)
# Ensure that all 12 files were looked up:
assert len(lookups) == 12
# Ensure that all 12 files were uploaded:
assert len(uploads) == 12