def test_scan_user_mytardis_exp_dataset(set_user_mytardis_exp_dataset_config):
from mydata.commands.scan import scan_cmd
from mydata.conf import settings
with requests_mock.Mocker() as mocker:
mock_testfacility_user_response(mocker, settings.general.mytardis_url)
mock_testusers_response(mocker, settings,
["testuser1", "testuser2", "testuser3"])
mock_test_facility_response(mocker, settings.general.mytardis_url)
mock_test_instrument_response(mocker, settings.general.mytardis_url)
runner = CliRunner()
result = runner.invoke(scan_cmd, [])
assert result.exit_code == 0
assert result.output == "%s\n" % textwrap.dedent("""
Scanning tests/testdata/testdata-user-mytardis-exp-dataset/ using the "Username / "MyTardis" / Experiment / Dataset" folder structure...
Found user folder: testuser1
Found user folder: testuser2
Found user folder: testuser3
Found 2 dataset folders in tests/testdata/testdata-user-mytardis-exp-dataset/
Datasets will be collected into 2 experiments.
""")
def test_scan_dataset_folders(set_dataset_config):
"""Test ability to scan the Dataset folder structure.
"""
from mydata.conf import settings
from mydata.tasks.folders import scan_folders
folders = []
# We don't need callbacks for these in this case:
found_exp = None
found_user = None
found_group = None
def found_dataset(folder):
folders.append(folder)
with requests_mock.Mocker() as mocker:
mock_testfacility_user_response(mocker, settings.general.mytardis_url)
mock_test_facility_response(mocker, settings.general.mytardis_url)
mock_test_instrument_response(mocker, settings.general.mytardis_url)
scan_folders(found_user, found_group, found_exp, found_dataset)
assert sorted([folder.name for folder in folders]) == ["Birds", "Flowers"]
assert sum([folder.num_files for folder in folders]) == 5
def test_scan_email_exp_dataset_folders(set_email_exp_dataset_config):
"""Test ability to scan the Email / Experiment / Dataset folder structure.
"""
from mydata.conf import settings
from mydata.tasks.folders import scan_folders
users = []
exps = []
folders = []
def found_user(user):
users.append(user)
def found_exp(exp_folder_name):
exps.append(exp_folder_name)
found_group = None
def found_dataset(folder):
folders.append(folder)
with requests_mock.Mocker() as mocker:
mock_testfacility_user_response(mocker, settings.general.mytardis_url)
mock_testusers_response(mocker, settings, ["testuser1", "testuser2"])
mock_test_facility_response(mocker, settings.general.mytardis_url)
mock_test_instrument_response(mocker, settings.general.mytardis_url)
scan_folders(found_user, found_group, found_exp, found_dataset)
assert sorted([user.username
for user in users]) == ["testuser1", "testuser2"]
assert sorted(exps) == ["Exp1", "Exp2"]
assert sorted([folder.name for folder in folders]) == ["Birds", "Flowers"]
assert sum([folder.num_files for folder in folders]) == 5
def test_scan_group_dataset_folders(set_group_dataset_config):
"""Test ability to scan the Group / Dataset folder structure.
"""
from mydata.conf import settings
from mydata.tasks.folders import scan_folders
groups = []
folders = []
# We don't need callback for finding users folders:
found_exp = None
found_user = None
def found_group(group):
groups.append(group)
def found_dataset(folder):
folders.append(folder)
with requests_mock.Mocker() as mocker:
mock_testfacility_user_response(mocker, settings.general.mytardis_url)
for group_name in ("TestFacility-Group1", "TestFacility-Group2"):
mock_get_group(mocker, settings.general.mytardis_url, group_name)
mock_test_facility_response(mocker, settings.general.mytardis_url)
mock_test_instrument_response(mocker, settings.general.mytardis_url)
scan_folders(found_user, found_group, found_exp, found_dataset)
assert sorted([group.name for group in groups]) == [
"TestFacility-Group1",
"TestFacility-Group2",
]
assert sorted([folder.name for folder in folders]) == ["Birds", "Flowers"]
assert sum([folder.num_files for folder in folders]) == 5
def test_scan_dataset(set_dataset_config):
from mydata.commands.scan import scan_cmd
from mydata.conf import settings
with requests_mock.Mocker() as mocker:
mock_testfacility_user_response(mocker, settings.general.mytardis_url)
mock_test_facility_response(mocker, settings.general.mytardis_url)
mock_test_instrument_response(mocker, settings.general.mytardis_url)
runner = CliRunner()
result = runner.invoke(scan_cmd, [])
assert result.exit_code == 0
assert result.output == "%s\n" % textwrap.dedent("""
Scanning tests/testdata/testdata-dataset/ using the "Dataset" folder structure...
Found 2 dataset folders in tests/testdata/testdata-dataset/
""")
def test_scan_group_instrument(set_group_instrument_config):
from mydata.commands.scan import scan_cmd
from mydata.conf import settings
with requests_mock.Mocker() as mocker:
mock_testfacility_user_response(mocker, settings.general.mytardis_url)
for group_name in ("TestFacility-Group1", "TestFacility-Group2"):
mock_get_group(mocker, settings.general.mytardis_url, group_name)
mock_test_facility_response(mocker, settings.general.mytardis_url)
mock_test_instrument_response(mocker, settings.general.mytardis_url)
runner = CliRunner()
result = runner.invoke(scan_cmd, [])
assert result.exit_code == 0
assert result.output == "%s\n" % textwrap.dedent(
"""
Scanning tests/testdata/testdata-group-instrument/ using the "User Group / Instrument / Full Name / Dataset" folder structure...
Found group folder: Group1
Found group folder: Group2
Found 8 dataset folders in tests/testdata/testdata-group-instrument/
"""
)
def test_filters_settings(set_user_exp_dataset_config):
"""Test ability to validate filters-related settings.
"""
from mydata.conf import settings
from mydata.models.settings.validation import validate_settings
from mydata.logs import logger
with requests_mock.Mocker() as mocker:
list_api_endpoints_url = ("%s/api/v1/?format=json" %
settings.general.mytardis_url)
mocker.get(list_api_endpoints_url, text=MOCK_API_ENDPOINTS_RESPONSE)
mock_testfacility_user_response(mocker, settings.general.mytardis_url)
mock_test_facility_response(mocker, settings.general.mytardis_url)
mock_test_instrument_response(mocker, settings.general.mytardis_url)
log_output = logger.get_value()
validate_settings()
expected_warning = "Files newer than 1 minute(s) are being ignored"
assert expected_warning in subtract(logger.get_value(), log_output)
old_value = settings.advanced.upload_invalid_user_or_group_folders
settings.advanced.upload_invalid_user_or_group_folders = False
log_output = logger.get_value()
validate_settings()
expected_warning = "Invalid user folders are being ignored"
assert expected_warning in subtract(logger.get_value(), log_output)
settings.advanced.upload_invalid_user_or_group_folders = old_value
old_value1 = settings.advanced.upload_invalid_user_or_group_folders
old_value2 = settings.advanced.folder_structure
settings.advanced.upload_invalid_user_or_group_folders = False
settings.advanced.folder_structure = "User Group / Experiment / Dataset"
log_output = logger.get_value()
validate_settings()
expected_warning = "Invalid user group folders are being ignored"
assert expected_warning in subtract(logger.get_value(), log_output)
settings.advanced.upload_invalid_user_or_group_folders = old_value1
settings.advanced.folder_structure = old_value2
old_value1 = settings.filters.user_filter
settings.filters.user_filter = "filter-string"
log_output = logger.get_value()
validate_settings()
expected_warning = "User folders are being filtered"
assert expected_warning in subtract(logger.get_value(), log_output)
settings.filters.user_filter = old_value1
old_value1 = settings.filters.user_filter
old_value2 = settings.advanced.folder_structure
settings.filters.user_filter = "filter-string"
settings.advanced.folder_structure = "User Group / Experiment / Dataset"
log_output = logger.get_value()
validate_settings()
expected_warning = "User group folders are being filtered"
assert expected_warning in subtract(logger.get_value(), log_output)
settings.filters.user_filter = old_value1
settings.advanced.folder_structure = old_value2
old_value = settings.filters.experiment_filter
settings.filters.experiment_filter = "filter-string"
log_output = logger.get_value()
validate_settings()
expected_warning = "Experiment folders are being filtered"
assert expected_warning in subtract(logger.get_value(), log_output)
settings.filters.experiment_filter = old_value
old_value = settings.filters.dataset_filter
settings.filters.dataset_filter = "filter-string"
log_output = logger.get_value()
validate_settings()
expected_warning = "Dataset folders are being filtered"
assert expected_warning in subtract(logger.get_value(), log_output)
settings.filters.dataset_filter = old_value
old_value = settings.filters.ignore_old_datasets
settings.filters.ignore_old_datasets = True
log_output = logger.get_value()
validate_settings()
expected_warning = "Old datasets are being ignored"
assert expected_warning in subtract(logger.get_value(), log_output)
settings.filters.ignore_old_datasets = old_value
old_value = settings.filters.ignore_new_datasets
settings.filters.ignore_new_datasets = True
log_output = logger.get_value()
validate_settings()
expected_warning = "New datasets are being ignored"
assert expected_warning in subtract(logger.get_value(), log_output)
settings.filters.ignore_new_datasets = old_value
def test_globs_validation(
use_includes_file,
use_excludes_file,
includes_file,
excludes_file,
expected_warning=None,
expected_exception_msg=None,
):
"""
Test globs files settings validation / warnings
"""
from mydata.conf import settings
from mydata.models.settings.validation import validate_settings
from mydata.utils.exceptions import InvalidSettings
from mydata.logs import logger
old_value1 = settings.filters.use_includes_file
old_value2 = settings.filters.use_excludes_file
old_value3 = settings.filters.includes_file
old_value4 = settings.filters.excludes_file
settings.filters.use_includes_file = use_includes_file
settings.filters.use_excludes_file = use_excludes_file
settings.filters.includes_file = includes_file
settings.filters.excludes_file = excludes_file
log_output = logger.get_value()
if expected_exception_msg:
with pytest.raises(InvalidSettings) as excinfo:
validate_settings()
assert str(excinfo.value) == expected_exception_msg
else:
validate_settings()
if expected_warning:
assert expected_warning in subtract(logger.get_value(),
log_output)
settings.filters.use_includes_file = old_value1
settings.filters.use_excludes_file = old_value2
settings.filters.includes_file = old_value3
settings.filters.excludes_file = old_value4
warning = ("Only files matching patterns in includes "
"file will be scanned for upload.")
message = "No includes file was specified."
test_globs_validation(
use_includes_file=True,
use_excludes_file=False,
includes_file="",
excludes_file="",
expected_warning=warning,
expected_exception_msg=message,
)
message = "Specified includes file doesn't exist."
test_globs_validation(
use_includes_file=True,
use_excludes_file=False,
includes_file="file/does/not/exist",
excludes_file="",
expected_warning=warning,
expected_exception_msg=message,
)
message = "Specified includes file path is not a file."
test_globs_validation(
use_includes_file=True,
use_excludes_file=False,
includes_file=".",
excludes_file="",
expected_warning=warning,
expected_exception_msg=message,
)
warning = ("Files matching patterns in excludes "
"file will not be scanned for upload.")
message = "No excludes file was specified."
test_globs_validation(
use_includes_file=False,
use_excludes_file=True,
includes_file="",
excludes_file="",
expected_warning=warning,
expected_exception_msg=message,
)
warning = ("Files matching patterns in excludes "
"file will not be scanned for upload, "
"unless they match patterns in the includes file.")
message = "No includes file was specified."
test_globs_validation(
use_includes_file=True,
use_excludes_file=True,
includes_file="",
excludes_file="",
expected_warning=warning,
expected_exception_msg=message,
)
def test_instrument_exceptions(set_exp_dataset_config):
"""Test ability to handle instrument-related exceptions.
"""
from mydata.conf import settings
from mydata.models.instrument import Instrument
with requests_mock.Mocker() as mocker:
mock_test_facility_response(mocker, settings.general.mytardis_url)
facility = settings.general.facility
assert facility
api_key = settings.general.api_key
settings.general.api_key = "invalid"
with requests_mock.Mocker() as mocker:
get_instrument_url = (
"%s/api/v1/instrument/?format=json&facility__id=1"
"&name=Unauthorized%%20Instrument") % settings.general.mytardis_url
mocker.get(get_instrument_url, status_code=401)
with pytest.raises(HTTPError) as excinfo:
_ = Instrument.get_instrument(facility, "Unauthorized Instrument")
assert excinfo.value.response.status_code == 401
with requests_mock.Mocker() as mocker:
post_instrument_url = "%s/api/v1/instrument/" % settings.general.mytardis_url
mocker.post(post_instrument_url, status_code=401)
with pytest.raises(HTTPError) as excinfo:
_ = Instrument.create_instrument(facility,
"Unauthorized Instrument")
assert excinfo.value.response.status_code == 401
settings.general.api_key = api_key
with requests_mock.Mocker() as mocker:
post_instrument_url = "%s/api/v1/instrument/" % settings.general.mytardis_url
mocker.post(post_instrument_url, status_code=500)
with pytest.raises(HTTPError) as excinfo:
_ = Instrument.create_instrument(facility, "Instrument name")
assert excinfo.value.response.status_code == 500
with requests_mock.Mocker() as mocker:
mock_test_facility_response(mocker, settings.general.mytardis_url)
mock_test_instrument_response(mocker, settings.general.mytardis_url)
instrument = settings.general.instrument
with requests_mock.Mocker() as mocker:
put_instrument_url = "%s/api/v1/instrument/1/" % settings.general.mytardis_url
mocker.put(put_instrument_url, status_code=500)
with pytest.raises(HTTPError) as excinfo:
instrument.rename("New instrument name")
assert excinfo.value.response.status_code == 500
with requests_mock.Mocker() as mocker:
mock_test_facility_response(mocker, settings.general.mytardis_url)
mock_test_instrument_response(mocker, settings.general.mytardis_url)
new_instrument_get_url = (
"%s/api/v1/instrument/?format=json"
"&facility__id=1&name=New%%20instrument%%20name"
) % settings.general.mytardis_url
mocker.get(new_instrument_get_url, text=EMPTY_LIST_RESPONSE)
put_instrument_url = "%s/api/v1/instrument/1/" % settings.general.mytardis_url
mocker.put(put_instrument_url, status_code=500)
with pytest.raises(HTTPError) as excinfo:
instrument.rename_instrument(
settings.general.facility_name,
settings.general.instrument_name,
"New instrument name",
)
assert excinfo.value.response.status_code == 500
def test_experiment_exceptions(set_exp_dataset_config):
"""Test ability to handle experiment-related exceptions.
"""
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
from mydata.conf import settings
from mydata.threads.flags import FLAGS
from mydata.models.experiment import Experiment
from mydata.models.folder import Folder
# MyData has the concept of a "default experiment",
# which depends on the UUID of the MyData instance:
settings.miscellaneous.uuid = "1234567890"
with requests_mock.Mocker() as mocker:
mock_testfacility_user_response(mocker, settings.general.mytardis_url)
owner = settings.general.default_owner
dataset_folder_name = "Flowers"
exp_folder_name = "Exp1"
location = os.path.join(settings.general.data_directory, exp_folder_name)
# LOOKING UP EXPERIMENTS
# Try to look up nonexistent experiment record with
# experiment title set manually, and with a user folder
# name, but no group folder name:
user_folder_name = owner.username
group_folder_name = None
folder = Folder(dataset_folder_name, location, user_folder_name,
group_folder_name, owner)
folder.experiment_title = exp_folder_name
with requests_mock.Mocker() as mocker:
get_exp_url = (
"%s/api/v1/mydata_experiment/?format=json&title=Exp1"
"&folder_structure=Experiment%%20/%%20Dataset"
"&user_folder_name=testfacility") % settings.general.mytardis_url
mocker.get(get_exp_url, text=EMPTY_LIST_RESPONSE)
existing_exp = Experiment.get_exp_for_folder(folder)
assert not existing_exp
# Look up existing experiment record with
# experiment title set manually, and with a user folder
# name, but no group folder name:
with requests_mock.Mocker() as mocker:
get_exp_url = (
"%s/api/v1/mydata_experiment/?format=json"
"&title=Existing%%20Experiment"
"&folder_structure=Experiment%%20/%%20Dataset"
"&user_folder_name=testfacility") % settings.general.mytardis_url
mocker.get(get_exp_url, text=EXISTING_EXP_RESPONSE)
user_folder_name = owner.username
group_folder_name = None
folder = Folder(dataset_folder_name, location, user_folder_name,
group_folder_name, owner)
folder.experiment_title = "Existing Experiment"
experiment = Experiment.get_exp_for_folder(folder)
assert experiment.title == "Existing Experiment"
# Look up one of many existing experiment records with
# experiment title set manually, and with a user folder
# name, but no group folder name:
mock_exp_dict = {
"meta": {
"limit": 20,
"next": None,
"offset": 0,
"previous": None,
"total_count": 2,
},
"objects": [
{
"id": 1,
"title": "Existing Experiment1"
},
{
"id": 2,
"title": "Existing Experiment2"
},
],
}
mock_exp_response = json.dumps(mock_exp_dict)
with requests_mock.Mocker() as mocker:
get_exp_url = (
"%s/api/v1/mydata_experiment/?format=json"
"&title=Multiple%%20Existing%%20Experiments"
"&folder_structure=Experiment%%20/%%20Dataset"
"&user_folder_name=testfacility") % settings.general.mytardis_url
mocker.get(get_exp_url, text=mock_exp_response)
user_folder_name = owner.username
group_folder_name = None
folder = Folder(dataset_folder_name, location, user_folder_name,
group_folder_name, owner)
folder.experiment_title = "Multiple Existing Experiments"
experiment = Experiment.get_exp_for_folder(folder)
assert experiment.title == "Existing Experiment1"
# Try to look up nonexistent experiment record with
# experiment title set manually, and with a group folder
# name, but no user folder name:
user_folder_name = None
group_folder_name = "Test Group1"
folder = Folder(dataset_folder_name, location, user_folder_name,
group_folder_name, owner)
folder.experiment_title = exp_folder_name
with requests_mock.Mocker() as mocker:
get_exp_url = ("%s/api/v1/mydata_experiment/?format=json&title=Exp1"
"&folder_structure=Experiment%%20/%%20Dataset"
"&group_folder_name=Test%%20Group1"
) % settings.general.mytardis_url
mocker.get(get_exp_url, text=EMPTY_LIST_RESPONSE)
existing_exp = Experiment.get_exp_for_folder(folder)
assert not existing_exp
# Look up existing experiment record with
# experiment title set manually, and with a group folder
# name, but no user folder name:
user_folder_name = None
group_folder_name = "Test Group1"
folder = Folder(dataset_folder_name, location, user_folder_name,
group_folder_name, owner)
folder.experiment_title = "Existing Experiment"
with requests_mock.Mocker() as mocker:
get_exp_url = (
"%s/api/v1/mydata_experiment/?format=json&title=Existing%%20Experiment"
"&folder_structure=Experiment%%20/%%20Dataset&group_folder_name=Test%%20Group1"
) % settings.general.mytardis_url
mocker.get(get_exp_url, text=EXISTING_EXP_RESPONSE)
experiment = Experiment.get_exp_for_folder(folder)
assert experiment.title == "Existing Experiment"
# Try to look up nonexistent experiment record with
# experiment title set manually, and with a user folder
# name, and a group folder name:
user_folder_name = owner.username
group_folder_name = "Test Group1"
folder = Folder(dataset_folder_name, location, user_folder_name,
group_folder_name, owner)
folder.experiment_title = exp_folder_name
with requests_mock.Mocker() as mocker:
get_exp_url = ("%s/api/v1/mydata_experiment/?format=json&title=Exp1"
"&folder_structure=Experiment%%20/%%20Dataset"
"&group_folder_name=Test%%20Group1"
) % settings.general.mytardis_url
mocker.get(get_exp_url, text=EMPTY_LIST_RESPONSE)
existing_exp = Experiment.get_exp_for_folder(folder)
assert not existing_exp
# Look up existing experiment record with
# experiment title set manually, and with a group folder
# name, and a user folder name:
user_folder_name = owner.username
group_folder_name = "Test Group1"
folder = Folder(dataset_folder_name, location, user_folder_name,
group_folder_name, owner)
folder.experiment_title = "Existing Experiment"
with requests_mock.Mocker() as mocker:
get_exp_url = (
"%s/api/v1/mydata_experiment/?format=json&title=Existing%%20Experiment"
"&folder_structure=Experiment%%20/%%20Dataset"
"&user_folder_name=testfacility&group_folder_name=Test%%20Group1"
) % settings.general.mytardis_url
mocker.get(get_exp_url, text=EXISTING_EXP_RESPONSE)
experiment = Experiment.get_exp_for_folder(folder)
assert experiment.title == "Existing Experiment"
# Try to look up nonexistent experiment record with
# experiment title set manually, with neither a user folder
# name, nor a group folder name:
user_folder_name = None
group_folder_name = None
folder = Folder(dataset_folder_name, location, user_folder_name,
group_folder_name, owner)
folder.experiment_title = exp_folder_name
with requests_mock.Mocker() as mocker:
get_exp_url = ("%s/api/v1/mydata_experiment/?format=json&title=Exp1"
"&folder_structure=Experiment%%20/%%20Dataset"
) % settings.general.mytardis_url
mocker.get(get_exp_url, text=EMPTY_LIST_RESPONSE)
existing_exp = Experiment.get_exp_for_folder(folder)
assert not existing_exp
# Look up existing experiment record with
# experiment title set manually, and with neither a user folder
# name, nor a group folder name:
user_folder_name = None
group_folder_name = None
folder = Folder(dataset_folder_name, location, user_folder_name,
group_folder_name, owner)
folder.experiment_title = "Existing Experiment"
with requests_mock.Mocker() as mocker:
get_exp_url = (
"%s/api/v1/mydata_experiment/?format=json&title=Existing%%20Experiment"
"&folder_structure=Experiment%%20/%%20Dataset"
) % settings.general.mytardis_url
mocker.get(get_exp_url, text=EXISTING_EXP_RESPONSE)
experiment = Experiment.get_exp_for_folder(folder)
assert experiment.title == "Existing Experiment"
# Try to look up experiment record with
# an invalid API key, which should give 401 (Unauthorized)
api_key = settings.general.api_key
settings.general.api_key = "invalid"
with requests_mock.Mocker() as mocker:
get_exp_url = (
"%s/api/v1/mydata_experiment/?format=json&title=Existing%%20Experiment"
"&folder_structure=Experiment%%20/%%20Dataset"
) % settings.general.mytardis_url
mocker.get(get_exp_url, status_code=401)
with pytest.raises(HTTPError) as excinfo:
_ = Experiment.get_exp_for_folder(folder)
assert excinfo.value.response.status_code == 401
settings.general.api_key = api_key
# Try to look up experiment record with a missing Schema,
# which can result in a 404 from the MyTardis API:
folder.experiment_title = "Missing Schema"
with requests_mock.Mocker() as mocker:
get_exp_url = (
"%s/api/v1/mydata_experiment/?format=json&title=Missing%%20Schema"
"&folder_structure=Experiment%%20/%%20Dataset"
) % settings.general.mytardis_url
mocker.get(get_exp_url, status_code=404)
with pytest.raises(HTTPError) as excinfo:
_ = Experiment.get_exp_for_folder(folder)
assert excinfo.value.response.status_code == 404
# Try to look up experiment record and handle a 404 of
# unknown origin from the MyTardis API:
folder.experiment_title = "Unknown 404"
with requests_mock.Mocker() as mocker:
get_exp_url = (
"%s/api/v1/mydata_experiment/?format=json&title=Unknown%%20404"
"&folder_structure=Experiment%%20/%%20Dataset"
) % settings.general.mytardis_url
mocker.get(get_exp_url, status_code=404)
with pytest.raises(HTTPError) as excinfo:
_ = Experiment.get_exp_for_folder(folder)
assert excinfo.value.response.status_code == 404
# CREATING EXPERIMENTS
# Try to create an experiment with a title specified manually
# and check that the title is correct:
FLAGS.test_run_running = False
folder.experiment_title = exp_folder_name
with requests_mock.Mocker() as mocker:
post_exp_url = "%s/api/v1/mydata_experiment/" % settings.general.mytardis_url
mocker.post(post_exp_url, text=EXP1_RESPONSE, status_code=201)
post_objectacl_url = "%s/api/v1/objectacl/" % settings.general.mytardis_url
mocker.post(post_objectacl_url, status_code=201)
mock_test_facility_response(mocker, settings.general.mytardis_url)
mock_test_instrument_response(mocker, settings.general.mytardis_url)
experiment = Experiment.create_exp_for_folder(folder)
assert experiment.title == exp_folder_name
# Try to create an experiment with a title specified manually,
# during a test run
FLAGS.test_run_running = True
folder.experiment_title = exp_folder_name
with requests_mock.Mocker() as mocker:
get_exp_url = ("%s/api/v1/mydata_experiment/?format=json&title=Exp1"
"&folder_structure=Experiment%%20/%%20Dataset"
) % settings.general.mytardis_url
mocker.get(get_exp_url, text=EMPTY_LIST_RESPONSE)
experiment = Experiment.get_or_create_exp_for_folder(folder)
assert experiment is None
FLAGS.test_run_running = False
# Get or create an experiment with a title specified manually,
# which already exists during a test run
FLAGS.test_run_running = True
folder.experiment_title = "Existing Experiment"
with requests_mock.Mocker() as mocker:
get_exp_url = (
"%s/api/v1/mydata_experiment/?format=json"
"&title=Existing%%20Experiment&folder_structure=Experiment%%20/%%20Dataset"
) % settings.general.mytardis_url
mocker.get(get_exp_url, text=EXISTING_EXP_RESPONSE)
experiment = Experiment.get_or_create_exp_for_folder(folder)
assert experiment.title == "Existing Experiment"
folder.experiment_title = exp_folder_name
FLAGS.test_run_running = False
# Try to create an experiment record with
# an invalid API key, which should give 401 (Unauthorized)
api_key = settings.general.api_key
settings.general.api_key = "invalid"
with requests_mock.Mocker() as mocker:
post_exp_url = (
"%s/api/v1/mydata_experiment/") % settings.general.mytardis_url
mocker.post(post_exp_url, status_code=401)
mock_test_facility_response(mocker, settings.general.mytardis_url)
mock_test_instrument_response(mocker, settings.general.mytardis_url)
with pytest.raises(HTTPError) as excinfo:
_ = Experiment.create_exp_for_folder(folder)
assert excinfo.value.response.status_code == 401
settings.general.api_key = api_key
# Now let's test experiment creation with the experiment's
# title determined automatically (from the instrument's name
# which becomes the default uploader name) and the user folder
# name or group folder name):
user_folder_name = owner.username
group_folder_name = None
folder = Folder(dataset_folder_name, location, user_folder_name,
group_folder_name, owner)
# Test case where MyTardis API returns a 404, e.g. because a
# requested Experiment Schema can't be found.
folder.experiment_title = "Request 404 from Fake MyTardis Server"
with requests_mock.Mocker() as mocker:
post_exp_url = (
"%s/api/v1/mydata_experiment/") % settings.general.mytardis_url
mocker.post(post_exp_url, status_code=404)
mock_test_facility_response(mocker, settings.general.mytardis_url)
mock_test_instrument_response(mocker, settings.general.mytardis_url)
with pytest.raises(HTTPError) as excinfo:
_ = Experiment.create_exp_for_folder(folder)
assert excinfo.value.response.status_code == 404
def test_validate_settings(set_exp_dataset_config):
"""Test ability to validate settings.
"""
from mydata.conf import settings
from mydata.models.settings.validation import validate_settings
from mydata.utils.exceptions import InvalidSettings
with requests_mock.Mocker() as mocker:
list_api_endpoints_url = ("%s/api/v1/?format=json" %
settings.general.mytardis_url)
mocker.get(list_api_endpoints_url, text=MOCK_API_ENDPOINTS_RESPONSE)
mock_testfacility_user_response(mocker, settings.general.mytardis_url)
mock_test_facility_response(mocker, settings.general.mytardis_url)
mock_test_instrument_response(mocker, settings.general.mytardis_url)
validate_settings()
old_value = settings.general.mytardis_url
settings.general.mytardis_url = ""
with pytest.raises(InvalidSettings) as excinfo:
validate_settings()
assert "Please enter a valid MyTardis URL" in str(excinfo.value)
settings.general.mytardis_url = old_value
old_value = settings.general.data_directory
settings.general.data_directory = ""
with pytest.raises(InvalidSettings) as excinfo:
validate_settings()
assert "Please enter a valid data directory" in str(excinfo.value)
settings.general.data_directory = old_value
old_value = settings.general.data_directory
settings.general.data_directory = "this/folder/does/not/exist"
with pytest.raises(InvalidSettings) as excinfo:
validate_settings()
assert "doesn't exist" in str(excinfo.value)
settings.general.data_directory = old_value
old_value = settings.general.instrument_name
old_value = settings.general.instrument_name
settings.general.instrument_name = ""
with pytest.raises(InvalidSettings) as excinfo:
validate_settings()
assert "Please enter a valid instrument name" in str(excinfo.value)
settings.general.instrument_name = old_value
old_value = settings.general.facility_name
settings.general.facility_name = ""
with pytest.raises(InvalidSettings) as excinfo:
validate_settings()
assert "Please enter a valid facility name" in str(excinfo.value)
settings.general.facility_name = old_value
old_value = settings.general.facility_name
settings.general.facility_name = "Invalid"
with pytest.raises(InvalidSettings) as excinfo:
validate_settings()
assert 'Facility "Invalid" was not found in MyTardis.' in str(
excinfo.value)
settings.general.facility_name = old_value
old_value = settings.general.contact_name
settings.general.contact_name = ""
with pytest.raises(InvalidSettings) as excinfo:
validate_settings()
assert "Please enter a valid contact name" in str(excinfo.value)
settings.general.contact_name = old_value
old_value = settings.general.contact_email
settings.general.contact_email = ""
with pytest.raises(InvalidSettings) as excinfo:
validate_settings()
assert "Please enter a valid contact email" in str(excinfo.value)
settings.general.contact_email = old_value
old_value = settings.general.contact_email
settings.general.contact_email = "invalid-email-address"
with pytest.raises(InvalidSettings) as excinfo:
validate_settings()
assert "Please enter a valid contact email" in str(excinfo.value)
settings.general.contact_email = old_value
old_value = settings.advanced.folder_structure
old_value2 = settings.advanced.validate_folder_structure
settings.advanced.folder_structure = "Email / Dataset"
settings.advanced.validate_folder_structure = True
with pytest.raises(InvalidSettings) as excinfo:
validate_settings()
assert re.match("Folder name .* is not a valid email address.",
str(excinfo.value))
settings.advanced.folder_structure = old_value
settings.advanced.validate_folder_structure = old_value2
old_value = settings.general.username
settings.general.username = ""
with pytest.raises(InvalidSettings) as excinfo:
validate_settings()
assert "Please enter a MyTardis username" in str(excinfo.value)
settings.general.username = old_value
old_value = settings.general.api_key
settings.general.api_key = ""
with pytest.raises(InvalidSettings) as excinfo:
validate_settings()
assert "Please enter your MyTardis API key" in str(excinfo.value)
settings.general.api_key = old_value
def test_upload_email_dataset_structure(set_email_dataset_config,
mock_key_pair):
"""Test uploading files from within the "Email / Dataset" folder structure
"""
from mydata.commands.upload import upload_cmd
from mydata.conf import settings
with requests_mock.Mocker() as mocker:
mock_uploader_creation_response(mocker, settings)
settings.uploader.ssh_key_pair = mock_key_pair
mock_get_urr(mocker,
settings,
mock_key_pair.fingerprint,
approved=False)
mock_testfacility_user_response(mocker, settings.general.mytardis_url)
mock_test_facility_response(mocker, settings.general.mytardis_url)
mock_test_instrument_response(mocker, settings.general.mytardis_url)
mock_testusers_response(mocker, settings, ["testuser1", "testuser2"])
for user in ("testuser1", "testuser2"):
name = "Test User1" if user == "testuser1" else "Test User2"
title = "Test Instrument - %s" % name
user_folder_name = "*****@*****.**" % user
mock_exp_creation(mocker, settings, title, user_folder_name)
mock_birds_flowers_dataset_creation(mocker, settings)
mock_birds_flowers_datafile_lookups(mocker, api_prefix="mydata_")
runner = CliRunner()
result = runner.invoke(upload_cmd, ["-vv"], input="y\n")
if result.exception:
raise result.exception
assert result.exit_code == 0
assert result.output == textwrap.dedent("""
Using MyData configuration in: %s
Scanning tests/testdata/testdata-email-dataset/ using the "Email / Dataset" folder structure...
Checking for approved upload method...
Using Multipart POST upload method.
Uploads via staging haven't yet been approved. Do you want to continue? [y/N]: y
Found user folder: [email protected]
Found user folder: [email protected]
Found 2 dataset folders in tests/testdata/testdata-email-dataset/
Data in Birds/ is being archived to http://127.0.0.1:9000/dataset/1002
Data in Flowers/ is being archived to http://127.0.0.1:9000/dataset/1001
4 of 5 files have been uploaded to MyTardis.
2 of 5 files have been verified by MyTardis.
1 of 5 files were found unverified without any DataFileObjects! Contact server admins!
2 of 5 files were newly uploaded in this session.
0 of 5 file lookups were found in the local cache.
File records on server without any DataFileObjects:
Dataset ID: 1001, Filename: Pond_Water_Hyacinth_Flowers.jpg
Failed lookups:
Black-beaked-sea-bird-close-up.jpg [500 Server Error: None for url: %s/api/v1/mydata_dataset_file/?format=json&dataset__id=1002&filename=Black-beaked-sea-bird-close-up.jpg&directory=]
Unverified lookups:
Pond_Water_Hyacinth_Flowers.jpg
Not found on MyTardis server:
1024px-Australian_Birds_@_Jurong_Bird_Park_(4374195521).jpg
Files uploaded:
1024px-Australian_Birds_@_Jurong_Bird_Park_(4374195521).jpg [Completed]
Pond_Water_Hyacinth_Flowers.jpg [Completed]
""" % (settings.config_path, settings.mytardis_url))
def test_dataset_exceptions(set_exp_dataset_config):
"""Test ability to handle dataset-related exceptions.
"""
from mydata.conf import settings
from mydata.models.dataset import Dataset
from mydata.models.experiment import Experiment
from mydata.models.folder import Folder
from mydata.threads.flags import FLAGS
with requests_mock.Mocker() as mocker:
mock_testfacility_user_response(mocker, settings.general.mytardis_url)
owner = settings.general.default_owner
dataset_folder_name = "Flowers"
exp_folder_name = "Exp1"
location = os.path.join(settings.general.data_directory, exp_folder_name)
# Test creating dataset record and ensure that no exception
# is raised:
user_folder_name = owner.username
group_folder_name = None
folder = Folder(dataset_folder_name, location, user_folder_name,
group_folder_name, owner)
folder.experimentTitle = "Existing Experiment"
mock_exp_response = build_list_response([{
"id": 1,
"title": "Existing Experiment"
}])
with requests_mock.Mocker() as mocker:
get_exp_url = (
"%s/api/v1/mydata_experiment/?format=json&title="
"&folder_structure=Experiment%%20/%%20Dataset"
"&user_folder_name=testfacility") % settings.general.mytardis_url
mocker.get(get_exp_url, text=mock_exp_response)
experiment = Experiment.get_exp_for_folder(folder)
assert experiment.title == "Existing Experiment"
folder.experiment = experiment
FLAGS.test_run_running = False
mock_dataset_response = build_list_response([{
"id": 1,
"description": "Flowers"
}])
with requests_mock.Mocker() as mocker:
get_dataset_url = ("%s/api/v1/dataset/?format=json&experiments__id=1"
"&description=Flowers&instrument__id=1"
) % settings.general.mytardis_url
mocker.get(get_dataset_url, text=mock_dataset_response)
mock_test_facility_response(mocker, settings.general.mytardis_url)
mock_test_instrument_response(mocker, settings.general.mytardis_url)
dataset = Dataset.create_dataset_if_necessary(folder)
assert dataset.description == dataset_folder_name
mock_dataset_response = EMPTY_LIST_RESPONSE
with requests_mock.Mocker() as mocker:
get_dataset_url = ("%s/api/v1/dataset/?format=json&experiments__id=1"
"&description=Flowers&instrument__id=1"
) % settings.general.mytardis_url
mocker.get(get_dataset_url, text=mock_dataset_response)
# Simulate creating dataset record during test run
# and ensure that no exception is raised:
FLAGS.test_run_running = True
dataset = Dataset.create_dataset_if_necessary(folder)
assert dataset is None
FLAGS.test_run_running = False
# Simulate retrieving existing dataset record during test run
# and ensure that no exception is raised:
with requests_mock.Mocker() as mocker:
get_dataset_url = ("%s/api/v1/dataset/?format=json&experiments__id=1"
"&description=Existing%%20Dataset&instrument__id=1"
) % settings.general.mytardis_url
mocker.get(get_dataset_url,
text=existing_dataset_response(1, "Existing Dataset"))
mock_test_facility_response(mocker, settings.general.mytardis_url)
FLAGS.test_run_running = True
folder.data_view_fields["name"] = "Existing Dataset"
dataset = Dataset.create_dataset_if_necessary(folder)
FLAGS.test_run_running = False
assert dataset.description == "Existing Dataset"
def test_uploader(set_username_dataset_config, mock_key_pair):
"""Test ability to create an Uploader and an Uploader Registration Request
"""
from mydata.conf import settings
from mydata.utils.exceptions import (
NoApprovedStorageBox,
StorageBoxAttributeNotFound,
StorageBoxOptionNotFound,
)
# Firstly, let's test the case where we don't have an existing uploader
# record, i.e. the GET query will return an empty list, so, we'll
# need to create a new uploader record with POST:
# Reset global settings' uploader instance, so we when we next call
# the settings.uploader property method, we'll generate a
# new Uploader instance, using the up-to-date
# settings.general.instrument_name:
settings.uploader = None
with requests_mock.Mocker() as mocker:
mock_test_facility_response(mocker, settings.general.mytardis_url)
mock_test_instrument_response(mocker, settings.general.mytardis_url)
mock_uploader_creation_response(mocker, settings)
settings.uploader.upload_uploader_info()
assert settings.uploader.name == "Test Instrument"
# Now let's test the case where we have an existing uploader record:
# Reset global settings' uploader instance, so we when we next call
# the settings.uploader property method, we'll generate a
# new Uploader instance, using the up-to-date
# settings.general.instrument_name:
settings.uploader = None
with requests_mock.Mocker() as mocker:
mock_test_facility_response(mocker, settings.general.mytardis_url)
mock_test_instrument_response(mocker, settings.general.mytardis_url)
mock_uploader_update_response(mocker, settings)
settings.uploader.upload_uploader_info()
assert settings.uploader.name == "Test Instrument"
settings.uploader.ssh_key_pair = mock_key_pair
# Now let's test requesting staging access when we don't have an
# existing UploaderRegistrationRequest:
with requests_mock.Mocker() as mocker:
mock_uploader_update_response(mocker, settings)
get_urr_url = (
"%s/api/v1/mydata_uploaderregistrationrequest/?format=json"
"&uploader__uuid=00000000001&requester_key_fingerprint=%s") % (
settings.general.mytardis_url,
settings.uploader.ssh_key_pair.fingerprint)
mocker.get(get_urr_url, text=EMPTY_LIST_RESPONSE)
post_urr_url = ("%s/api/v1/mydata_uploaderregistrationrequest/"
) % settings.general.mytardis_url
mocker.post(post_urr_url, text=CREATED_URR_RESPONSE)
settings.uploader.request_staging_access()
urr = settings.uploader.upload_to_staging_request
assert not urr.approved
assert not urr.approved_storage_box
with pytest.raises(NoApprovedStorageBox):
_ = urr.scp_hostname
with pytest.raises(NoApprovedStorageBox):
_ = urr.scp_username
with pytest.raises(NoApprovedStorageBox):
_ = urr.scp_port
with pytest.raises(NoApprovedStorageBox):
_ = urr.location
# Now let's test requesting staging access when we have an
# existing UploaderRegistrationRequest:
with requests_mock.Mocker() as mocker:
mock_uploader_update_response(mocker, settings)
get_urr_url = (
"%s/api/v1/mydata_uploaderregistrationrequest/?format=json"
"&uploader__uuid=00000000001&requester_key_fingerprint=%s") % (
settings.general.mytardis_url,
settings.uploader.ssh_key_pair.fingerprint)
mocker.get(get_urr_url, text=MOCK_URR_MISSING_SBOX_ATTRS)
settings.uploader.request_staging_access()
urr = settings.uploader.upload_to_staging_request
assert urr.approved
assert urr.approved_storage_box
with pytest.raises(StorageBoxAttributeNotFound):
_ = urr.scp_hostname
with pytest.raises(StorageBoxAttributeNotFound):
_ = urr.scp_username
# If the scp_port storage box attribute is missing, it defaults to 22:
assert urr.scp_port == "22"
with pytest.raises(StorageBoxOptionNotFound):
_ = urr.location
def test_config_generate_command():
"""Test mydata config generate
Mock the MyTardis API responses required for settings validation
"""
from mydata.commands.config import config_cmd
from mydata.conf import settings
with tempfile.NamedTemporaryFile() as tmpfile:
settings.config_path = tmpfile.name
settings.set_default_config()
with requests_mock.Mocker() as mocker:
inputs = dict(
mytardis_url="http://mytardis.example.com",
username="******",
api_key="api_key1",
facility="Test Facility",
instrument="Test Instrument",
data_directory=os.path.join(".", "tests", "testdata",
"testdata-email-dataset"),
contact_name="Joe Bloggs",
contact_email="*****@*****.**",
)
mock_api_endpoints_response(mocker, inputs["mytardis_url"])
mock_testfacility_user_response(mocker, inputs["mytardis_url"])
mock_testuser_response(mocker, settings, inputs["username"])
mock_test_facility_response(mocker, inputs["mytardis_url"])
mock_test_instrument_response(mocker, inputs["mytardis_url"])
runner = CliRunner()
stdin = "\n".join([
inputs["mytardis_url"],
inputs["username"],
inputs["api_key"],
inputs["facility"],
inputs["instrument"],
inputs["data_directory"],
inputs["contact_name"],
inputs["contact_email"],
])
result = runner.invoke(config_cmd, ["generate"], input=stdin)
if result.exception:
raise result.exception
assert result.output == textwrap.dedent("""
MyTardis URL: http://mytardis.example.com
MyTardis Username: testuser1
MyTardis API key: api_key1
Facility Name: Test Facility
Instrument Name: Test Instrument
Data Directory: ./tests/testdata/testdata-email-dataset
Contact Name: Joe Bloggs
Contact Email: [email protected]
Wrote settings to: %s
""" % settings.config_path)
# We need to ensure that changes to settings singleton don't propagate
# to subsequent tests.
# For other tests, we're doing this in tests/fixtures.py, but this
# test doesn't use a fixture:
unload_modules()
def test_post_uploads(set_username_dataset_config):
"""
Test POST uploads
"""
from mydata.conf import settings
from mydata.tasks.folders import scan_folders
from mydata.tasks.uploads import upload_folder
from mydata.models.lookup import LookupStatus
from mydata.models.upload import UploadStatus, UploadMethod
users = []
folders = []
def found_user(user):
users.append(user)
found_exp = None
found_group = None
def found_dataset(folder):
folders.append(folder)
with requests_mock.Mocker() as mocker:
mock_testfacility_user_response(mocker, settings.general.mytardis_url)
mock_testusers_response(mocker, settings, ["testuser1", "testuser2"])
mock_invalid_user_response(mocker, settings)
mock_test_facility_response(mocker, settings.general.mytardis_url)
mock_test_instrument_response(mocker, settings.general.mytardis_url)
scan_folders(found_user, found_group, found_exp, found_dataset)
assert sorted([user.username for user in users]) == [
"INVALID_USER",
"testuser1",
"testuser2",
]
assert sorted([folder.name for folder in folders]) == [
"Birds",
"Dataset with spaces",
"Flowers",
"InvalidUserDataset1",
]
assert sum([folder.num_files for folder in folders]) == 12
with requests_mock.Mocker() as mocker:
mock_test_facility_response(mocker, settings.general.mytardis_url)
mock_test_instrument_response(mocker, settings.general.mytardis_url)
for username, name in [
("testuser1", "Test User1"),
("testuser2", "Test User2"),
("INVALID_USER", "INVALID_USER (USER NOT FOUND IN MYTARDIS)"),
]:
title = "Test Instrument - %s" % name
user_folder_name = username
mock_exp_creation(mocker, settings, title, user_folder_name)
for folder in folders:
get_dataset_url = (
"%s/api/v1/dataset/?format=json&experiments__id=1"
"&description=%s&instrument__id=1") % (
settings.general.mytardis_url, quote(folder.name))
mocker.get(get_dataset_url, text=EMPTY_LIST_RESPONSE)
get_df_url_template = Template((
"%s/api/v1/mydata_dataset_file/?format=json&dataset__id=1&filename=$filename&directory="
) % settings.general.mytardis_url)
for dfi in range(0, folder.num_files):
datafile_path = folder.get_datafile_path(dfi)
datafile_name = os.path.basename(datafile_path)
get_datafile_url = get_df_url_template.substitute(
filename=quote(datafile_name))
mocker.get(get_datafile_url, text=EMPTY_LIST_RESPONSE)
post_dataset_url = "%s/api/v1/dataset/" % settings.general.mytardis_url
post_datafile_url = ("%s/api/v1/mydata_dataset_file/" %
settings.general.mytardis_url)
mocker.post(post_datafile_url, status_code=201)
lookups = []
uploads = []
def lookup_callback(lookup):
msg = "File: %s had unexpected lookup result: %s" % (
lookup.filename,
lookup.message,
)
assert lookup.status == LookupStatus.NOT_FOUND, msg
lookups.append(lookup)
def upload_callback(upload):
msg = "File: %s had unexpected upload result: %s" % (
upload.filename,
upload.message,
)
assert upload.status == UploadStatus.COMPLETED, msg
uploads.append(upload)
for folder in folders:
mock_dataset_response = created_dataset_response(1, folder.name)
mocker.post(post_dataset_url, text=mock_dataset_response)
upload_folder(folder, lookup_callback, upload_callback,
UploadMethod.MULTIPART_POST)
# Ensure that all 12 files were looked up:
assert len(lookups) == 12
# Ensure that all 12 files were uploaded:
assert len(uploads) == 12
