class MaximumWorkflowInvocationDurationTestCase(integration_util.IntegrationTestCase):
framework_tool_and_types = True
def setUp(self):
super(MaximumWorkflowInvocationDurationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.workflow_populator = WorkflowPopulator(self.galaxy_interactor)
@classmethod
def handle_galaxy_config_kwds(cls, config):
config["maximum_workflow_invocation_duration"] = 20
def do_test(self):
workflow = self.workflow_populator.load_workflow_from_resource("test_workflow_pause")
workflow_id = self.workflow_populator.create_workflow(workflow)
history_id = self.dataset_populator.new_history()
hda1 = self.dataset_populator.new_dataset(history_id, content="1 2 3")
index_map = {
'0': dict(src="hda", id=hda1["id"])
}
request = {}
request["history"] = "hist_id=%s" % history_id
request["inputs"] = dumps(index_map)
request["inputs_by"] = 'step_index'
url = "workflows/%s/invocations" % (workflow_id)
invocation_response = self._post(url, data=request)
invocation_url = url + "/" + invocation_response.json()["id"]
time.sleep(5)
state = self._get(invocation_url).json()["state"]
assert state != "failed", state
time.sleep(35)
state = self._get(invocation_url).json()["state"]
assert state == "failed", state
class JobRecoveryAfterHandledIntegerationTestCase(
integration_util.IntegrationTestCase):
framework_tool_and_types = True
def setUp(self):
super(JobRecoveryAfterHandledIntegerationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
@classmethod
def handle_galaxy_config_kwds(cls, config):
config["job_config_file"] = DELAY_JOB_CONFIG_FILE
def handle_reconfigure_galaxy_config_kwds(self, config):
config["job_config_file"] = SIMPLE_JOB_CONFIG_FILE
def test_recovery(self):
history_id = self.dataset_populator.new_history()
self.dataset_populator.run_tool(
"exit_code_oom",
{},
history_id,
assert_ok=False,
).json()
self.restart(
handle_reconfig=self.handle_reconfigure_galaxy_config_kwds)
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.dataset_populator.wait_for_history(history_id, assert_ok=True)
class MaximumWorkflowInvocationDurationTestCase(
integration_util.IntegrationTestCase):
"""Start a Pulsar job."""
framework_tool_and_types = True
def setUp(self):
super(MaximumWorkflowInvocationDurationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.workflow_populator = WorkflowPopulator(self.galaxy_interactor)
@classmethod
def handle_galaxy_config_kwds(cls, config):
config["maximum_workflow_invocation_duration"] = 20
def do_test(self):
workflow = self.workflow_populator.load_workflow_from_resource(
"test_workflow_pause")
workflow_id = self.workflow_populator.create_workflow(workflow)
history_id = self.dataset_populator.new_history()
hda1 = self.dataset_populator.new_dataset(history_id, content="1 2 3")
index_map = {'0': dict(src="hda", id=hda1["id"])}
request = {}
request["history"] = "hist_id=%s" % history_id
request["inputs"] = dumps(index_map)
request["inputs_by"] = 'step_index'
url = "workflows/%s/invocations" % (workflow_id)
invocation_response = self._post(url, data=request)
invocation_url = url + "/" + invocation_response.json()["id"]
time.sleep(5)
state = self._get(invocation_url).json()["state"]
assert state != "failed", state
time.sleep(35)
state = self._get(invocation_url).json()["state"]
assert state == "failed", state
def setUp(self):
super(HistoryContentsApiTestCase, self).setUp()
self.history_id = self._new_history()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.dataset_collection_populator = DatasetCollectionPopulator(
self.galaxy_interactor)
self.library_populator = LibraryPopulator(self.galaxy_interactor)
class JobRecoveryAfterHandledIntegerationTestCase(integration_util.IntegrationTestCase):
framework_tool_and_types = True
def setUp(self):
super(JobRecoveryAfterHandledIntegerationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
@classmethod
def handle_galaxy_config_kwds(cls, config):
config["job_config_file"] = DELAY_JOB_CONFIG_FILE
def handle_reconfigure_galaxy_config_kwds(self, config):
config["job_config_file"] = SIMPLE_JOB_CONFIG_FILE
def test_recovery(self):
history_id = self.dataset_populator.new_history()
self.dataset_populator.run_tool(
"exit_code_oom",
{},
history_id,
assert_ok=False,
).json()
self.restart(handle_reconfig=self.handle_reconfigure_galaxy_config_kwds)
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.dataset_populator.wait_for_history(history_id, assert_ok=True)
def test_400_on_invalid_embedded_content(self):
dataset_populator = DatasetPopulator(self.galaxy_interactor)
valid_id = dataset_populator.new_history()
page_request = self._test_page_payload(slug="invalid-id-encding")
page_request["content"] = '''<p>Page!<div class="embedded-item" id="CoolObject-%s"></div></p>''' % valid_id
page_response = self._post("pages", page_request)
self._assert_status_code_is(page_response, 400)
self._assert_error_code_is(page_response, error_codes.USER_REQUEST_INVALID_PARAMETER)
assert "embedded HTML content" in page_response.text
class DataManagerIntegrationTestCase(integration_util.IntegrationTestCase, UsesShed):
"""Test data manager installation and table reload through the API"""
framework_tool_and_types = True
use_shared_connection_for_amqp = True
def setUp(self):
super(DataManagerIntegrationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
@classmethod
def handle_galaxy_config_kwds(cls, config):
try:
import watchdog # noqa: F401
except ImportError:
raise SkipTest("watchdog library is not available")
cls.configure_shed_and_conda(config)
config["tool_data_path"] = cls.shed_tool_data_dir
config["watch_tool_data_dir"] = True
cls.username = cls.get_secure_ascii_digits()
config["admin_users"] = "*****@*****.**" % cls.username
@skip_if_toolshed_down
def test_data_manager_installation_table_reload(self):
"""
Test that we can install data managers, create a new dbkey, and use that dbkey in a downstream data manager.
"""
self.install_repository("devteam", "data_manager_fetch_genome_dbkeys_all_fasta", "b1bc53e9bbc5")
self.install_repository("devteam", "data_manager_sam_fasta_index_builder", "406896e00d0e", 'https://testtoolshed.g2.bx.psu.edu')
with self._different_user(email="*****@*****.**" % self.username):
with self.dataset_populator.test_history() as history_id:
run_response = self.dataset_populator.run_tool(tool_id=FETCH_TOOL_ID,
inputs=FETCH_GENOME_DBKEYS_ALL_FASTA_INPUT,
history_id=history_id,
assert_ok=False)
self.dataset_populator.wait_for_tool_run(history_id=history_id, run_response=run_response, timeout=CONDA_AUTO_INSTALL_JOB_TIMEOUT)
run_response = self.dataset_populator.run_tool(tool_id=SAM_FASTA_ID,
inputs=SAM_FASTA_INPUT,
history_id=history_id,
assert_ok=False)
self.dataset_populator.wait_for_tool_run(history_id=history_id, run_response=run_response, timeout=CONDA_AUTO_INSTALL_JOB_TIMEOUT)
def test_data_manager_manual(self):
"""
Test that data_manager_manual works, which uses a signigicant amount of Galaxy-internal code
"""
self.install_repository('iuc', 'data_manager_manual', '1ed87dee9e68')
with self._different_user(email="*****@*****.**" % self.username):
with self.dataset_populator.test_history() as history_id:
self.dataset_populator.run_tool(tool_id=DATA_MANAGER_MANUAL_ID,
inputs=DATA_MANAGER_MANUAL_INPUT,
history_id=history_id)
@classmethod
def get_secure_ascii_digits(cls, n=12):
return ''.join(random.SystemRandom().choice(string.ascii_lowercase + string.digits) for _ in range(12))
class FailJobWhenToolUnavailableTestCase(integration_util.IntegrationTestCase):
require_admin_user = True
def setUp(self):
super(FailJobWhenToolUnavailableTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.workflow_populator = WorkflowPopulator(self.galaxy_interactor)
self.history_id = self.dataset_populator.new_history()
@classmethod
def handle_galaxy_config_kwds(
cls,
config,
):
# config["jobs_directory"] = cls.jobs_directory
# Disable tool dependency resolution.
config["tool_dependency_dir"] = "none"
def test_fail_job_when_tool_unavailable(self):
self.workflow_populator.run_workflow("""
class: GalaxyWorkflow
steps:
- label: sleep
run:
class: GalaxyTool
command: sleep 20s && echo 'hello world 2' > '$output1'
outputs:
output1:
format: txt
- tool_id: cat1
state:
input1:
$link: sleep#output1
queries:
input2:
$link: sleep#output1
""",
history_id=self.history_id,
assert_ok=False,
wait=False)
# Wait until workflow is fully scheduled, otherwise can't test effect of removing tool from queued job
time.sleep(10)
self._app.toolbox.remove_tool_by_id('cat1')
self.dataset_populator.wait_for_history(self.history_id,
assert_ok=False)
state_details = self.galaxy_interactor.get(
'histories/%s' % self.history_id).json()['state_details']
assert state_details['running'] == 0
assert state_details['ok'] == 1
assert state_details['error'] == 1
failed_hda = self.dataset_populator.get_history_dataset_details(
history_id=self.history_id, assert_ok=False, details=True)
assert failed_hda['state'] == 'error'
job = self.galaxy_interactor.get("jobs/%s" %
failed_hda['creating_job']).json()
assert job['state'] == 'error'
def test_recovery(self):
history_id = self.dataset_populator.new_history()
self.dataset_populator.run_tool(
"exit_code_oom",
{},
history_id,
assert_ok=False,
).json()
self.restart(
handle_reconfig=self.handle_reconfigure_galaxy_config_kwds)
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.dataset_populator.wait_for_history(history_id, assert_ok=True)
class TestProvenance(api.ApiTestCase):
def setUp(self):
super(TestProvenance, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
def test_show_prov(self):
history_id = self.dataset_populator.new_history()
new_dataset1 = self.dataset_populator.new_dataset(history_id, content='for prov')
prov_response = self._get("histories/%s/contents/%s/provenance" % (history_id, new_dataset1["id"]))
self._assert_status_code_is(prov_response, 200)
self._assert_has_keys(prov_response.json(), "job_id", "id", "stdout", "stderr", "parameters", "tool_id")
class TestProvenance(api.ApiTestCase):
def setUp(self):
super(TestProvenance, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
def test_show_prov(self):
history_id = self.dataset_populator.new_history()
new_dataset1 = self.dataset_populator.new_dataset(history_id,
content='for prov')
prov_response = self._get("histories/%s/contents/%s/provenance" %
(history_id, new_dataset1["id"]))
self._assert_status_code_is(prov_response, 200)
self._assert_has_keys(prov_response.json(), "job_id", "id", "stdout",
"stderr", "parameters", "tool_id")
class PageJsonEncodingIntegrationTestCase(integration_util.IntegrationTestCase
):
def setUp(self):
super(PageJsonEncodingIntegrationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
def test_page_encoding(self):
history_id = self.dataset_populator.new_history()
request = dict(
slug="mypage",
title="MY PAGE",
content=
'''<p>Page!<div class="embedded-item" id="History-%s"></div></p>'''
% history_id,
)
page_response = self._post("pages", request)
api_asserts.assert_status_code_is_ok(page_response)
sa_session = self._app.model.context
page_revision = sa_session.query(model.PageRevision).all()[0]
assert '''id="History-1"''' in page_revision.content, page_revision.content
assert '''id="History-%s"''' % history_id not in page_revision.content, page_revision.content
show_page_response = self._get("pages/%s" % page_response.json()["id"])
api_asserts.assert_status_code_is_ok(show_page_response)
content = show_page_response.json()["content"]
assert '''id="History-1"''' not in content, content
assert '''id="History-%s"''' % history_id in content, content
class BaseCheckUploadContentConfigurationTestCase(integration_util.IntegrationTestCase):
framework_tool_and_types = True
def setUp(self):
super(BaseCheckUploadContentConfigurationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.history_id = self.dataset_populator.new_history()
class BaseUploadContentConfigurationTestCase(integration_util.IntegrationTestCase):
framework_tool_and_types = True
def setUp(self):
super(BaseUploadContentConfigurationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.library_populator = LibraryPopulator(self.galaxy_interactor)
self.history_id = self.dataset_populator.new_history()
def test_legacy_r_mapping(self):
"""
"""
tool_id = "legacy_R"
dataset_populator = DatasetPopulator(self.galaxy_interactor)
history_id = dataset_populator.new_history()
endpoint = "tools/%s/install_dependencies" % tool_id
data = {'id': tool_id}
create_response = self._post(endpoint, data=data, admin=True)
self._assert_status_code_is(create_response, 200)
payload = dataset_populator.run_tool_payload(
tool_id=tool_id,
inputs={},
history_id=history_id,
)
create_response = self._post("tools", data=payload)
self._assert_status_code_is(create_response, 200)
dataset_populator.wait_for_history(history_id, assert_ok=True)
class BaseWorkflowHandlerConfigurationTestCase(integration_util.IntegrationTestCase):
framework_tool_and_types = True
def setUp(self):
super(BaseWorkflowHandlerConfigurationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.workflow_populator = WorkflowPopulator(self.galaxy_interactor)
self.history_id = self.dataset_populator.new_history()
@classmethod
def handle_galaxy_config_kwds(cls, config):
config["job_config_file"] = WORKFLOW_HANDLER_CONFIGURATION_JOB_CONF
def _invoke_n_workflows(self, n):
workflow_id = self.workflow_populator.upload_yaml_workflow(PAUSE_WORKFLOW)
history_id = self.history_id
hda1 = self.dataset_populator.new_dataset(history_id, content="1 2 3")
index_map = {
'0': dict(src="hda", id=hda1["id"])
}
request = {}
request["history"] = "hist_id=%s" % history_id
request["inputs"] = dumps(index_map)
request["inputs_by"] = 'step_index'
url = "workflows/%s/invocations" % (workflow_id)
for i in range(n):
self._post(url, data=request)
def _get_workflow_invocations(self):
# Consider exposing handler via the API to reduce breaking
# into Galaxy's internal state.
app = self._app
history_id = app.security.decode_id(self.history_id)
sa_session = app.model.context.current
history = sa_session.query(app.model.History).get(history_id)
workflow_invocations = history.workflow_invocations
return workflow_invocations
@property
def is_app_workflow_scheduler(self):
return self._app.workflow_scheduling_manager.request_monitor is not None
class BaseUploadContentConfigurationTestCase(
integration_util.IntegrationTestCase):
framework_tool_and_types = True
def setUp(self):
super(BaseUploadContentConfigurationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.library_populator = LibraryPopulator(self.galaxy_interactor)
self.history_id = self.dataset_populator.new_history()
def fetch_target(self, target, assert_ok=False, attach_test_file=False):
payload = {
"history_id": self.history_id,
"targets": json.dumps([target]),
}
if attach_test_file:
payload["__files"] = {
"files_0|file_data":
open(self.test_data_resolver.get_filename("4.bed"))
}
response = self.dataset_populator.fetch(payload, assert_ok=assert_ok)
return response
@classmethod
def temp_config_dir(cls, name):
# realpath here to get around problems with symlinks being blocked.
return os.path.realpath(
os.path.join(cls._test_driver.galaxy_test_tmp_dir, name))
def _write_file(self, dir_path, content, filename="test"):
"""Helper for writing ftp/server dir files."""
self._ensure_directory(dir_path)
path = os.path.join(dir_path, filename)
with open(path, "w") as f:
f.write(content)
return path
def _ensure_directory(self, path):
if not os.path.exists(path):
os.makedirs(path)
def test_recovery(self):
history_id = self.dataset_populator.new_history()
self.dataset_populator.run_tool(
"exit_code_oom",
{},
history_id,
assert_ok=False,
).json()
self.restart(handle_reconfig=self.handle_reconfigure_galaxy_config_kwds)
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.dataset_populator.wait_for_history(history_id, assert_ok=True)
def setUp(self):
super(ObjectStoreJobsIntegrationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
with self.dataset_populator.test_history() as history_id:
hda1 = self.dataset_populator.new_dataset(
history_id, content=TEST_INPUT_FILES_CONTENT)
create_10_inputs = {
"input1": {
"src": "hda",
"id": hda1["id"]
},
"input2": {
"src": "hda",
"id": hda1["id"]
},
}
self.dataset_populator.run_tool(
"create_10",
create_10_inputs,
history_id,
assert_ok=True,
)
self.dataset_populator.wait_for_history(history_id)
class BaseUploadContentConfigurationTestCase(integration_util.IntegrationTestCase):
framework_tool_and_types = True
def setUp(self):
super(BaseUploadContentConfigurationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.library_populator = LibraryPopulator(self.galaxy_interactor)
self.history_id = self.dataset_populator.new_history()
def fetch_target(self, target, assert_ok=False, attach_test_file=False):
payload = {
"history_id": self.history_id,
"targets": json.dumps([target]),
}
if attach_test_file:
payload["__files"] = {"files_0|file_data": open(self.test_data_resolver.get_filename("4.bed"))}
response = self.dataset_populator.fetch(payload, assert_ok=assert_ok)
return response
@classmethod
def temp_config_dir(cls, name):
# realpath here to get around problems with symlinks being blocked.
return os.path.realpath(os.path.join(cls._test_driver.galaxy_test_tmp_dir, name))
def _write_file(self, dir_path, content, filename="test"):
"""Helper for writing ftp/server dir files."""
self._ensure_directory(dir_path)
path = os.path.join(dir_path, filename)
with open(path, "w") as f:
f.write(content)
return path
def _ensure_directory(self, path):
if not os.path.exists(path):
os.makedirs(path)
class ObjectStoreJobsIntegrationTestCase(integration_util.IntegrationTestCase):
framework_tool_and_types = True
@classmethod
def handle_galaxy_config_kwds(cls, config):
temp_directory = cls._test_driver.mkdtemp()
cls.object_stores_parent = temp_directory
for disk_store_file_name in ["files1", "files2", "files3"]:
disk_store_path = os.path.join(temp_directory, disk_store_file_name)
os.makedirs(disk_store_path)
setattr(cls, "%s_path" % disk_store_file_name, disk_store_path)
config_path = os.path.join(temp_directory, "object_store_conf.xml")
with open(config_path, "w") as f:
f.write(DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE.safe_substitute({"temp_directory": temp_directory}))
config["object_store_config_file"] = config_path
def setUp(self):
super(ObjectStoreJobsIntegrationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
def test_tool_simple_constructs(self):
with self.dataset_populator.test_history() as history_id:
hda1 = self.dataset_populator.new_dataset(history_id, content="1 2 3")
create_10_inputs = {
"input1": {"src": "hda", "id": hda1["id"]},
"input2": {"src": "hda", "id": hda1["id"]},
}
self.dataset_populator.run_tool(
"create_10",
create_10_inputs,
history_id,
assert_ok=True,
)
self.dataset_populator.wait_for_history(history_id)
files_1_count = _files_count(self.files1_path)
files_2_count = _files_count(self.files2_path)
files_3_count = _files_count(self.files3_path)
# Ensure no files written to the secondary/inactive hierarchical disk store.
assert files_3_count == 0
# Ensure the 10 inputs were written to one of the distributed object store's disk
# stores (it will have either 10 or 11 depeending on whether the input was also
# written there. The other disk store may or may not have the input file so should
# have at most one file.
assert (files_1_count >= 10) or (files_2_count >= 10)
assert (files_1_count <= 1) or (files_2_count <= 1)
# Other sanity checks on the test - just make sure the test was setup as intended
# and not actually testing object store behavior.
assert (files_1_count <= 11) and (files_2_count <= 11)
assert (files_1_count >= 0) and (files_2_count >= 0)
class BaseEmbeddedPulsarContainerIntegrationTestCase(
integration_util.IntegrationTestCase):
framework_tool_and_types = True
@classmethod
def handle_galaxy_config_kwds(cls, config):
cls.jobs_directory = cls._test_driver.mkdtemp()
config["jobs_directory"] = cls.jobs_directory
config["job_config_file"] = cls.job_config_file
disable_dependency_resolution(config)
def setUp(self):
super(BaseEmbeddedPulsarContainerIntegrationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.history_id = self.dataset_populator.new_history()
@classmethod
def setUpClass(cls):
skip_if_container_type_unavailable(cls)
super(BaseEmbeddedPulsarContainerIntegrationTestCase, cls).setUpClass()
class DataManagerIntegrationTestCase(integration_util.IntegrationTestCase, UsesShed):
"""Test data manager installation and table reload through the API"""
framework_tool_and_types = True
def setUp(self):
super(DataManagerIntegrationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
@classmethod
def handle_galaxy_config_kwds(cls, config):
try:
import watchdog # noqa: F401
except ImportError:
raise SkipTest("watchdog library is not available")
cls.configure_shed_and_conda(config)
config["tool_data_path"] = cls.shed_tool_data_dir
config["watch_tool_data_dir"] = True
cls.username = cls.get_secure_ascii_digits()
config["admin_users"] = "*****@*****.**" % cls.username
def test_data_manager_installation_table_reload(self):
"""
Test that we can install data managers, create a new dbkey, and use that dbkey in a downstream data manager.
"""
self.install_repository("devteam", "data_manager_fetch_genome_dbkeys_all_fasta", "b1bc53e9bbc5")
self.install_repository("devteam", "data_manager_sam_fasta_index_builder", "1865e693d8b2")
with self._different_user(email="*****@*****.**" % self.username):
with self.dataset_populator.test_history() as history_id:
run_response = self.dataset_populator.run_tool(tool_id=FETCH_TOOL_ID,
inputs=FETCH_GENOME_DBKEYS_ALL_FASTA_INPUT,
history_id=history_id,
assert_ok=False)
self.dataset_populator.wait_for_tool_run(history_id=history_id, run_response=run_response, timeout=CONDA_AUTO_INSTALL_JOB_TIMEOUT)
run_response = self.dataset_populator.run_tool(tool_id=SAM_FASTA_ID,
inputs=SAM_FASTA_INPUT,
history_id=history_id,
assert_ok=False)
self.dataset_populator.wait_for_tool_run(history_id=history_id, run_response=run_response, timeout=CONDA_AUTO_INSTALL_JOB_TIMEOUT)
@classmethod
def get_secure_ascii_digits(cls, n=12):
return ''.join(random.SystemRandom().choice(string.ascii_lowercase + string.digits) for _ in range(12))
def test_runs_on_mule(self):
tool_id = 'config_vars'
expect_server_name = self.expected_server_name
dataset_populator = DatasetPopulator(self.galaxy_interactor)
history_id = dataset_populator.new_history()
payload = dataset_populator.run_tool(
tool_id=tool_id,
inputs={'var': 'server_name'},
history_id=history_id,
)
dataset_id = payload['outputs'][0]['id']
dataset_populator.wait_for_dataset(history_id, dataset_id, assert_ok=True)
output = dataset_populator.get_history_dataset_content(history_id, dataset_id=dataset_id).strip()
assert output.startswith(expect_server_name), (
"Job handler's server name '{output}' does not start with expected string '{expected}'".format(
output=output,
expected=expect_server_name,
)
)
def test_legacy_r_mapping(self):
"""
"""
tool_id = "legacy_R"
dataset_populator = DatasetPopulator(self.galaxy_interactor)
history_id = dataset_populator.new_history()
endpoint = "tools/%s/install_dependencies" % tool_id
data = {'id': tool_id}
create_response = self._post(endpoint, data=data, admin=True)
self._assert_status_code_is(create_response, 200)
payload = dataset_populator.run_tool_payload(
tool_id=tool_id,
inputs={},
history_id=history_id,
)
create_response = self._post("tools", data=payload)
self._assert_status_code_is(create_response, 200)
dataset_populator.wait_for_history(history_id, assert_ok=True)
class MaximumWorkflowJobsPerSchedulingIterationTestCase(integration_util.IntegrationTestCase):
framework_tool_and_types = True
def setUp(self):
super(MaximumWorkflowJobsPerSchedulingIterationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.workflow_populator = WorkflowPopulator(self.galaxy_interactor)
self.dataset_collection_populator = DatasetCollectionPopulator(self.galaxy_interactor)
@classmethod
def handle_galaxy_config_kwds(cls, config):
config["maximum_workflow_jobs_per_scheduling_iteration"] = 1
def do_test(self):
workflow_id = self.workflow_populator.upload_yaml_workflow("""
class: GalaxyWorkflow
steps:
- type: input_collection
- tool_id: collection_creates_pair
state:
input1:
$link: 0
- tool_id: collection_paired_test
state:
f1:
$link: 1#paired_output
- tool_id: cat_list
state:
input1:
$link: 2#out1
""")
with self.dataset_populator.test_history() as history_id:
hdca1 = self.dataset_collection_populator.create_list_in_history(history_id, contents=["a\nb\nc\nd\n", "e\nf\ng\nh\n"]).json()
self.dataset_populator.wait_for_history(history_id, assert_ok=True)
inputs = {
'0': {"src": "hdca", "id": hdca1["id"]},
}
invocation_id = self.workflow_populator.invoke_workflow(history_id, workflow_id, inputs)
self.workflow_populator.wait_for_workflow(history_id, workflow_id, invocation_id)
self.dataset_populator.wait_for_history(history_id, assert_ok=True)
self.assertEqual("a\nc\nb\nd\ne\ng\nf\nh\n", self.dataset_populator.get_history_dataset_content(history_id, hid=0))
def setUp(self):
super(HistoryContentsApiTestCase, self).setUp()
self.history_id = self._new_history()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.dataset_collection_populator = DatasetCollectionPopulator(self.galaxy_interactor)
self.library_populator = LibraryPopulator(self.galaxy_interactor)
def setUp(self):
super(ScriptsIntegrationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.config_dir = tempfile.mkdtemp()
class ScriptsIntegrationTestCase(integration_util.IntegrationTestCase):
def setUp(self):
super(ScriptsIntegrationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.config_dir = tempfile.mkdtemp()
@classmethod
def handle_galaxy_config_kwds(cls, config):
cls._raw_config = config
def test_helper(self):
script = "helper.py"
self._scripts_check_argparse_help(script)
history_id = self.dataset_populator.new_history()
dataset = self.dataset_populator.new_dataset(history_id, wait=True)
dataset_id = dataset["id"]
config_file = self.write_config_file()
output = self._scripts_check_output(script, ["-c", config_file, "--decode-id", dataset_id])
assert "Decoded " in output
def test_cleanup(self):
script = "cleanup_datasets/cleanup_datasets.py"
self._scripts_check_argparse_help(script)
history_id = self.dataset_populator.new_history()
delete_response = self.dataset_populator._delete("histories/%s" % history_id)
assert delete_response.status_code == 200
assert delete_response.json()["purged"] is False
config_file = self.write_config_file()
output = self._scripts_check_output(script, ["-c", config_file, "--days", "0", "--purge_histories"])
print(output)
history_response = self.dataset_populator._get("histories/%s" % history_id)
assert history_response.status_code == 200
assert history_response.json()["purged"] is True, history_response.json()
def test_pgcleanup(self):
self._skip_if_not_postgres()
script = "cleanup_datasets/pgcleanup.py"
self._scripts_check_argparse_help(script)
history_id = self.dataset_populator.new_history()
delete_response = self.dataset_populator._delete("histories/%s" % history_id)
assert delete_response.status_code == 200
assert delete_response.json()["purged"] is False
config_file = self.write_config_file()
output = self._scripts_check_output(script, ["-c", config_file, "--older-than", "0", "--sequence", "purge_deleted_histories"])
print(output)
history_response = self.dataset_populator._get("histories/%s" % history_id)
assert history_response.status_code == 200
assert history_response.json()["purged"] is True, history_response.json()
def test_set_user_disk_usage(self):
script = "set_user_disk_usage.py"
self._scripts_check_argparse_help(script)
history_id = self.dataset_populator.new_history()
self.dataset_populator.new_dataset(history_id, wait=True)
config_file = self.write_config_file()
output = self._scripts_check_output(script, ["-c", config_file])
# verify the script runs to completion without crashing
assert "100% complete" in output, output
def test_set_dataset_sizes(self):
script = "set_dataset_sizes.py"
self._scripts_check_argparse_help(script)
# TODO: change the size of the dataset and verify this works.
history_id = self.dataset_populator.new_history()
self.dataset_populator.new_dataset(history_id, wait=True)
config_file = self.write_config_file()
output = self._scripts_check_output(script, ["-c", config_file])
# verify the script runs to completion without crashing
assert "Completed 100%" in output, output
def test_populate_uuid(self):
script = "cleanup_datasets/populate_uuid.py"
self._scripts_check_argparse_help(script)
history_id = self.dataset_populator.new_history()
self.dataset_populator.new_dataset(history_id, wait=True)
config_file = self.write_config_file()
output = self._scripts_check_output(script, ["-c", config_file])
assert "Complete" in output
@integration_util.skip_if_jenkins
def test_grt_export(self):
script = "grt/export.py"
self._scripts_check_argparse_help(script)
history_id = self.dataset_populator.new_history()
self.dataset_populator.new_dataset(history_id, wait=True)
config_file = self.write_config_file()
grt_config_file = os.path.join(self.config_dir, "grt.yml")
with open(grt_config_file, "w") as f:
yaml.dump({"grt": {"share_toolbox": True}, "sanitization": {"tools": []}, "tool_params": {}}, f)
self._scripts_check_output(script, ["-c", config_file, "-g", grt_config_file, "-r", self.config_dir])
report_files = os.listdir(self.config_dir)
json_files = [j for j in report_files if j.endswith(".json")]
assert len(json_files) == 1, "Expected one json report file in [%s]" % json_files
json_file = os.path.join(self.config_dir, json_files[0])
with open(json_file, "r") as f:
export = json.load(f)
assert export["version"] == 2
def test_admin_cleanup_datasets(self):
self._scripts_check_argparse_help("cleanup_datasets/admin_cleanup_datasets.py")
@skip_unless_module("flask_socketio")
def test_communication_server(self):
self._scripts_check_argparse_help("communication/communication_server.py")
def test_secret_decoder_ring(self):
script = "secret_decoder_ring.py"
self._scripts_check_argparse_help(script)
config_file = self.write_config_file()
output = self._scripts_check_output(script, ["-c", config_file, "encode", "1"])
encoded_id = output.strip()
output = self._scripts_check_output(script, ["-c", config_file, "decode", encoded_id])
assert output.strip() == "1"
def test_database_scripts(self):
self._scripts_check_argparse_help("create_db.py")
self._scripts_check_argparse_help("manage_db.py")
# TODO: test creating a smaller database - e.g. tool install database based on fresh
# config file.
def test_galaxy_main(self):
self._scripts_check_argparse_help("galaxy-main")
def test_runtime_stats(self):
self._skip_if_not_postgres()
self._scripts_check_argparse_help("runtime_stats.py")
def _skip_if_not_postgres(self):
if not self._app.config.database_connection.startswith("post"):
raise unittest.SkipTest("Test only valid for postgres")
def _scripts_check_argparse_help(self, script):
# Test imports and argparse response to --help with 0 exit code.
output = self._scripts_check_output(script, ["--help"])
# Test -h, --help in printed output message.
assert "-h, --help" in output
def _scripts_check_output(self, script, args):
cwd = galaxy_directory()
cmd = ["python", os.path.join(cwd, "scripts", script)] + args
clean_env = {
"PATH": os.environ.get("PATH", None),
} # Don't let testing environment variables interfere with config.
return unicodify(subprocess.check_output(cmd, cwd=cwd, env=clean_env))
def write_config_file(self):
config_dir = self.config_dir
path = os.path.join(config_dir, "galaxy.yml")
self._test_driver.temp_directories.extend([config_dir])
with open(path, "w") as f:
yaml.dump({"galaxy": self._raw_config}, f)
return path
def setUp(self):
super(ToolsUploadTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
def setUp(self):
super(DataManagerIntegrationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
def setUp(self):
super(JobRecoveryAfterHandledIntegerationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
def setUp(self):
super(BaseWorkflowHandlerConfigurationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.workflow_populator = WorkflowPopulator(self.galaxy_interactor)
self.history_id = self.dataset_populator.new_history()
def setUp(self):
super(RolesApiTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
def setUp(self):
super(HistoriesApiTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.dataset_collection_populator = DatasetCollectionPopulator(
self.galaxy_interactor)
def setUp(self):
super(BaseUploadContentConfigurationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.library_populator = LibraryPopulator(self.galaxy_interactor)
self.history_id = self.dataset_populator.new_history()
def setUp(self):
super(MaximumWorkflowInvocationDurationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.workflow_populator = WorkflowPopulator(self.galaxy_interactor)
class HistoryContentsApiTestCase(api.ApiTestCase, TestsDatasets):
def setUp(self):
super(HistoryContentsApiTestCase, self).setUp()
self.history_id = self._new_history()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.dataset_collection_populator = DatasetCollectionPopulator(self.galaxy_interactor)
self.library_populator = LibraryPopulator(self.galaxy_interactor)
def test_index_hda_summary(self):
hda1 = self._new_dataset(self.history_id)
contents_response = self._get("histories/%s/contents" % self.history_id)
hda_summary = self.__check_for_hda(contents_response, hda1)
assert "display_types" not in hda_summary # Quick summary, not full details
def test_make_private_and_public(self):
hda1 = self._wait_for_new_hda()
update_url = "histories/%s/contents/%s/permissions" % (self.history_id, hda1["id"])
role_id = self.dataset_populator.user_private_role_id()
# Give manage permission to the user.
payload = {
"access": [],
"manage": [role_id],
}
update_response = self._update_permissions(update_url, payload, admin=True)
self._assert_status_code_is(update_response, 200)
self._assert_other_user_can_access(hda1["id"])
# Then we restrict access.
payload = {
"action": "make_private",
}
update_response = self._update_permissions(update_url, payload)
self._assert_status_code_is(update_response, 200)
self._assert_other_user_cannot_access(hda1["id"])
# Then we restrict access.
payload = {
"action": "remove_restrictions",
}
update_response = self._update_permissions(update_url, payload)
self._assert_status_code_is(update_response, 200)
self._assert_other_user_can_access(hda1["id"])
def test_set_permissions_add_admin_history_contents(self):
self._verify_dataset_permissions("history_contents")
def test_set_permissions_add_admin_datasets(self):
self._verify_dataset_permissions("dataset")
def _verify_dataset_permissions(self, api_endpoint):
hda1 = self._wait_for_new_hda()
hda_id = hda1["id"]
if api_endpoint == "history_contents":
update_url = "histories/%s/contents/%s/permissions" % (self.history_id, hda_id)
else:
update_url = "datasets/%s/permissions" % hda_id
role_id = self.dataset_populator.user_private_role_id()
payload = {
"access": [role_id],
"manage": [role_id],
}
# Other users cannot modify permissions.
with self._different_user():
update_response = self._update_permissions(update_url, payload)
self._assert_status_code_is(update_response, 403)
# First the details render for another user.
self._assert_other_user_can_access(hda_id)
# Then we restrict access.
update_response = self._update_permissions(update_url, payload, admin=True)
self._assert_status_code_is(update_response, 200)
# Finally the details don't render.
self._assert_other_user_cannot_access(hda_id)
# But they do for the original user.
contents_response = self._get("histories/%s/contents/%s" % (self.history_id, hda_id)).json()
assert "name" in contents_response
update_response = self._update_permissions(update_url, payload)
self._assert_status_code_is(update_response, 200)
payload = {
"access": [role_id],
"manage": [role_id],
}
update_response = self._update_permissions(update_url, payload)
self._assert_status_code_is(update_response, 200)
self._assert_other_user_cannot_access(hda_id)
user_id = self.dataset_populator.user_id()
with self._different_user():
different_user_id = self.dataset_populator.user_id()
combined_user_role = self.dataset_populator.create_role([user_id, different_user_id], description="role for testing permissions")
payload = {
"access": [combined_user_role["id"]],
"manage": [role_id],
}
update_response = self._update_permissions(update_url, payload)
self._assert_status_code_is(update_response, 200)
# Now other user can see dataset again with access permission.
self._assert_other_user_can_access(hda_id)
# access doesn't imply management though...
with self._different_user():
update_response = self._update_permissions(update_url, payload)
self._assert_status_code_is(update_response, 403)
def _assert_other_user_cannot_access(self, history_content_id):
with self._different_user():
contents_response = self._get("histories/%s/contents/%s" % (self.history_id, history_content_id)).json()
assert "name" not in contents_response
def _assert_other_user_can_access(self, history_content_id):
with self._different_user():
contents_response = self._get("histories/%s/contents/%s" % (self.history_id, history_content_id)).json()
assert "name" in contents_response
def test_index_hda_all_details(self):
hda1 = self._new_dataset(self.history_id)
contents_response = self._get("histories/%s/contents?details=all" % self.history_id)
hda_details = self.__check_for_hda(contents_response, hda1)
self.__assert_hda_has_full_details(hda_details)
def test_index_hda_detail_by_id(self):
hda1 = self._new_dataset(self.history_id)
contents_response = self._get("histories/%s/contents?details=%s" % (self.history_id, hda1["id"]))
hda_details = self.__check_for_hda(contents_response, hda1)
self.__assert_hda_has_full_details(hda_details)
def test_show_hda(self):
hda1 = self._new_dataset(self.history_id)
show_response = self.__show(hda1)
self._assert_status_code_is(show_response, 200)
self.__assert_matches_hda(hda1, show_response.json())
def test_hda_copy(self):
hda1 = self._new_dataset(self.history_id)
create_data = dict(
source='hda',
content=hda1["id"],
)
second_history_id = self._new_history()
assert self.__count_contents(second_history_id) == 0
create_response = self._post("histories/%s/contents" % second_history_id, create_data)
self._assert_status_code_is(create_response, 200)
assert self.__count_contents(second_history_id) == 1
def test_library_copy(self):
ld = self.library_populator.new_library_dataset("lda_test_library")
create_data = dict(
source='library',
content=ld["id"],
)
assert self.__count_contents(self.history_id) == 0
create_response = self._post("histories/%s/contents" % self.history_id, create_data)
self._assert_status_code_is(create_response, 200)
assert self.__count_contents(self.history_id) == 1
def test_update(self):
hda1 = self._wait_for_new_hda()
assert str(hda1["deleted"]).lower() == "false"
update_response = self._raw_update(hda1["id"], dict(deleted=True))
self._assert_status_code_is(update_response, 200)
show_response = self.__show(hda1)
assert str(show_response.json()["deleted"]).lower() == "true"
update_response = self._raw_update(hda1["id"], dict(name="Updated Name"))
assert self.__show(hda1).json()["name"] == "Updated Name"
update_response = self._raw_update(hda1["id"], dict(name="Updated Name"))
assert self.__show(hda1).json()["name"] == "Updated Name"
unicode_name = u'ржевский сапоги'
update_response = self._raw_update(hda1["id"], dict(name=unicode_name))
updated_hda = self.__show(hda1).json()
assert updated_hda["name"] == unicode_name, updated_hda
quoted_name = '"Mooo"'
update_response = self._raw_update(hda1["id"], dict(name=quoted_name))
updated_hda = self.__show(hda1).json()
assert updated_hda["name"] == quoted_name, quoted_name
def test_update_type_failures(self):
hda1 = self._wait_for_new_hda()
update_response = self._raw_update(hda1["id"], dict(deleted='not valid'))
self._assert_status_code_is(update_response, 400)
def _wait_for_new_hda(self):
hda1 = self._new_dataset(self.history_id)
self._wait_for_history(self.history_id)
return hda1
def _raw_update(self, item_id, data, admin=False, history_id=None):
history_id = history_id or self.history_id
key_param = "use_admin_key" if admin else "use_key"
update_url = self._api_url("histories/%s/contents/%s" % (history_id, item_id), **{key_param: True})
update_response = put(update_url, json=data)
return update_response
def _update_permissions(self, url, data, admin=False):
key_param = "use_admin_key" if admin else "use_key"
update_url = self._api_url(url, **{key_param: True})
update_response = put(update_url, json=data)
return update_response
def test_delete(self):
hda1 = self._new_dataset(self.history_id)
self._wait_for_history(self.history_id)
assert str(self.__show(hda1).json()["deleted"]).lower() == "false"
delete_response = self._delete("histories/%s/contents/%s" % (self.history_id, hda1["id"]))
assert delete_response.status_code < 300 # Something in the 200s :).
assert str(self.__show(hda1).json()["deleted"]).lower() == "true"
def test_purge(self):
hda1 = self._new_dataset(self.history_id)
self._wait_for_history(self.history_id)
assert str(self.__show(hda1).json()["deleted"]).lower() == "false"
assert str(self.__show(hda1).json()["purged"]).lower() == "false"
data = {'purge': True}
delete_response = self._delete("histories/%s/contents/%s" % (self.history_id, hda1["id"]), data=data)
assert delete_response.status_code < 300 # Something in the 200s :).
assert str(self.__show(hda1).json()["deleted"]).lower() == "true"
assert str(self.__show(hda1).json()["purged"]).lower() == "true"
def test_dataset_collection_creation_on_contents(self):
payload = self.dataset_collection_populator.create_pair_payload(
self.history_id,
type="dataset_collection"
)
endpoint = "histories/%s/contents" % self.history_id
self._check_pair_creation(endpoint, payload)
def test_dataset_collection_creation_on_typed_contents(self):
payload = self.dataset_collection_populator.create_pair_payload(
self.history_id,
)
endpoint = "histories/%s/contents/dataset_collections" % self.history_id
self._check_pair_creation(endpoint, payload)
def test_dataset_collection_create_from_exisiting_datasets_with_new_tags(self):
with self.dataset_populator.test_history() as history_id:
hda_id = self.dataset_populator.new_dataset(history_id, content="1 2 3")['id']
hda2_id = self.dataset_populator.new_dataset(history_id, content="1 2 3")['id']
update_response = self._raw_update(hda2_id, dict(tags=['existing:tag']), history_id=history_id).json()
assert update_response['tags'] == ['existing:tag']
creation_payload = {'collection_type': 'list',
'history_id': history_id,
'element_identifiers': json.dumps([{'id': hda_id,
'src': 'hda',
'name': 'element_id1',
'tags': ['my_new_tag']},
{'id': hda2_id,
'src': 'hda',
'name': 'element_id2',
'tags': ['another_new_tag']}
]),
'type': 'dataset_collection',
'copy_elements': True}
r = self._post("histories/%s/contents" % self.history_id, creation_payload).json()
assert r['elements'][0]['object']['id'] != hda_id, "HDA has not been copied"
assert len(r['elements'][0]['object']['tags']) == 1
assert r['elements'][0]['object']['tags'][0] == 'my_new_tag'
assert len(r['elements'][1]['object']['tags']) == 2, r['elements'][1]['object']['tags']
original_hda = self.dataset_populator.get_history_dataset_details(history_id=history_id, dataset_id=hda_id)
assert len(original_hda['tags']) == 0, original_hda['tags']
def _check_pair_creation(self, endpoint, payload):
pre_collection_count = self.__count_contents(type="dataset_collection")
pre_dataset_count = self.__count_contents(type="dataset")
pre_combined_count = self.__count_contents(type="dataset,dataset_collection")
dataset_collection_response = self._post(endpoint, payload)
dataset_collection = self.__check_create_collection_response(dataset_collection_response)
post_collection_count = self.__count_contents(type="dataset_collection")
post_dataset_count = self.__count_contents(type="dataset")
post_combined_count = self.__count_contents(type="dataset,dataset_collection")
# Test filtering types with index.
assert pre_collection_count == 0
assert post_collection_count == 1
assert post_combined_count == pre_dataset_count + 1
assert post_combined_count == pre_combined_count + 1
assert pre_dataset_count == post_dataset_count
# Test show dataset colleciton.
collection_url = "histories/%s/contents/dataset_collections/%s" % (self.history_id, dataset_collection["id"])
show_response = self._get(collection_url)
self._assert_status_code_is(show_response, 200)
dataset_collection = show_response.json()
self._assert_has_keys(dataset_collection, "url", "name", "deleted")
assert not dataset_collection["deleted"]
delete_response = delete(self._api_url(collection_url, use_key=True))
self._assert_status_code_is(delete_response, 200)
show_response = self._get(collection_url)
dataset_collection = show_response.json()
assert dataset_collection["deleted"]
@skip_without_tool("collection_creates_list")
def test_jobs_summary_simple_hdca(self):
create_response = self.dataset_collection_populator.create_list_in_history(self.history_id, contents=["a\nb\nc\nd", "e\nf\ng\nh"])
hdca_id = create_response.json()["id"]
run = self.dataset_populator.run_collection_creates_list(self.history_id, hdca_id)
collections = run['output_collections']
collection = collections[0]
jobs_summary_url = "histories/%s/contents/dataset_collections/%s/jobs_summary" % (self.history_id, collection["id"])
jobs_summary_response = self._get(jobs_summary_url)
self._assert_status_code_is(jobs_summary_response, 200)
jobs_summary = jobs_summary_response.json()
self._assert_has_keys(jobs_summary, "populated_state", "states")
@skip_without_tool("cat1")
def test_jobs_summary_implicit_hdca(self):
create_response = self.dataset_collection_populator.create_pair_in_history(self.history_id, contents=["123", "456"])
hdca_id = create_response.json()["id"]
inputs = {
"input1": {'batch': True, 'values': [{'src': 'hdca', 'id': hdca_id}]},
}
run = self.dataset_populator.run_tool("cat1", inputs=inputs, history_id=self.history_id)
self.dataset_populator.wait_for_history_jobs(self.history_id)
collections = run['implicit_collections']
collection = collections[0]
jobs_summary_url = "histories/%s/contents/dataset_collections/%s/jobs_summary" % (self.history_id, collection["id"])
jobs_summary_response = self._get(jobs_summary_url)
self._assert_status_code_is(jobs_summary_response, 200)
jobs_summary = jobs_summary_response.json()
self._assert_has_keys(jobs_summary, "populated_state", "states")
states = jobs_summary["states"]
assert states.get("ok") == 2, states
def test_dataset_collection_hide_originals(self):
payload = self.dataset_collection_populator.create_pair_payload(
self.history_id,
type="dataset_collection"
)
payload["hide_source_items"] = True
dataset_collection_response = self._post("histories/%s/contents" % self.history_id, payload)
self.__check_create_collection_response(dataset_collection_response)
contents_response = self._get("histories/%s/contents" % self.history_id)
datasets = [d for d in contents_response.json() if d["history_content_type"] == "dataset" and d["hid"] in [1, 2]]
# Assert two datasets in source were hidden.
assert len(datasets) == 2
assert not datasets[0]["visible"]
assert not datasets[1]["visible"]
def test_update_dataset_collection(self):
payload = self.dataset_collection_populator.create_pair_payload(
self.history_id,
type="dataset_collection"
)
dataset_collection_response = self._post("histories/%s/contents" % self.history_id, payload)
self._assert_status_code_is(dataset_collection_response, 200)
hdca = dataset_collection_response.json()
update_url = self._api_url("histories/%s/contents/dataset_collections/%s" % (self.history_id, hdca["id"]), use_key=True)
# Awkward json.dumps required here because of https://trello.com/c/CQwmCeG6
body = json.dumps(dict(name="newnameforpair"))
update_response = put(update_url, data=body)
self._assert_status_code_is(update_response, 200)
show_response = self.__show(hdca)
assert str(show_response.json()["name"]) == "newnameforpair"
def test_hdca_copy(self):
hdca = self.dataset_collection_populator.create_pair_in_history(self.history_id).json()
hdca_id = hdca["id"]
second_history_id = self._new_history()
create_data = dict(
source='hdca',
content=hdca_id,
)
assert len(self._get("histories/%s/contents/dataset_collections" % second_history_id).json()) == 0
create_response = self._post("histories/%s/contents/dataset_collections" % second_history_id, create_data)
self.__check_create_collection_response(create_response)
contents = self._get("histories/%s/contents/dataset_collections" % second_history_id).json()
assert len(contents) == 1
new_forward, _ = self.__get_paired_response_elements(contents[0])
self._assert_has_keys(new_forward, "history_id")
assert new_forward["history_id"] == self.history_id
def test_hdca_copy_and_elements(self):
hdca = self.dataset_collection_populator.create_pair_in_history(self.history_id).json()
hdca_id = hdca["id"]
second_history_id = self._new_history()
create_data = dict(
source='hdca',
content=hdca_id,
copy_elements=True,
)
assert len(self._get("histories/%s/contents/dataset_collections" % second_history_id).json()) == 0
create_response = self._post("histories/%s/contents/dataset_collections" % second_history_id, create_data)
self.__check_create_collection_response(create_response)
contents = self._get("histories/%s/contents/dataset_collections" % second_history_id).json()
assert len(contents) == 1
new_forward, _ = self.__get_paired_response_elements(contents[0])
self._assert_has_keys(new_forward, "history_id")
assert new_forward["history_id"] == second_history_id
def __get_paired_response_elements(self, contents):
hdca = self.__show(contents).json()
self._assert_has_keys(hdca, "name", "deleted", "visible", "elements")
elements = hdca["elements"]
assert len(elements) == 2
element0 = elements[0]
element1 = elements[1]
self._assert_has_keys(element0, "object")
self._assert_has_keys(element1, "object")
return element0["object"], element1["object"]
def test_hdca_from_library_datasets(self):
ld = self.library_populator.new_library_dataset("el1")
ldda_id = ld["ldda_id"]
element_identifiers = [{"name": "el1", "src": "ldda", "id": ldda_id}]
create_data = dict(
history_id=self.history_id,
type="dataset_collection",
name="Test From Library",
element_identifiers=json.dumps(element_identifiers),
collection_type="list",
)
create_response = self._post("histories/%s/contents/dataset_collections" % self.history_id, create_data)
hdca = self.__check_create_collection_response(create_response)
elements = hdca["elements"]
assert len(elements) == 1
hda = elements[0]["object"]
assert hda["hda_ldda"] == "hda"
assert hda["history_content_type"] == "dataset"
assert hda["copied_from_ldda_id"] == ldda_id
def test_hdca_from_inaccessible_library_datasets(self):
library, library_dataset = self.library_populator.new_library_dataset_in_private_library("HDCACreateInaccesibleLibrary")
ldda_id = library_dataset["id"]
element_identifiers = [{"name": "el1", "src": "ldda", "id": ldda_id}]
create_data = dict(
history_id=self.history_id,
type="dataset_collection",
name="Test From Library",
element_identifiers=json.dumps(element_identifiers),
collection_type="list",
)
with self._different_user():
second_history_id = self._new_history()
create_response = self._post("histories/%s/contents/dataset_collections" % second_history_id, create_data)
self._assert_status_code_is(create_response, 403)
def __check_create_collection_response(self, response):
self._assert_status_code_is(response, 200)
dataset_collection = response.json()
self._assert_has_keys(dataset_collection, "url", "name", "deleted", "visible", "elements")
return dataset_collection
def __show(self, contents):
show_response = self._get("histories/%s/contents/%ss/%s" % (self.history_id, contents["history_content_type"], contents["id"]))
return show_response
def __count_contents(self, history_id=None, **kwds):
if history_id is None:
history_id = self.history_id
contents_response = self._get("histories/%s/contents" % history_id, kwds)
return len(contents_response.json())
def __assert_hda_has_full_details(self, hda_details):
self._assert_has_keys(hda_details, "display_types", "display_apps")
def __check_for_hda(self, contents_response, hda):
self._assert_status_code_is(contents_response, 200)
contents = contents_response.json()
assert len(contents) == 1
hda_summary = contents[0]
self.__assert_matches_hda(hda, hda_summary)
return hda_summary
def __assert_matches_hda(self, input_hda, query_hda):
self._assert_has_keys(query_hda, "id", "name")
assert input_hda["name"] == query_hda["name"]
assert input_hda["id"] == query_hda["id"]
class LibrariesApiTestCase(api.ApiTestCase, TestsDatasets):
def setUp(self):
super(LibrariesApiTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.dataset_collection_populator = DatasetCollectionPopulator(self.galaxy_interactor)
self.library_populator = LibraryPopulator(self.galaxy_interactor)
def test_create(self):
data = dict(name="CreateTestLibrary")
create_response = self._post("libraries", data=data, admin=True)
self._assert_status_code_is(create_response, 200)
library = create_response.json()
self._assert_has_keys(library, "name")
assert library["name"] == "CreateTestLibrary"
def test_delete(self):
library = self.library_populator.new_library("DeleteTestLibrary")
create_response = self._delete("libraries/%s" % library["id"], admin=True)
self._assert_status_code_is(create_response, 200)
library = create_response.json()
self._assert_has_keys(library, "deleted")
assert library["deleted"] is True
# Test undeleting
data = dict(undelete='true')
create_response = self._delete("libraries/%s" % library["id"], data=data, admin=True)
library = create_response.json()
self._assert_status_code_is(create_response, 200)
assert library["deleted"] is False
def test_nonadmin(self):
# Anons can't create libs
data = dict(name="CreateTestLibrary")
create_response = self._post("libraries", data=data, admin=False, anon=True)
self._assert_status_code_is(create_response, 403)
# Anons can't delete libs
library = self.library_populator.new_library("AnonDeleteTestLibrary")
create_response = self._delete("libraries/%s" % library["id"], admin=False, anon=True)
self._assert_status_code_is(create_response, 403)
# Anons can't update libs
data = dict(name="ChangedName", description="ChangedDescription", synopsis='ChangedSynopsis')
create_response = self._patch("libraries/%s" % library["id"], data=data, admin=False, anon=True)
self._assert_status_code_is(create_response, 403)
def test_update(self):
library = self.library_populator.new_library("UpdateTestLibrary")
data = dict(name='ChangedName', description='ChangedDescription', synopsis='ChangedSynopsis')
create_response = self._patch("libraries/%s" % library["id"], data=data, admin=True)
self._assert_status_code_is(create_response, 200)
library = create_response.json()
self._assert_has_keys(library, 'name', 'description', 'synopsis')
assert library['name'] == 'ChangedName'
assert library['description'] == 'ChangedDescription'
assert library['synopsis'] == 'ChangedSynopsis'
def test_create_private_library_permissions(self):
library = self.library_populator.new_library("PermissionTestLibrary")
library_id = library["id"]
role_id = self.library_populator.user_private_role_id()
self.library_populator.set_permissions(library_id, role_id)
create_response = self._create_folder(library)
self._assert_status_code_is(create_response, 200)
def test_create_dataset_denied(self):
library = self.library_populator.new_private_library("ForCreateDatasets")
folder_response = self._create_folder(library)
self._assert_status_code_is(folder_response, 200)
folder_id = folder_response.json()[0]['id']
history_id = self.dataset_populator.new_history()
hda_id = self.dataset_populator.new_dataset(history_id, content="1 2 3")['id']
with self._different_user():
payload = {'from_hda_id': hda_id}
create_response = self._post("folders/%s/contents" % folder_id, payload)
self._assert_status_code_is(create_response, 403)
def test_show_private_dataset_permissions(self):
library, library_dataset = self.library_populator.new_library_dataset_in_private_library("ForCreateDatasets", wait=True)
with self._different_user():
response = self.library_populator.show_ldda(library["id"], library_dataset["id"])
# TODO: this should really be 403 and a proper JSON exception.
self._assert_status_code_is(response, 400)
def test_create_dataset(self):
library, library_dataset = self.library_populator.new_library_dataset_in_private_library("ForCreateDatasets", wait=True)
self._assert_has_keys(library_dataset, "peek", "data_type")
assert library_dataset["peek"].find("create_test") >= 0
assert library_dataset["file_ext"] == "txt", library_dataset["file_ext"]
def test_create_dataset_in_folder(self):
library = self.library_populator.new_private_library("ForCreateDatasets")
folder_response = self._create_folder(library)
self._assert_status_code_is(folder_response, 200)
folder_id = folder_response.json()[0]['id']
history_id = self.dataset_populator.new_history()
hda_id = self.dataset_populator.new_dataset(history_id, content="1 2 3")['id']
payload = {'from_hda_id': hda_id}
create_response = self._post("folders/%s/contents" % folder_id, payload)
self._assert_status_code_is(create_response, 200)
self._assert_has_keys(create_response.json(), "name", "id")
def test_update_dataset_in_folder(self):
library = self.library_populator.new_private_library("ForUpdateDataset")
folder_response = self._create_folder(library)
self._assert_status_code_is(folder_response, 200)
folder_id = folder_response.json()[0]['id']
history_id = self.dataset_populator.new_history()
hda_id = self.dataset_populator.new_dataset(history_id, content="1 2 3")['id']
payload = {'from_hda_id': hda_id, 'create_type': 'file', 'folder_id': folder_id}
ld = self._post("libraries/%s/contents" % folder_id, payload)
data = {'name': 'updated_name', 'file_ext': 'fastq', 'misc_info': 'updated_info', 'genome_build': 'updated_genome_build'}
create_response = self._patch("libraries/datasets/%s" % ld.json()["id"], data=data)
self._assert_status_code_is(create_response, 200)
self._assert_has_keys(create_response.json(), "name", "file_ext", "misc_info", "genome_build")
def test_invalid_update_dataset_in_folder(self):
library = self.library_populator.new_private_library("ForInvalidUpdateDataset")
folder_response = self._create_folder(library)
self._assert_status_code_is(folder_response, 200)
folder_id = folder_response.json()[0]['id']
history_id = self.dataset_populator.new_history()
hda_id = self.dataset_populator.new_dataset(history_id, content="1 2 3")['id']
payload = {'from_hda_id': hda_id, 'create_type': 'file', 'folder_id': folder_id}
ld = self._post("libraries/%s/contents" % folder_id, payload)
data = {'file_ext': 'nonexisting_type'}
create_response = self._patch("libraries/datasets/%s" % ld.json()["id"], data=data)
self._assert_status_code_is(create_response, 400)
assert 'This Galaxy does not recognize the datatype of:' in create_response.json()['err_msg']
def test_create_datasets_in_library_from_collection(self):
library = self.library_populator.new_private_library("ForCreateDatasetsFromCollection")
folder_response = self._create_folder(library)
self._assert_status_code_is(folder_response, 200)
folder_id = folder_response.json()[0]['id']
history_id = self.dataset_populator.new_history()
hdca_id = self.dataset_collection_populator.create_list_in_history(history_id, contents=["xxx", "yyy"]).json()["id"]
payload = {'from_hdca_id': hdca_id, 'create_type': 'file', 'folder_id': folder_id}
create_response = self._post("libraries/%s/contents" % library['id'], payload)
self._assert_status_code_is(create_response, 200)
def test_create_datasets_in_folder_from_collection(self):
library = self.library_populator.new_private_library("ForCreateDatasetsFromCollection")
history_id = self.dataset_populator.new_history()
hdca_id = self.dataset_collection_populator.create_list_in_history(history_id, contents=["xxx", "yyy"]).json()["id"]
folder_response = self._create_folder(library)
self._assert_status_code_is(folder_response, 200)
folder_id = folder_response.json()[0]['id']
payload = {'from_hdca_id': hdca_id}
create_response = self._post("folders/%s/contents" % folder_id, payload)
self._assert_status_code_is(create_response, 200)
assert len(create_response.json()) == 2
# Also test that anything different from a flat dataset collection list
# is refused
hdca_pair_id = self.dataset_collection_populator.create_list_of_pairs_in_history(history_id).json()['id']
payload = {'from_hdca_id': hdca_pair_id}
create_response = self._post("folders/%s/contents" % folder_id, payload)
self._assert_status_code_is(create_response, 501)
assert create_response.json()['err_msg'] == 'Cannot add nested collections to library. Please flatten your collection first.'
def _create_folder(self, library):
create_data = dict(
folder_id=library["root_folder_id"],
create_type="folder",
name="New Folder",
)
return self._post("libraries/%s/contents" % library["id"], data=create_data)
def setUp(self):
super(MaximumWorkflowInvocationDurationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.workflow_populator = WorkflowPopulator(self.galaxy_interactor)
class DataManagerIntegrationTestCase(integration_util.IntegrationTestCase):
"""Test data manager installation and table reload through the API"""
framework_tool_and_types = True
def setUp(self):
super(DataManagerIntegrationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
@classmethod
def handle_galaxy_config_kwds(cls, config):
try:
import watchdog # noqa: F401
except ImportError:
raise SkipTest("watchdog library is not available")
cls.username = cls.get_secure_ascii_digits()
cls.conda_tmp_prefix = tempfile.mkdtemp()
cls.shed_tools_dir = tempfile.mkdtemp()
cls.shed_tool_data_dir = tempfile.mkdtemp()
cls._test_driver.temp_directories.extend([cls.conda_tmp_prefix, cls.shed_tool_data_dir, cls.shed_tools_dir])
config["conda_auto_init"] = True
config["conda_auto_install"] = True
config["conda_prefix"] = os.path.join(cls.conda_tmp_prefix, 'conda')
config["tool_sheds_config_file"] = TOOL_SHEDS_CONF
config["tool_config_file"] = os.path.join(cls.shed_tools_dir, 'shed_tool_conf.xml')
config["shed_data_manager_config_file"] = os.path.join(cls.shed_tool_data_dir, 'shed_data_manager_config_file')
config["shed_tool_data_table_config"] = os.path.join(cls.shed_tool_data_dir, 'shed_data_table_conf.xml')
config["shed_tool_data_path"] = cls.shed_tool_data_dir
config["tool_data_path"] = cls.shed_tool_data_dir
config["watch_tool_data_dir"] = True
config["admin_users"] = "*****@*****.**" % cls.username
with open(config["tool_config_file"], 'w') as tool_conf_file:
tool_conf_file.write(SHED_TOOL_CONF.substitute(shed_tools_path=cls.shed_tools_dir))
with open(config["shed_data_manager_config_file"], 'w') as shed_data_config:
shed_data_config.write(SHED_DATA_MANAGER_CONF)
with open(config["shed_tool_data_table_config"], 'w') as shed_data_table_config:
shed_data_table_config.write(SHED_DATA_TABLES)
def test_data_manager_installation_table_reload(self):
"""
Test that we can install data managers, create a new dbkey, and use that dbkey in a downstream data manager.
"""
create_response = self._post('/tool_shed_repositories/new/install_repository_revision', data=CREATE_DBKEY_PAYLOAD, admin=True)
self._assert_status_code_is(create_response, 200)
create_response = self._post('/tool_shed_repositories/new/install_repository_revision', data=SAM_FASTA_PAYLOAD, admin=True)
self._assert_status_code_is(create_response, 200)
with self._different_user(email="*****@*****.**" % self.username):
with self.dataset_populator.test_history() as history_id:
run_response = self.dataset_populator.run_tool(tool_id=FETCH_TOOL_ID,
inputs=FETCH_GENOME_DBKEYS_ALL_FASTA_INPUT,
history_id=history_id,
assert_ok=False)
self.dataset_populator.wait_for_tool_run(history_id=history_id, run_response=run_response)
run_response = self.dataset_populator.run_tool(tool_id=SAM_FASTA_ID,
inputs=SAM_FASTA_INPUT,
history_id=history_id,
assert_ok=False)
self.dataset_populator.wait_for_tool_run(history_id=history_id, run_response=run_response)
def create_local_user(self):
"""Creates a local user and returns the user id."""
password = self.get_secure_ascii_digits()
payload = {'username': self.username,
'password': password,
'email': "*****@*****.**" % self.username}
create_response = self._post('/users', data=payload, admin=True)
self._assert_status_code_is(create_response, 200)
response = create_response.json()
return response['id']
def create_api_key_for_user(self, user_id):
create_response = self._post("/users/%s/api_key" % user_id, data={}, admin=True)
self._assert_status_code_is(create_response, 200)
return create_response.json()
@classmethod
def get_secure_ascii_digits(cls, n=12):
return ''.join(random.SystemRandom().choice(string.ascii_lowercase + string.digits) for _ in range(12))
def setUp(self):
super(ObjectStoreJobsIntegrationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
def setUp(self):
super(FailJobWhenToolUnavailableTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.workflow_populator = WorkflowPopulator(self.galaxy_interactor)
self.history_id = self.dataset_populator.new_history()
class HistoriesApiTestCase(api.ApiTestCase):
def setUp(self):
super(HistoriesApiTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.dataset_collection_populator = DatasetCollectionPopulator(self.galaxy_interactor)
def test_create_history(self):
# Create a history.
create_response = self._create_history("TestHistory1")
created_id = create_response["id"]
# Make sure new history appears in index of user's histories.
index_response = self._get("histories").json()
indexed_history = [h for h in index_response if h["id"] == created_id][0]
self.assertEquals(indexed_history["name"], "TestHistory1")
def test_show_history(self):
history_id = self._create_history("TestHistoryForShow")["id"]
show_response = self._show(history_id)
self._assert_has_key(
show_response,
'id', 'name', 'annotation', 'size', 'contents_url',
'state', 'state_details', 'state_ids'
)
state_details = show_response["state_details"]
state_ids = show_response["state_ids"]
states = [
'discarded', 'empty', 'error', 'failed_metadata', 'new',
'ok', 'paused', 'queued', 'running', 'setting_metadata', 'upload'
]
assert isinstance(state_details, dict)
assert isinstance(state_ids, dict)
self._assert_has_keys(state_details, *states)
self._assert_has_keys(state_ids, *states)
def test_show_most_recently_used(self):
history_id = self._create_history("TestHistoryRecent")["id"]
show_response = self._get("histories/most_recently_used").json()
assert show_response["id"] == history_id
def test_index_order(self):
slightly_older_history_id = self._create_history("TestHistorySlightlyOlder")["id"]
newer_history_id = self._create_history("TestHistoryNewer")["id"]
index_response = self._get("histories").json()
assert index_response[0]["id"] == newer_history_id
assert index_response[1]["id"] == slightly_older_history_id
def test_delete(self):
# Setup a history and ensure it is in the index
history_id = self._create_history("TestHistoryForDelete")["id"]
index_response = self._get("histories").json()
assert index_response[0]["id"] == history_id
show_response = self._show(history_id)
assert not show_response["deleted"]
# Delete the history
self._delete("histories/%s" % history_id)
# Check can view it - but it is deleted
show_response = self._show(history_id)
assert show_response["deleted"]
# Verify it is dropped from history index
index_response = self._get("histories").json()
assert len(index_response) == 0 or index_response[0]["id"] != history_id
# Add deleted filter to index to view it
index_response = self._get("histories", {"deleted": "true"}).json()
assert index_response[0]["id"] == history_id
def test_purge(self):
history_id = self._create_history("TestHistoryForPurge")["id"]
data = {'purge': True}
self._delete("histories/%s" % history_id, data=data)
show_response = self._show(history_id)
assert show_response["deleted"]
assert show_response["purged"]
def test_undelete(self):
history_id = self._create_history("TestHistoryForDeleteAndUndelete")["id"]
self._delete("histories/%s" % history_id)
self._post("histories/deleted/%s/undelete" % history_id)
show_response = self._show(history_id)
assert not show_response["deleted"]
def test_update(self):
history_id = self._create_history("TestHistoryForUpdating")["id"]
self._update(history_id, {"name": "New Name"})
show_response = self._show(history_id)
assert show_response["name"] == "New Name"
unicode_name = u'桜ゲノム'
self._update(history_id, {"name": unicode_name})
show_response = self._show(history_id)
assert show_response["name"] == unicode_name, show_response
quoted_name = "'MooCow'"
self._update(history_id, {"name": quoted_name})
show_response = self._show(history_id)
assert show_response["name"] == quoted_name
self._update(history_id, {"deleted": True})
show_response = self._show(history_id)
assert show_response["deleted"], show_response
self._update(history_id, {"deleted": False})
show_response = self._show(history_id)
assert not show_response["deleted"]
self._update(history_id, {"published": True})
show_response = self._show(history_id)
assert show_response["published"]
self._update(history_id, {"genome_build": "hg18"})
show_response = self._show(history_id)
assert show_response["genome_build"] == "hg18"
self._update(history_id, {"annotation": "The annotation is cool"})
show_response = self._show(history_id)
assert show_response["annotation"] == "The annotation is cool"
self._update(history_id, {"annotation": unicode_name})
show_response = self._show(history_id)
assert show_response["annotation"] == unicode_name, show_response
self._update(history_id, {"annotation": quoted_name})
show_response = self._show(history_id)
assert show_response["annotation"] == quoted_name
def test_update_invalid_attribute(self):
history_id = self._create_history("TestHistoryForInvalidUpdating")["id"]
put_response = self._update(history_id, {"invalidkey": "moo"})
assert "invalidkey" not in put_response.json()
def test_update_invalid_types(self):
history_id = self._create_history("TestHistoryForUpdatingInvalidTypes")["id"]
for str_key in ["name", "annotation"]:
assert self._update(history_id, {str_key: False}).status_code == 400
for bool_key in ['deleted', 'importable', 'published']:
assert self._update(history_id, {bool_key: "a string"}).status_code == 400
assert self._update(history_id, {"tags": "a simple string"}).status_code == 400
assert self._update(history_id, {"tags": [True]}).status_code == 400
def test_invalid_keys(self):
invalid_history_id = "1234123412341234"
assert self._get("histories/%s" % invalid_history_id).status_code == 400
assert self._update(invalid_history_id, {"name": "new name"}).status_code == 400
assert self._delete("histories/%s" % invalid_history_id).status_code == 400
assert self._post("histories/deleted/%s/undelete" % invalid_history_id).status_code == 400
def test_create_anonymous_fails(self):
post_data = dict(name="CannotCreate")
# Using lower-level _api_url will cause key to not be injected.
histories_url = self._api_url("histories")
create_response = post(url=histories_url, data=post_data)
self._assert_status_code_is(create_response, 403)
def test_import_export(self):
history_name = "for_export"
history_id = self.dataset_populator.new_history(name=history_name)
self.dataset_populator.new_dataset(history_id, content="1 2 3")
imported_history_id = self._reimport_history(history_id, history_name)
contents_response = self._get("histories/%s/contents" % imported_history_id)
self._assert_status_code_is(contents_response, 200)
contents = contents_response.json()
assert len(contents) == 1
imported_content = self.dataset_populator.get_history_dataset_content(
history_id=imported_history_id,
dataset_id=contents[0]["id"]
)
assert imported_content == "1 2 3\n"
def test_import_export_collection(self):
from nose.plugins.skip import SkipTest
raise SkipTest("Collection import/export not yet implemented")
history_name = "for_export_with_collections"
history_id = self.dataset_populator.new_history(name=history_name)
self.dataset_collection_populator.create_list_in_history(history_id, contents=["Hello", "World"])
imported_history_id = self._reimport_history(history_id, history_name)
contents_response = self._get("histories/%s/contents" % imported_history_id)
self._assert_status_code_is(contents_response, 200)
contents = contents_response.json()
assert len(contents) == 3
def _reimport_history(self, history_id, history_name):
# Ensure the history is ready to go...
self.dataset_populator.wait_for_history(history_id, assert_ok=True)
# Export the history.
download_path = self._export(history_id)
# Create download for history
full_download_url = "%s%s?key=%s" % (self.url, download_path, self.galaxy_interactor.api_key)
download_response = get(full_download_url)
self._assert_status_code_is(download_response, 200)
def history_names():
history_index = self._get("histories")
return dict((h["name"], h) for h in history_index.json())
import_name = "imported from archive: %s" % history_name
assert import_name not in history_names()
import_data = dict(archive_source=full_download_url, archive_type="url")
import_response = self._post("histories", data=import_data)
self._assert_status_code_is(import_response, 200)
def has_history_with_name():
histories = history_names()
return histories.get(import_name, None)
imported_history = wait_on(has_history_with_name, desc="import history")
imported_history_id = imported_history["id"]
self.dataset_populator.wait_for_history(imported_history_id)
return imported_history_id
def test_create_tag(self):
post_data = dict(name="TestHistoryForTag")
history_id = self._post("histories", data=post_data).json()["id"]
tag_data = dict(value="awesometagvalue")
tag_url = "histories/%s/tags/awesometagname" % history_id
tag_create_response = self._post(tag_url, data=tag_data)
self._assert_status_code_is(tag_create_response, 200)
def _export(self, history_id):
export_url = self._api_url("histories/%s/exports" % history_id, use_key=True)
put_response = put(export_url)
self._assert_status_code_is(put_response, 202)
def export_ready_response():
put_response = put(export_url)
if put_response.status_code == 202:
return None
return put_response
put_response = wait_on(export_ready_response, desc="export ready")
self._assert_status_code_is(put_response, 200)
response = put_response.json()
self._assert_has_keys(response, "download_url")
download_path = response["download_url"]
return download_path
def _show(self, history_id):
return self._get("histories/%s" % history_id).json()
def _update(self, history_id, data):
update_url = self._api_url("histories/%s" % history_id, use_key=True)
put_response = put(update_url, json=data)
return put_response
def _create_history(self, name):
post_data = dict(name=name)
create_response = self._post("histories", data=post_data).json()
self._assert_has_keys(create_response, "name", "id")
self.assertEquals(create_response["name"], name)
return create_response
def setUp(self):
super(DatasetCollectionApiTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.dataset_collection_populator = DatasetCollectionPopulator(self.galaxy_interactor)
self.history_id = self.dataset_populator.new_history()
def setUp(self):
super(MaximumWorkflowJobsPerSchedulingIterationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.workflow_populator = WorkflowPopulator(self.galaxy_interactor)
self.dataset_collection_populator = DatasetCollectionPopulator(self.galaxy_interactor)
class HistoriesApiTestCase(api.ApiTestCase):
def setUp(self):
super(HistoriesApiTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.dataset_collection_populator = DatasetCollectionPopulator(
self.galaxy_interactor)
def test_create_history(self):
# Create a history.
create_response = self._create_history("TestHistory1")
created_id = create_response["id"]
# Make sure new history appears in index of user's histories.
index_response = self._get("histories").json()
indexed_history = [h for h in index_response
if h["id"] == created_id][0]
self.assertEquals(indexed_history["name"], "TestHistory1")
def test_show_history(self):
history_id = self._create_history("TestHistoryForShow")["id"]
show_response = self._show(history_id)
self._assert_has_key(show_response, 'id', 'name', 'annotation', 'size',
'contents_url', 'state', 'state_details',
'state_ids')
state_details = show_response["state_details"]
state_ids = show_response["state_ids"]
states = [
'discarded', 'empty', 'error', 'failed_metadata', 'new', 'ok',
'paused', 'queued', 'running', 'setting_metadata', 'upload'
]
assert isinstance(state_details, dict)
assert isinstance(state_ids, dict)
self._assert_has_keys(state_details, *states)
self._assert_has_keys(state_ids, *states)
def test_show_most_recently_used(self):
history_id = self._create_history("TestHistoryRecent")["id"]
show_response = self._get("histories/most_recently_used").json()
assert show_response["id"] == history_id
def test_index_order(self):
slightly_older_history_id = self._create_history(
"TestHistorySlightlyOlder")["id"]
newer_history_id = self._create_history("TestHistoryNewer")["id"]
index_response = self._get("histories").json()
assert index_response[0]["id"] == newer_history_id
assert index_response[1]["id"] == slightly_older_history_id
def test_delete(self):
# Setup a history and ensure it is in the index
history_id = self._create_history("TestHistoryForDelete")["id"]
index_response = self._get("histories").json()
assert index_response[0]["id"] == history_id
show_response = self._show(history_id)
assert not show_response["deleted"]
# Delete the history
self._delete("histories/%s" % history_id)
# Check can view it - but it is deleted
show_response = self._show(history_id)
assert show_response["deleted"]
# Verify it is dropped from history index
index_response = self._get("histories").json()
assert len(
index_response) == 0 or index_response[0]["id"] != history_id
# Add deleted filter to index to view it
index_response = self._get("histories", {"deleted": "true"}).json()
assert index_response[0]["id"] == history_id
def test_purge(self):
history_id = self._create_history("TestHistoryForPurge")["id"]
data = {'purge': True}
self._delete("histories/%s" % history_id, data=data)
show_response = self._show(history_id)
assert show_response["deleted"]
assert show_response["purged"]
def test_undelete(self):
history_id = self._create_history(
"TestHistoryForDeleteAndUndelete")["id"]
self._delete("histories/%s" % history_id)
self._post("histories/deleted/%s/undelete" % history_id)
show_response = self._show(history_id)
assert not show_response["deleted"]
def test_update(self):
history_id = self._create_history("TestHistoryForUpdating")["id"]
self._update(history_id, {"name": "New Name"})
show_response = self._show(history_id)
assert show_response["name"] == "New Name"
unicode_name = u'桜ゲノム'
self._update(history_id, {"name": unicode_name})
show_response = self._show(history_id)
assert show_response["name"] == unicode_name, show_response
quoted_name = "'MooCow'"
self._update(history_id, {"name": quoted_name})
show_response = self._show(history_id)
assert show_response["name"] == quoted_name
self._update(history_id, {"deleted": True})
show_response = self._show(history_id)
assert show_response["deleted"], show_response
self._update(history_id, {"deleted": False})
show_response = self._show(history_id)
assert not show_response["deleted"]
self._update(history_id, {"published": True})
show_response = self._show(history_id)
assert show_response["published"]
self._update(history_id, {"genome_build": "hg18"})
show_response = self._show(history_id)
assert show_response["genome_build"] == "hg18"
self._update(history_id, {"annotation": "The annotation is cool"})
show_response = self._show(history_id)
assert show_response["annotation"] == "The annotation is cool"
self._update(history_id, {"annotation": unicode_name})
show_response = self._show(history_id)
assert show_response["annotation"] == unicode_name, show_response
self._update(history_id, {"annotation": quoted_name})
show_response = self._show(history_id)
assert show_response["annotation"] == quoted_name
def test_update_invalid_attribute(self):
history_id = self._create_history(
"TestHistoryForInvalidUpdating")["id"]
put_response = self._update(history_id, {"invalidkey": "moo"})
assert "invalidkey" not in put_response.json()
def test_update_invalid_types(self):
history_id = self._create_history(
"TestHistoryForUpdatingInvalidTypes")["id"]
for str_key in ["name", "annotation"]:
assert self._update(history_id, {
str_key: False
}).status_code == 400
for bool_key in ['deleted', 'importable', 'published']:
assert self._update(history_id, {
bool_key: "a string"
}).status_code == 400
assert self._update(history_id, {
"tags": "a simple string"
}).status_code == 400
assert self._update(history_id, {"tags": [True]}).status_code == 400
def test_invalid_keys(self):
invalid_history_id = "1234123412341234"
assert self._get("histories/%s" %
invalid_history_id).status_code == 400
assert self._update(invalid_history_id, {
"name": "new name"
}).status_code == 400
assert self._delete("histories/%s" %
invalid_history_id).status_code == 400
assert self._post("histories/deleted/%s/undelete" %
invalid_history_id).status_code == 400
def test_create_anonymous_fails(self):
post_data = dict(name="CannotCreate")
# Using lower-level _api_url will cause key to not be injected.
histories_url = self._api_url("histories")
create_response = post(url=histories_url, data=post_data)
self._assert_status_code_is(create_response, 403)
def test_import_export(self):
history_name = "for_export"
history_id = self.dataset_populator.new_history(name=history_name)
self.dataset_populator.new_dataset(history_id, content="1 2 3")
imported_history_id = self._reimport_history(history_id, history_name)
contents_response = self._get("histories/%s/contents" %
imported_history_id)
self._assert_status_code_is(contents_response, 200)
contents = contents_response.json()
assert len(contents) == 1
imported_content = self.dataset_populator.get_history_dataset_content(
history_id=imported_history_id, dataset_id=contents[0]["id"])
assert imported_content == "1 2 3\n"
def test_import_export_collection(self):
from nose.plugins.skip import SkipTest
raise SkipTest("Collection import/export not yet implemented")
history_name = "for_export_with_collections"
history_id = self.dataset_populator.new_history(name=history_name)
self.dataset_collection_populator.create_list_in_history(
history_id, contents=["Hello", "World"])
imported_history_id = self._reimport_history(history_id, history_name)
contents_response = self._get("histories/%s/contents" %
imported_history_id)
self._assert_status_code_is(contents_response, 200)
contents = contents_response.json()
assert len(contents) == 3
def _reimport_history(self, history_id, history_name):
# Ensure the history is ready to go...
self.dataset_populator.wait_for_history(history_id, assert_ok=True)
# Export the history.
download_path = self._export(history_id)
# Create download for history
full_download_url = "%s%s?key=%s" % (self.url, download_path,
self.galaxy_interactor.api_key)
download_response = get(full_download_url)
self._assert_status_code_is(download_response, 200)
def history_names():
history_index = self._get("histories")
return dict((h["name"], h) for h in history_index.json())
import_name = "imported from archive: %s" % history_name
assert import_name not in history_names()
import_data = dict(archive_source=full_download_url,
archive_type="url")
import_response = self._post("histories", data=import_data)
self._assert_status_code_is(import_response, 200)
def has_history_with_name():
histories = history_names()
return histories.get(import_name, None)
imported_history = wait_on(has_history_with_name,
desc="import history")
imported_history_id = imported_history["id"]
self.dataset_populator.wait_for_history(imported_history_id)
return imported_history_id
def test_create_tag(self):
post_data = dict(name="TestHistoryForTag")
history_id = self._post("histories", data=post_data).json()["id"]
tag_data = dict(value="awesometagvalue")
tag_url = "histories/%s/tags/awesometagname" % history_id
tag_create_response = self._post(tag_url, data=tag_data)
self._assert_status_code_is(tag_create_response, 200)
def _export(self, history_id):
export_url = self._api_url("histories/%s/exports" % history_id,
use_key=True)
put_response = put(export_url)
self._assert_status_code_is(put_response, 202)
def export_ready_response():
put_response = put(export_url)
if put_response.status_code == 202:
return None
return put_response
put_response = wait_on(export_ready_response, desc="export ready")
self._assert_status_code_is(put_response, 200)
response = put_response.json()
self._assert_has_keys(response, "download_url")
download_path = response["download_url"]
return download_path
def _show(self, history_id):
return self._get("histories/%s" % history_id).json()
def _update(self, history_id, data):
update_url = self._api_url("histories/%s" % history_id, use_key=True)
put_response = put(update_url, json=data)
return put_response
def _create_history(self, name):
post_data = dict(name=name)
create_response = self._post("histories", data=post_data).json()
self._assert_has_keys(create_response, "name", "id")
self.assertEquals(create_response["name"], name)
return create_response
def setUp(self):
super(BaseJobEnvironmentIntegrationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
class RolesApiTestCase(api.ApiTestCase):
def setUp(self):
super(RolesApiTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
def test_list_and_show(self):
def check_roles_response(response):
assert response.status_code == 200
as_list = response.json()
assert isinstance(as_list, list)
assert len(as_list) > 0
for role in as_list:
RolesApiTestCase.check_role_dict(role)
user_role_id = self.dataset_populator.user_private_role_id()
with self._different_user():
different_user_role_id = self.dataset_populator.user_private_role_id()
admin_roles_response = self._get("roles", admin=True)
user_roles_response = self._get("roles")
check_roles_response(admin_roles_response)
check_roles_response(user_roles_response)
admin_roles_response_ids = [r["id"] for r in admin_roles_response.json()]
user_roles_response_ids = [r["id"] for r in user_roles_response.json()]
# User can see their own private role not the other users, admin can see both.
assert user_role_id in user_roles_response_ids
assert different_user_role_id not in user_roles_response_ids
assert user_role_id in admin_roles_response_ids
assert different_user_role_id in admin_roles_response_ids
# Check showing a valid, role.
role_response = self._get("roles/%s" % user_role_id)
assert role_response.status_code == 200
role = role_response.json()
RolesApiTestCase.check_role_dict(role, assert_id=user_role_id)
def test_create_valid(self):
name = self.dataset_populator.get_random_name()
description = "A test role."
payload = {
"name": name,
"description": description,
"user_ids": json.dumps([self.dataset_populator.user_id()]),
}
response = self._post("roles", payload, admin=True)
assert_status_code_is(response, 200)
# TODO: Why does this return a singleton list - that is bad - should be deprecated
# and return a single role.
role = response.json()[0]
RolesApiTestCase.check_role_dict(role)
assert role["name"] == name
assert role["description"] == description
user_roles_response = self._get("roles")
with self._different_user():
different_user_roles_response = self._get("roles")
user_roles_response_ids = [r["id"] for r in user_roles_response.json()]
different_user_roles_response_ids = [r["id"] for r in different_user_roles_response.json()]
# This new role is public, all users see it.
assert role["id"] in user_roles_response_ids
assert role["id"] in different_user_roles_response_ids
def test_show_error_codes(self):
# Bad role ids are 400.
response = self._get("roles/badroleid")
assert_status_code_is(response, 400)
# Trying to access roles are errors - should probably be 403 not 400 though?
with self._different_user():
different_user_role_id = self.dataset_populator.user_private_role_id()
response = self._get("roles/%s" % different_user_role_id)
assert_status_code_is(response, 400)
def test_create_only_admin(self):
response = self._post("roles")
assert_status_code_is(response, 403)
@staticmethod
def check_role_dict(role_dict, assert_id=None):
assert_has_keys(role_dict, "id", "name", "model_class", "url")
assert role_dict["model_class"] == "Role"
if assert_id is not None:
assert role_dict["id"] == assert_id
class LibrariesApiTestCase(api.ApiTestCase, TestsDatasets):
def setUp(self):
super(LibrariesApiTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.dataset_collection_populator = DatasetCollectionPopulator(self.galaxy_interactor)
self.library_populator = LibraryPopulator(self.galaxy_interactor)
def test_create(self):
data = dict(name="CreateTestLibrary")
create_response = self._post("libraries", data=data, admin=True)
self._assert_status_code_is(create_response, 200)
library = create_response.json()
self._assert_has_keys(library, "name")
assert library["name"] == "CreateTestLibrary"
def test_delete(self):
library = self.library_populator.new_library("DeleteTestLibrary")
create_response = self._delete("libraries/%s" % library["id"], admin=True)
self._assert_status_code_is(create_response, 200)
library = create_response.json()
self._assert_has_keys(library, "deleted")
assert library["deleted"] is True
# Test undeleting
data = dict(undelete='true')
create_response = self._delete("libraries/%s" % library["id"], data=data, admin=True)
library = create_response.json()
self._assert_status_code_is(create_response, 200)
assert library["deleted"] is False
def test_nonadmin(self):
# Anons can't create libs
data = dict(name="CreateTestLibrary")
create_response = self._post("libraries", data=data, admin=False, anon=True)
self._assert_status_code_is(create_response, 403)
# Anons can't delete libs
library = self.library_populator.new_library("AnonDeleteTestLibrary")
create_response = self._delete("libraries/%s" % library["id"], admin=False, anon=True)
self._assert_status_code_is(create_response, 403)
# Anons can't update libs
data = dict(name="ChangedName", description="ChangedDescription", synopsis='ChangedSynopsis')
create_response = self._patch("libraries/%s" % library["id"], data=data, admin=False, anon=True)
self._assert_status_code_is(create_response, 403)
def test_update(self):
library = self.library_populator.new_library("UpdateTestLibrary")
data = dict(name='ChangedName', description='ChangedDescription', synopsis='ChangedSynopsis')
create_response = self._patch("libraries/%s" % library["id"], data=data, admin=True)
self._assert_status_code_is(create_response, 200)
library = create_response.json()
self._assert_has_keys(library, 'name', 'description', 'synopsis')
assert library['name'] == 'ChangedName'
assert library['description'] == 'ChangedDescription'
assert library['synopsis'] == 'ChangedSynopsis'
def test_create_private_library_permissions(self):
library = self.library_populator.new_library("PermissionTestLibrary")
library_id = library["id"]
role_id = self.library_populator.user_private_role_id()
self.library_populator.set_permissions(library_id, role_id)
create_response = self._create_folder(library)
self._assert_status_code_is(create_response, 200)
def test_create_dataset_denied(self):
library = self.library_populator.new_private_library("ForCreateDatasets")
folder_response = self._create_folder(library)
self._assert_status_code_is(folder_response, 200)
folder_id = folder_response.json()[0]['id']
history_id = self.dataset_populator.new_history()
hda_id = self.dataset_populator.new_dataset(history_id, content="1 2 3")['id']
with self._different_user():
payload = {'from_hda_id': hda_id}
create_response = self._post("folders/%s/contents" % folder_id, payload)
self._assert_status_code_is(create_response, 403)
def test_show_private_dataset_permissions(self):
library, library_dataset = self.library_populator.new_library_dataset_in_private_library("ForCreateDatasets", wait=True)
with self._different_user():
response = self.library_populator.show_ldda(library["id"], library_dataset["id"])
# TODO: this should really be 403 and a proper JSON exception.
self._assert_status_code_is(response, 400)
def test_create_dataset(self):
library, library_dataset = self.library_populator.new_library_dataset_in_private_library("ForCreateDatasets", wait=True)
self._assert_has_keys(library_dataset, "peek", "data_type")
assert library_dataset["peek"].find("create_test") >= 0
assert library_dataset["file_ext"] == "txt", library_dataset["file_ext"]
def test_fetch_upload_to_folder(self):
history_id, library, destination = self._setup_fetch_to_folder("flat_zip")
items = [{"src": "files", "dbkey": "hg19", "info": "my cool bed"}]
targets = [{
"destination": destination,
"items": items
}]
payload = {
"history_id": history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
"__files": {"files_0|file_data": open(self.test_data_resolver.get_filename("4.bed"))},
}
self.dataset_populator.fetch(payload)
dataset = self.library_populator.get_library_contents_with_path(library["id"], "/4.bed")
assert dataset["file_size"] == 61, dataset
assert dataset["genome_build"] == "hg19", dataset
assert dataset["misc_info"] == "my cool bed", dataset
assert dataset["file_ext"] == "bed", dataset
def test_fetch_zip_to_folder(self):
history_id, library, destination = self._setup_fetch_to_folder("flat_zip")
bed_test_data_path = self.test_data_resolver.get_filename("4.bed.zip")
targets = [{
"destination": destination,
"items_from": "archive", "src": "files",
}]
payload = {
"history_id": history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
"__files": {"files_0|file_data": open(bed_test_data_path)}
}
self.dataset_populator.fetch(payload)
dataset = self.library_populator.get_library_contents_with_path(library["id"], "/4.bed")
assert dataset["file_size"] == 61, dataset
def test_fetch_single_url_to_folder(self):
history_id, library, destination = self._setup_fetch_to_folder("single_url")
items = [{"src": "url", "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed"}]
targets = [{
"destination": destination,
"items": items
}]
payload = {
"history_id": history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
}
self.dataset_populator.fetch(payload)
dataset = self.library_populator.get_library_contents_with_path(library["id"], "/4.bed")
assert dataset["file_size"] == 61, dataset
def test_fetch_url_archive_to_folder(self):
history_id, library, destination = self._setup_fetch_to_folder("single_url")
targets = [{
"destination": destination,
"items_from": "archive",
"src": "url",
"url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed.zip",
}]
payload = {
"history_id": history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
}
self.dataset_populator.fetch(payload)
dataset = self.library_populator.get_library_contents_with_path(library["id"], "/4.bed")
assert dataset["file_size"] == 61, dataset
@unittest.skip # reference URLs changed, checksums now invalid.
def test_fetch_bagit_archive_to_folder(self):
history_id, library, destination = self._setup_fetch_to_folder("bagit_archive")
example_bag_path = self.test_data_resolver.get_filename("example-bag.zip")
targets = [{
"destination": destination,
"items_from": "bagit_archive", "src": "files",
}]
payload = {
"history_id": history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
"__files": {"files_0|file_data": open(example_bag_path)},
}
self.dataset_populator.fetch(payload)
dataset = self.library_populator.get_library_contents_with_path(library["id"], "/README.txt")
assert dataset["file_size"] == 66, dataset
dataset = self.library_populator.get_library_contents_with_path(library["id"], "/bdbag-profile.json")
assert dataset["file_size"] == 723, dataset
def _setup_fetch_to_folder(self, test_name):
return self.library_populator.setup_fetch_to_folder(test_name)
def test_create_dataset_in_folder(self):
library = self.library_populator.new_private_library("ForCreateDatasets")
folder_response = self._create_folder(library)
self._assert_status_code_is(folder_response, 200)
folder_id = folder_response.json()[0]['id']
history_id = self.dataset_populator.new_history()
hda_id = self.dataset_populator.new_dataset(history_id, content="1 2 3")['id']
payload = {'from_hda_id': hda_id}
create_response = self._post("folders/%s/contents" % folder_id, payload)
self._assert_status_code_is(create_response, 200)
self._assert_has_keys(create_response.json(), "name", "id")
def test_update_dataset_in_folder(self):
library = self.library_populator.new_private_library("ForUpdateDataset")
folder_response = self._create_folder(library)
self._assert_status_code_is(folder_response, 200)
folder_id = folder_response.json()[0]['id']
history_id = self.dataset_populator.new_history()
hda_id = self.dataset_populator.new_dataset(history_id, content="1 2 3")['id']
payload = {'from_hda_id': hda_id, 'create_type': 'file', 'folder_id': folder_id}
ld = self._post("libraries/%s/contents" % folder_id, payload)
data = {'name': 'updated_name', 'file_ext': 'fastq', 'misc_info': 'updated_info', 'genome_build': 'updated_genome_build'}
create_response = self._patch("libraries/datasets/%s" % ld.json()["id"], data=data)
self._assert_status_code_is(create_response, 200)
self._assert_has_keys(create_response.json(), "name", "file_ext", "misc_info", "genome_build")
def test_invalid_update_dataset_in_folder(self):
library = self.library_populator.new_private_library("ForInvalidUpdateDataset")
folder_response = self._create_folder(library)
self._assert_status_code_is(folder_response, 200)
folder_id = folder_response.json()[0]['id']
history_id = self.dataset_populator.new_history()
hda_id = self.dataset_populator.new_dataset(history_id, content="1 2 3")['id']
payload = {'from_hda_id': hda_id, 'create_type': 'file', 'folder_id': folder_id}
ld = self._post("libraries/%s/contents" % folder_id, payload)
data = {'file_ext': 'nonexisting_type'}
create_response = self._patch("libraries/datasets/%s" % ld.json()["id"], data=data)
self._assert_status_code_is(create_response, 400)
assert 'This Galaxy does not recognize the datatype of:' in create_response.json()['err_msg']
def test_create_datasets_in_library_from_collection(self):
library = self.library_populator.new_private_library("ForCreateDatasetsFromCollection")
folder_response = self._create_folder(library)
self._assert_status_code_is(folder_response, 200)
folder_id = folder_response.json()[0]['id']
history_id = self.dataset_populator.new_history()
hdca_id = self.dataset_collection_populator.create_list_in_history(history_id, contents=["xxx", "yyy"]).json()["id"]
payload = {'from_hdca_id': hdca_id, 'create_type': 'file', 'folder_id': folder_id}
create_response = self._post("libraries/%s/contents" % library['id'], payload)
self._assert_status_code_is(create_response, 200)
def test_create_datasets_in_folder_from_collection(self):
library = self.library_populator.new_private_library("ForCreateDatasetsFromCollection")
history_id = self.dataset_populator.new_history()
hdca_id = self.dataset_collection_populator.create_list_in_history(history_id, contents=["xxx", "yyy"]).json()["id"]
folder_response = self._create_folder(library)
self._assert_status_code_is(folder_response, 200)
folder_id = folder_response.json()[0]['id']
payload = {'from_hdca_id': hdca_id}
create_response = self._post("folders/%s/contents" % folder_id, payload)
self._assert_status_code_is(create_response, 200)
assert len(create_response.json()) == 2
# Also test that anything different from a flat dataset collection list
# is refused
hdca_pair_id = self.dataset_collection_populator.create_list_of_pairs_in_history(history_id).json()['id']
payload = {'from_hdca_id': hdca_pair_id}
create_response = self._post("folders/%s/contents" % folder_id, payload)
self._assert_status_code_is(create_response, 501)
assert create_response.json()['err_msg'] == 'Cannot add nested collections to library. Please flatten your collection first.'
def _create_folder(self, library):
create_data = dict(
folder_id=library["root_folder_id"],
create_type="folder",
name="New Folder",
)
return self._post("libraries/%s/contents" % library["id"], data=create_data)
class ObjectStoreJobsIntegrationTestCase(integration_util.IntegrationTestCase):
framework_tool_and_types = True
@classmethod
def handle_galaxy_config_kwds(cls, config):
temp_directory = cls._test_driver.mkdtemp()
cls.object_stores_parent = temp_directory
for disk_store_file_name in [
"files_default", "files_static", "files_dynamic_ebs",
"files_dynamic_s3"
]:
disk_store_path = os.path.join(temp_directory,
disk_store_file_name)
os.makedirs(disk_store_path)
setattr(cls, "%s_path" % disk_store_file_name, disk_store_path)
config_path = os.path.join(temp_directory, "object_store_conf.xml")
with open(config_path, "w") as f:
f.write(
DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE.safe_substitute(
{"temp_directory": temp_directory}))
config["object_store_config_file"] = config_path
config["job_config_file"] = JOB_CONFIG_FILE
config[
"job_resource_params_file"] = JOB_RESOURCE_PARAMETERS_CONFIG_FILE
def setUp(self):
super(ObjectStoreJobsIntegrationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
def _object_store_counts(self):
files_default_count = _files_count(self.files_default_path)
files_static_count = _files_count(self.files_static_path)
files_dynamic_count = _files_count(self.files_dynamic_path)
return files_default_count, files_static_count, files_dynamic_count
def _assert_file_counts(self, default, static, dynamic_ebs, dynamic_s3):
files_default_count = _files_count(self.files_default_path)
files_static_count = _files_count(self.files_static_path)
files_dynamic_ebs_count = _files_count(self.files_dynamic_ebs_path)
files_dynamic_s3_count = _files_count(self.files_dynamic_s3_path)
assert default == files_default_count
assert static == files_static_count
assert dynamic_ebs == files_dynamic_ebs_count
assert dynamic_s3 == files_dynamic_s3_count
def test_tool_simple_constructs(self):
with self.dataset_populator.test_history() as history_id:
def _run_tool(tool_id, inputs):
self.dataset_populator.run_tool(
tool_id,
inputs,
history_id,
assert_ok=True,
)
self.dataset_populator.wait_for_history(history_id)
self._assert_file_counts(0, 0, 0, 0)
hda1 = self.dataset_populator.new_dataset(history_id,
content="1 2 3")
self.dataset_populator.wait_for_history(history_id)
hda1_input = {"src": "hda", "id": hda1["id"]}
# One file uploaded, added to default object store ID.
self._assert_file_counts(1, 0, 0, 0)
# should create two files in static object store.
_run_tool("multi_data_param", {"f1": hda1_input, "f2": hda1_input})
self._assert_file_counts(1, 2, 0, 0)
# should create two files in ebs object store.
create_10_inputs = {
"input1": hda1_input,
"input2": hda1_input,
}
_run_tool("create_10", create_10_inputs)
self._assert_file_counts(1, 2, 10, 0)
# should create 10 files in S3 object store.
create_10_inputs = {
"__job_resource|__job_resource__select": "yes",
"__job_resource|how_store": "slow",
"input1": hda1_input,
"input2": hda1_input,
}
_run_tool("create_10", create_10_inputs)
self._assert_file_counts(1, 2, 10, 10)
def setUp(self):
super(DataManagerIntegrationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
def setUp(self):
super(WorkflowSyncTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.workflow_populator = WorkflowPopulator(self.galaxy_interactor)
class ToolsUploadTestCase(api.ApiTestCase):
def setUp(self):
super(ToolsUploadTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
def test_upload1_paste(self):
with self.dataset_populator.test_history() as history_id:
payload = self.dataset_populator.upload_payload(
history_id, 'Hello World')
create_response = self._post("tools", data=payload)
self._assert_has_keys(create_response.json(), 'outputs')
def test_upload1_paste_bad_datatype(self):
# Check that you get a nice message if you upload an incorrect datatype
with self.dataset_populator.test_history() as history_id:
file_type = "johnsawesomebutfakedatatype"
payload = self.dataset_populator.upload_payload(
history_id, 'Hello World', file_type=file_type)
create = self._post("tools", data=payload).json()
self._assert_has_keys(create, 'err_msg')
assert file_type in create['err_msg']
def test_upload_posix_newline_fixes(self):
windows_content = ONE_TO_SIX_ON_WINDOWS
result_content = self._upload_and_get_content(windows_content)
self.assertEquals(result_content, ONE_TO_SIX_WITH_TABS)
def test_upload_disable_posix_fix(self):
windows_content = ONE_TO_SIX_ON_WINDOWS
result_content = self._upload_and_get_content(windows_content,
to_posix_lines=None)
self.assertEquals(result_content, windows_content)
def test_upload_tab_to_space(self):
table = ONE_TO_SIX_WITH_SPACES
result_content = self._upload_and_get_content(table,
space_to_tab="Yes")
self.assertEquals(result_content, ONE_TO_SIX_WITH_TABS)
def test_upload_tab_to_space_off_by_default(self):
table = ONE_TO_SIX_WITH_SPACES
result_content = self._upload_and_get_content(table)
self.assertEquals(result_content, table)
@skip_without_datatype("rdata")
def test_rdata_not_decompressed(self):
# Prevent regression of https://github.com/galaxyproject/galaxy/issues/753
rdata_path = TestDataResolver().get_filename("1.RData")
rdata_metadata = self._upload_and_get_details(open(rdata_path, "rb"),
file_type="auto")
self.assertEquals(rdata_metadata["file_ext"], "rdata")
@skip_without_datatype("velvet")
def test_composite_datatype(self):
with self.dataset_populator.test_history() as history_id:
dataset = self._velvet_upload(history_id,
extra_inputs={
"files_1|url_paste":
"roadmaps content",
"files_1|type": "upload_dataset",
"files_2|url_paste":
"log content",
"files_2|type": "upload_dataset",
})
roadmaps_content = self._get_roadmaps_content(history_id, dataset)
assert roadmaps_content.strip(
) == "roadmaps content", roadmaps_content
@skip_without_datatype("velvet")
def test_composite_datatype_space_to_tab(self):
# Like previous test but set one upload with space_to_tab to True to
# verify that works.
with self.dataset_populator.test_history() as history_id:
dataset = self._velvet_upload(history_id,
extra_inputs={
"files_1|url_paste":
"roadmaps content",
"files_1|type": "upload_dataset",
"files_1|space_to_tab": "Yes",
"files_2|url_paste":
"log content",
"files_2|type": "upload_dataset",
})
roadmaps_content = self._get_roadmaps_content(history_id, dataset)
assert roadmaps_content.strip(
) == "roadmaps\tcontent", roadmaps_content
@skip_without_datatype("velvet")
def test_composite_datatype_posix_lines(self):
# Like previous test but set one upload with space_to_tab to True to
# verify that works.
with self.dataset_populator.test_history() as history_id:
dataset = self._velvet_upload(history_id,
extra_inputs={
"files_1|url_paste":
"roadmaps\rcontent",
"files_1|type": "upload_dataset",
"files_1|space_to_tab": "Yes",
"files_2|url_paste":
"log\rcontent",
"files_2|type": "upload_dataset",
})
roadmaps_content = self._get_roadmaps_content(history_id, dataset)
assert roadmaps_content.strip(
) == "roadmaps\ncontent", roadmaps_content
def test_upload_dbkey(self):
with self.dataset_populator.test_history() as history_id:
payload = self.dataset_populator.upload_payload(history_id,
"Test123",
dbkey="hg19")
run_response = self.dataset_populator.tools_post(payload)
self.dataset_populator.wait_for_tool_run(history_id, run_response)
datasets = run_response.json()["outputs"]
assert datasets[0].get("genome_build") == "hg19", datasets[0]
def test_upload_multiple_files_1(self):
with self.dataset_populator.test_history() as history_id:
payload = self.dataset_populator.upload_payload(
history_id,
"Test123",
dbkey="hg19",
extra_inputs={
"files_1|url_paste": "SecondOutputContent",
"files_1|NAME": "SecondOutputName",
"files_1|file_type": "tabular",
"files_1|dbkey": "hg18",
"file_count": "2",
})
run_response = self.dataset_populator.tools_post(payload)
self.dataset_populator.wait_for_tool_run(history_id, run_response)
datasets = run_response.json()["outputs"]
assert len(datasets) == 2, datasets
content = self.dataset_populator.get_history_dataset_content(
history_id, dataset=datasets[0])
assert content.strip() == "Test123"
assert datasets[0]["file_ext"] == "txt"
assert datasets[0]["genome_build"] == "hg19", datasets
content = self.dataset_populator.get_history_dataset_content(
history_id, dataset=datasets[1])
assert content.strip() == "SecondOutputContent"
assert datasets[1]["file_ext"] == "tabular"
assert datasets[1]["genome_build"] == "hg18", datasets
def test_upload_multiple_files_2(self):
with self.dataset_populator.test_history() as history_id:
payload = self.dataset_populator.upload_payload(
history_id,
"Test123",
file_type="tabular",
dbkey="hg19",
extra_inputs={
"files_1|url_paste": "SecondOutputContent",
"files_1|NAME": "SecondOutputName",
"files_1|file_type": "txt",
"files_1|dbkey": "hg18",
"file_count": "2",
})
run_response = self.dataset_populator.tools_post(payload)
self.dataset_populator.wait_for_tool_run(history_id, run_response)
datasets = run_response.json()["outputs"]
assert len(datasets) == 2, datasets
content = self.dataset_populator.get_history_dataset_content(
history_id, dataset=datasets[0])
assert content.strip() == "Test123"
assert datasets[0]["file_ext"] == "tabular", datasets
assert datasets[0]["genome_build"] == "hg19", datasets
content = self.dataset_populator.get_history_dataset_content(
history_id, dataset=datasets[1])
assert content.strip() == "SecondOutputContent"
assert datasets[1]["file_ext"] == "txt"
assert datasets[1]["genome_build"] == "hg18", datasets
def test_upload_multiple_files_3(self):
with self.dataset_populator.test_history() as history_id:
payload = self.dataset_populator.upload_payload(
history_id,
"Test123",
file_type="tabular",
dbkey="hg19",
extra_inputs={
"files_0|file_type": "txt",
"files_0|dbkey": "hg18",
"files_1|url_paste": "SecondOutputContent",
"files_1|NAME": "SecondOutputName",
"files_1|file_type": "txt",
"files_1|dbkey": "hg18",
"file_count": "2",
})
run_response = self.dataset_populator.tools_post(payload)
self.dataset_populator.wait_for_tool_run(history_id, run_response)
datasets = run_response.json()["outputs"]
assert len(datasets) == 2, datasets
content = self.dataset_populator.get_history_dataset_content(
history_id, dataset=datasets[0])
assert content.strip() == "Test123"
assert datasets[0]["file_ext"] == "txt", datasets
assert datasets[0]["genome_build"] == "hg18", datasets
content = self.dataset_populator.get_history_dataset_content(
history_id, dataset=datasets[1])
assert content.strip() == "SecondOutputContent"
assert datasets[1]["file_ext"] == "txt"
assert datasets[1]["genome_build"] == "hg18", datasets
def test_upload_multiple_files_no_dbkey(self):
with self.dataset_populator.test_history() as history_id:
payload = self.dataset_populator.upload_payload(
history_id,
"Test123",
file_type="tabular",
dbkey=None,
extra_inputs={
"files_0|file_type": "txt",
"files_1|url_paste": "SecondOutputContent",
"files_1|NAME": "SecondOutputName",
"files_1|file_type": "txt",
"file_count": "2",
})
run_response = self.dataset_populator.tools_post(payload)
self.dataset_populator.wait_for_tool_run(history_id, run_response)
datasets = run_response.json()["outputs"]
assert len(datasets) == 2, datasets
content = self.dataset_populator.get_history_dataset_content(
history_id, dataset=datasets[0])
assert content.strip() == "Test123"
assert datasets[0]["file_ext"] == "txt", datasets
assert datasets[0]["genome_build"] == "?", datasets
content = self.dataset_populator.get_history_dataset_content(
history_id, dataset=datasets[1])
assert content.strip() == "SecondOutputContent"
assert datasets[1]["file_ext"] == "txt"
assert datasets[1]["genome_build"] == "?", datasets
def test_upload_multiple_files_space_to_tab(self):
with self.dataset_populator.test_history() as history_id:
payload = self.dataset_populator.upload_payload(
history_id,
content=ONE_TO_SIX_WITH_SPACES,
file_type="tabular",
dbkey="hg19",
extra_inputs={
"files_0|file_type": "txt",
"files_0|space_to_tab": "Yes",
"files_1|url_paste": ONE_TO_SIX_WITH_SPACES,
"files_1|NAME": "SecondOutputName",
"files_1|file_type": "txt",
"files_2|url_paste": ONE_TO_SIX_WITH_SPACES,
"files_2|NAME": "ThirdOutputName",
"files_2|file_type": "txt",
"files_2|space_to_tab": "Yes",
"file_count": "3",
})
run_response = self.dataset_populator.tools_post(payload)
self.dataset_populator.wait_for_tool_run(history_id, run_response)
datasets = run_response.json()["outputs"]
assert len(datasets) == 3, datasets
content = self.dataset_populator.get_history_dataset_content(
history_id, dataset=datasets[0])
assert content == ONE_TO_SIX_WITH_TABS
content = self.dataset_populator.get_history_dataset_content(
history_id, dataset=datasets[1])
assert content == ONE_TO_SIX_WITH_SPACES
content = self.dataset_populator.get_history_dataset_content(
history_id, dataset=datasets[2])
assert content == ONE_TO_SIX_WITH_TABS
def test_multiple_files_posix_lines(self):
with self.dataset_populator.test_history() as history_id:
payload = self.dataset_populator.upload_payload(
history_id,
content=ONE_TO_SIX_ON_WINDOWS,
file_type="tabular",
dbkey="hg19",
extra_inputs={
"files_0|file_type": "txt",
"files_0|to_posix_lines": "Yes",
"files_1|url_paste": ONE_TO_SIX_ON_WINDOWS,
"files_1|NAME": "SecondOutputName",
"files_1|file_type": "txt",
"files_1|to_posix_lines": None,
"files_2|url_paste": ONE_TO_SIX_ON_WINDOWS,
"files_2|NAME": "ThirdOutputName",
"files_2|file_type": "txt",
"file_count": "3",
})
run_response = self.dataset_populator.tools_post(payload)
self.dataset_populator.wait_for_tool_run(history_id, run_response)
datasets = run_response.json()["outputs"]
assert len(datasets) == 3, datasets
content = self.dataset_populator.get_history_dataset_content(
history_id, dataset=datasets[0])
assert content == ONE_TO_SIX_WITH_TABS
content = self.dataset_populator.get_history_dataset_content(
history_id, dataset=datasets[1])
assert content == ONE_TO_SIX_ON_WINDOWS
content = self.dataset_populator.get_history_dataset_content(
history_id, dataset=datasets[2])
assert content == ONE_TO_SIX_WITH_TABS
def _velvet_upload(self, history_id, extra_inputs):
payload = self.dataset_populator.upload_payload(
history_id,
"sequences content",
file_type="velvet",
extra_inputs=extra_inputs,
)
run_response = self.dataset_populator.tools_post(payload)
self.dataset_populator.wait_for_tool_run(history_id, run_response)
datasets = run_response.json()["outputs"]
assert len(datasets) == 1
dataset = datasets[0]
return dataset
def _get_roadmaps_content(self, history_id, dataset):
roadmaps_content = self.dataset_populator.get_history_dataset_content(
history_id, dataset=dataset, filename="Roadmaps")
return roadmaps_content
def _upload_and_get_content(self, content, **upload_kwds):
history_id, new_dataset = self._upload(content, **upload_kwds)
return self.dataset_populator.get_history_dataset_content(
history_id, dataset=new_dataset)
def _upload_and_get_details(self, content, **upload_kwds):
history_id, new_dataset = self._upload(content, **upload_kwds)
return self.dataset_populator.get_history_dataset_details(
history_id, dataset=new_dataset)
def _upload(self, content, **upload_kwds):
history_id = self.dataset_populator.new_history()
new_dataset = self.dataset_populator.new_dataset(history_id,
content=content,
**upload_kwds)
self.dataset_populator.wait_for_history(history_id, assert_ok=True)
return history_id, new_dataset
def setUp(self):
super(HistoriesApiTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.dataset_collection_populator = DatasetCollectionPopulator(self.galaxy_interactor)
class DatasetCollectionApiTestCase(api.ApiTestCase):
def setUp(self):
super(DatasetCollectionApiTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.dataset_collection_populator = DatasetCollectionPopulator(self.galaxy_interactor)
self.history_id = self.dataset_populator.new_history()
def test_create_pair_from_history(self):
payload = self.dataset_collection_populator.create_pair_payload(
self.history_id,
instance_type="history",
)
create_response = self._post("dataset_collections", payload)
dataset_collection = self._check_create_response(create_response)
returned_datasets = dataset_collection["elements"]
assert len(returned_datasets) == 2, dataset_collection
def test_create_list_from_history(self):
element_identifiers = self.dataset_collection_populator.list_identifiers(self.history_id)
payload = dict(
instance_type="history",
history_id=self.history_id,
element_identifiers=json.dumps(element_identifiers),
collection_type="list",
)
create_response = self._post("dataset_collections", payload)
dataset_collection = self._check_create_response(create_response)
returned_datasets = dataset_collection["elements"]
assert len(returned_datasets) == 3, dataset_collection
def test_create_list_of_existing_pairs(self):
pair_payload = self.dataset_collection_populator.create_pair_payload(
self.history_id,
instance_type="history",
)
pair_create_response = self._post("dataset_collections", pair_payload)
dataset_collection = self._check_create_response(pair_create_response)
hdca_id = dataset_collection["id"]
element_identifiers = [
dict(name="test1", src="hdca", id=hdca_id)
]
payload = dict(
instance_type="history",
history_id=self.history_id,
element_identifiers=json.dumps(element_identifiers),
collection_type="list",
)
create_response = self._post("dataset_collections", payload)
dataset_collection = self._check_create_response(create_response)
returned_collections = dataset_collection["elements"]
assert len(returned_collections) == 1, dataset_collection
def test_create_list_of_new_pairs(self):
identifiers = self.dataset_collection_populator.nested_collection_identifiers(self.history_id, "list:paired")
payload = dict(
collection_type="list:paired",
instance_type="history",
history_id=self.history_id,
name="a nested collection",
element_identifiers=json.dumps(identifiers),
)
create_response = self._post("dataset_collections", payload)
dataset_collection = self._check_create_response(create_response)
assert dataset_collection["collection_type"] == "list:paired"
assert dataset_collection["name"] == "a nested collection"
returned_collections = dataset_collection["elements"]
assert len(returned_collections) == 1, dataset_collection
pair_1_element = returned_collections[0]
self._assert_has_keys(pair_1_element, "element_identifier", "element_index", "object")
assert pair_1_element["element_identifier"] == "test_level_1", pair_1_element
assert pair_1_element["element_index"] == 0, pair_1_element
pair_1_object = pair_1_element["object"]
self._assert_has_keys(pair_1_object, "collection_type", "elements", "element_count")
self.assertEqual(pair_1_object["collection_type"], "paired")
self.assertEqual(pair_1_object["populated"], True)
pair_elements = pair_1_object["elements"]
assert len(pair_elements) == 2
pair_1_element_1 = pair_elements[0]
assert pair_1_element_1["element_index"] == 0
def test_list_download(self):
fetch_response = self.dataset_collection_populator.create_list_in_history(self.history_id, direct_upload=True).json()
dataset_collection = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response)
returned_dce = dataset_collection["elements"]
assert len(returned_dce) == 3, dataset_collection
create_response = self._download_dataset_collection(history_id=self.history_id, hdca_id=dataset_collection['id'])
self._assert_status_code_is(create_response, 200)
tar_contents = tarfile.open(fileobj=BytesIO(create_response.content))
namelist = tar_contents.getnames()
assert len(namelist) == 3, "Expected 3 elements in [%s]" % namelist
collection_name = dataset_collection['name']
for element, zip_path in zip(returned_dce, namelist):
assert "%s/%s.%s" % (collection_name, element['element_identifier'], element['object']['file_ext']) == zip_path
def test_pair_download(self):
fetch_response = self.dataset_collection_populator.create_pair_in_history(self.history_id, direct_upload=True).json()
dataset_collection = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response)
returned_dce = dataset_collection["elements"]
assert len(returned_dce) == 2, dataset_collection
hdca_id = dataset_collection['id']
create_response = self._download_dataset_collection(history_id=self.history_id, hdca_id=hdca_id)
self._assert_status_code_is(create_response, 200)
tar_contents = tarfile.open(fileobj=BytesIO(create_response.content))
namelist = tar_contents.getnames()
assert len(namelist) == 2, "Expected 2 elements in [%s]" % namelist
collection_name = dataset_collection['name']
for element, zip_path in zip(returned_dce, namelist):
assert "%s/%s.%s" % (collection_name, element['element_identifier'], element['object']['file_ext']) == zip_path
def test_list_pair_download(self):
fetch_response = self.dataset_collection_populator.create_list_of_pairs_in_history(self.history_id).json()
dataset_collection = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response)
returned_dce = dataset_collection["elements"]
assert len(returned_dce) == 1, dataset_collection
list_collection_name = dataset_collection['name']
pair = returned_dce[0]
create_response = self._download_dataset_collection(history_id=self.history_id, hdca_id=dataset_collection['id'])
self._assert_status_code_is(create_response, 200)
tar_contents = tarfile.open(fileobj=BytesIO(create_response.content))
namelist = tar_contents.getnames()
assert len(namelist) == 2, "Expected 2 elements in [%s]" % namelist
pair_collection_name = pair['element_identifier']
for element, zip_path in zip(pair['object']['elements'], namelist):
assert "%s/%s/%s.%s" % (list_collection_name, pair_collection_name, element['element_identifier'], element['object']['file_ext']) == zip_path
def test_list_list_download(self):
dataset_collection = self.dataset_collection_populator.create_list_of_list_in_history(self.history_id).json()
self.dataset_collection_populator.wait_for_dataset_collection(dataset_collection, assert_ok=True)
returned_dce = dataset_collection["elements"]
assert len(returned_dce) == 1, dataset_collection
create_response = self._download_dataset_collection(history_id=self.history_id, hdca_id=dataset_collection['id'])
self._assert_status_code_is(create_response, 200)
tar_contents = tarfile.open(fileobj=BytesIO(create_response.content))
namelist = tar_contents.getnames()
assert len(namelist) == 3, "Expected 3 elements in [%s]" % namelist
def test_list_list_list_download(self):
dataset_collection = self.dataset_collection_populator.create_list_of_list_in_history(self.history_id, collection_type='list:list:list').json()
self.dataset_collection_populator.wait_for_dataset_collection(dataset_collection, assert_ok=True)
returned_dce = dataset_collection["elements"]
assert len(returned_dce) == 1, dataset_collection
create_response = self._download_dataset_collection(history_id=self.history_id, hdca_id=dataset_collection['id'])
self._assert_status_code_is(create_response, 200)
tar_contents = tarfile.open(fileobj=BytesIO(create_response.content))
namelist = tar_contents.getnames()
assert len(namelist) == 3, "Expected 3 elements in [%s]" % namelist
def test_hda_security(self):
element_identifiers = self.dataset_collection_populator.pair_identifiers(self.history_id)
self.dataset_populator.make_private(self.history_id, element_identifiers[0]["id"])
with self._different_user():
history_id = self.dataset_populator.new_history()
payload = dict(
instance_type="history",
history_id=history_id,
element_identifiers=json.dumps(element_identifiers),
collection_type="paired",
)
create_response = self._post("dataset_collections", payload)
self._assert_status_code_is(create_response, 403)
def test_enforces_unique_names(self):
element_identifiers = self.dataset_collection_populator.list_identifiers(self.history_id)
element_identifiers[2]["name"] = element_identifiers[0]["name"]
payload = dict(
instance_type="history",
history_id=self.history_id,
element_identifiers=json.dumps(element_identifiers),
collection_type="list",
)
create_response = self._post("dataset_collections", payload)
self._assert_status_code_is(create_response, 400)
def test_upload_collection(self):
elements = [{"src": "files", "dbkey": "hg19", "info": "my cool bed", "tags": ["name:data1", "group:condition:treated", "machine:illumina"]}]
targets = [{
"destination": {"type": "hdca"},
"elements": elements,
"collection_type": "list",
"name": "Test upload",
"tags": ["name:collection1"]
}]
payload = {
"history_id": self.history_id,
"targets": json.dumps(targets),
"__files": {"files_0|file_data": open(self.test_data_resolver.get_filename("4.bed"))},
}
self.dataset_populator.fetch(payload)
hdca = self._assert_one_collection_created_in_history()
self.assertEqual(hdca["name"], "Test upload")
hdca_tags = hdca["tags"]
assert len(hdca_tags) == 1
assert "name:collection1" in hdca_tags
assert len(hdca["elements"]) == 1, hdca
element0 = hdca["elements"][0]
assert element0["element_identifier"] == "4.bed"
dataset0 = element0["object"]
assert dataset0["file_size"] == 61
dataset_tags = dataset0["tags"]
assert len(dataset_tags) == 3, dataset0
def test_upload_nested(self):
elements = [{"name": "samp1", "elements": [{"src": "files", "dbkey": "hg19", "info": "my cool bed"}]}]
targets = [{
"destination": {"type": "hdca"},
"elements": elements,
"collection_type": "list:list",
"name": "Test upload",
}]
payload = {
"history_id": self.history_id,
"targets": json.dumps(targets),
"__files": {"files_0|file_data": open(self.test_data_resolver.get_filename("4.bed"))},
}
self.dataset_populator.fetch(payload)
hdca = self._assert_one_collection_created_in_history()
self.assertEqual(hdca["name"], "Test upload")
assert len(hdca["elements"]) == 1, hdca
element0 = hdca["elements"][0]
assert element0["element_identifier"] == "samp1"
def test_upload_collection_from_url(self):
elements = [{"src": "url", "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed", "info": "my cool bed"}]
targets = [{
"destination": {"type": "hdca"},
"elements": elements,
"collection_type": "list",
}]
payload = {
"history_id": self.history_id,
"targets": json.dumps(targets),
"__files": {"files_0|file_data": open(self.test_data_resolver.get_filename("4.bed"))},
}
self.dataset_populator.fetch(payload)
hdca = self._assert_one_collection_created_in_history()
assert len(hdca["elements"]) == 1, hdca
element0 = hdca["elements"][0]
assert element0["element_identifier"] == "4.bed"
assert element0["object"]["file_size"] == 61
def _assert_one_collection_created_in_history(self):
contents_response = self._get("histories/%s/contents/dataset_collections" % self.history_id)
self._assert_status_code_is(contents_response, 200)
contents = contents_response.json()
assert len(contents) == 1
hdca = contents[0]
assert hdca["history_content_type"] == "dataset_collection"
hdca_id = hdca["id"]
collection_response = self._get("histories/%s/contents/dataset_collections/%s" % (self.history_id, hdca_id))
self._assert_status_code_is(collection_response, 200)
return collection_response.json()
def _check_create_response(self, create_response):
self._assert_status_code_is(create_response, 200)
dataset_collection = create_response.json()
self._assert_has_keys(dataset_collection, "elements", "url", "name", "collection_type", "element_count")
return dataset_collection
def _download_dataset_collection(self, history_id, hdca_id):
return self._get("histories/%s/contents/dataset_collections/%s/download" % (history_id, hdca_id))