import os
from galaxy_test.driver import integration_util
SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
EMBEDDED_PULSAR_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "embedded_pulsar_metadata_extended_job_conf.yml")
class EmbeddedAndExtendedMetadataPulsarIntegrationInstance(integration_util.IntegrationInstance):
"""Describe a Galaxy test instance with embedded pulsar configured."""
framework_tool_and_types = True
@classmethod
def handle_galaxy_config_kwds(cls, config):
config["job_config_file"] = EMBEDDED_PULSAR_JOB_CONFIG_FILE
config['object_store_store_by'] = 'uuid'
config['metadata_strategy'] = 'extended'
config['retry_metadata_internally'] = False
instance = integration_util.integration_module_instance(EmbeddedAndExtendedMetadataPulsarIntegrationInstance)
test_tools = integration_util.integration_tool_runner(
[
"simple_constructs",
"metadata_bam",
# "job_properties", # https://github.com/galaxyproject/galaxy/issues/11813
]
)
test_files = [f for f in os.listdir(TEST_FILE_DIR) if "." in f]
files = [os.path.join(TEST_FILE_DIR, f) for f in test_files]
datatypes = [find_datatype(registry, f) for f in test_files]
uploadable = [datatype.file_ext in registry.upload_file_formats for datatype in datatypes]
test_data_description = [TestData(*items) for items in zip(files, datatypes, uploadable)]
return {os.path.basename(data.path): data for data in test_data_description}
class UploadTestDatatypeDataTestCase(BaseUploadContentConfigurationInstance):
framework_tool_and_types = False
datatypes_conf_override = DATATYPES_CONFIG
object_store_config = None
object_store_config_path = None
instance = integration_util.integration_module_instance(UploadTestDatatypeDataTestCase)
registry = Registry()
registry.load_datatypes(root_dir=GALAXY_ROOT, config=DATATYPES_CONFIG)
TEST_CASES = collect_test_data(registry)
@pytest.mark.parametrize('test_data', TEST_CASES.values(), ids=list(TEST_CASES.keys()))
def test_upload_datatype_auto(instance, test_data, temp_file):
upload_datatype_helper(instance, test_data, temp_file)
def upload_datatype_helper(instance, test_data, temp_file):
is_compressed = False
for is_method in (is_bz2, is_gzip, is_zip):
framework_tool_and_types = True
@classmethod
def handle_galaxy_config_kwds(cls, config):
amqp_url = os.environ.get("GALAXY_TEST_AMQP_URL", None)
if amqp_url is None:
pytest.skip("External AMQP URL not configured for test")
jobs_directory = os.path.join(cls._test_driver.mkdtemp(),
"pulsar_staging")
safe_makedirs(jobs_directory)
job_conf_template = string.Template(JOB_CONF_TEMPLATE)
job_conf_str = job_conf_template.substitute(
amqp_url=AMQP_URL,
jobs_directory=jobs_directory,
galaxy_home=os.path.join(SCRIPT_DIRECTORY, os.pardir))
with tempfile.NamedTemporaryFile(suffix="_mq_job_conf.yml",
mode="w",
delete=False) as job_conf:
job_conf.write(job_conf_str)
config["job_config_file"] = job_conf.name
infrastructure_url = "http://localhost:$GALAXY_WEB_PORT"
config["galaxy_infrastructure_url"] = infrastructure_url
instance = integration_util.integration_module_instance(
EmbeddedMessageQueuePulsarIntegrationInstance)
test_tools = integration_util.integration_tool_runner(["simple_constructs"])
"""Integration tests for the flush_per_n_datasets setting."""
from galaxy_test.driver import integration_util
class FlushPerNDatasetsTestCase(integration_util.IntegrationInstance):
"""Describe a Galaxy test instance with embedded pulsar configured."""
framework_tool_and_types = True
@classmethod
def handle_galaxy_config_kwds(cls, config):
config["flush_per_n_datasets"] = 1
instance = integration_util.integration_module_instance(FlushPerNDatasetsTestCase)
test_tools = integration_util.integration_tool_runner([
'collection_creates_dynamic_nested',
'collection_creates_dynamic_list_of_pairs'
])
}
}
new_dataset = self.dataset_populator.fetch(
payload, assert_ok=True).json()["outputs"][0]
self.dataset_populator.wait_for_history(history_id, assert_ok=True)
content = self.dataset_populator.get_history_dataset_content(
history_id=history_id,
dataset_id=new_dataset['id'],
)
assert content == upload_content
class ExtendedMetadataIntegrationInstance(integration_util.IntegrationInstance
):
"""Describe a Galaxy test instance with embedded pulsar configured."""
framework_tool_and_types = True
@classmethod
def handle_galaxy_config_kwds(cls, config):
config["metadata_strategy"] = "extended"
config["tool_evaluation_strategy"] = "remote"
config["object_store_store_by"] = "uuid"
config["retry_metadata_internally"] = False
instance = integration_util.integration_module_instance(
ExtendedMetadataIntegrationInstance)
test_tools = integration_util.integration_tool_runner(TEST_TOOL_IDS)
"""Run various framework tool tests with outputs_to_working_directory."""
from galaxy_test.driver import integration_util
class JobOutputsToWorkingDirectoryIntegrationInstance(
integration_util.IntegrationInstance):
"""Describe a Galaxy test instance with outputs_to_working_directory enabled."""
framework_tool_and_types = True
@classmethod
def handle_galaxy_config_kwds(cls, config):
config["outputs_to_working_directory"] = True
instance = integration_util.integration_module_instance(
JobOutputsToWorkingDirectoryIntegrationInstance)
test_tools = integration_util.integration_tool_runner([
"output_format", "output_empty_work_dir",
"collection_creates_pair_from_work_dir"
])
class UploadTestDosDiskAndDiskTestCase(BaseObjectstoreUploadTest):
object_store_template = DISTRIBUTED_OBJECT_STORE_CONFIG
@classmethod
def get_object_store_kwargs(cls):
return {'temp_directory': cls.object_stores_parent}
class UploadTestDosIrodsAndDiskTestCase(IrodsUploadTestDatatypeDataTestCase):
object_store_template = DISTRIBUTED_IRODS_OBJECT_STORE_CONFIG
distributed_instance = integration_util.integration_module_instance(
UploadTestDosDiskAndDiskTestCase)
irods_instance = integration_util.integration_module_instance(
IrodsUploadTestDatatypeDataTestCase)
distributed_and_irods_instance = integration_util.integration_module_instance(
UploadTestDosIrodsAndDiskTestCase)
@pytest.mark.parametrize('test_data',
TEST_CASES.values(),
ids=list(TEST_CASES.keys()))
def test_upload_datatype_dos_disk_and_disk(distributed_instance, test_data,
temp_file):
upload_datatype_helper(distributed_instance, test_data, temp_file)
@pytest.mark.parametrize('test_data',
super().setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.history_id = self.dataset_populator.new_history()
@classmethod
def setUpClass(cls):
skip_if_container_type_unavailable(cls)
super().setUpClass()
class EmbeddedSingularityPulsarIntegrationTestCase(
BaseEmbeddedPulsarContainerIntegrationTestCase, MulledJobTestCases):
# singularity passes $HOME by default
default_container_home_dir = os.environ.get('HOME', '/')
job_config_file = EMBEDDED_PULSAR_JOB_CONFIG_FILE_SINGULARITY
container_type = 'singularity'
class EmbeddedDockerPulsarIntegrationTestCase(
BaseEmbeddedPulsarContainerIntegrationTestCase, MulledJobTestCases):
job_config_file = EMBEDDED_PULSAR_JOB_CONFIG_FILE_DOCKER
container_type = 'docker'
instance = integration_util.integration_module_instance(
EmbeddedSingularityPulsarIntegrationTestCase)
test_tools = integration_util.integration_tool_runner([
"tool_directory_docker",
])
"""Integration tests for the Pulsar embedded runner with outputs to working directory."""
import os
from galaxy_test.driver import integration_util
SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
EMBEDDED_PULSAR_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "embedded_pulsar_job_conf.yml")
class EmbeddedCopyWorkingPulsarIntegrationInstance(integration_util.IntegrationInstance):
"""Describe a Galaxy test instance with embedded pulsar configured."""
framework_tool_and_types = True
@classmethod
def handle_galaxy_config_kwds(cls, config):
config["job_config_file"] = EMBEDDED_PULSAR_JOB_CONFIG_FILE
config["outputs_to_working_directory"] = True
instance = integration_util.integration_module_instance(EmbeddedCopyWorkingPulsarIntegrationInstance)
test_tools = integration_util.integration_tool_runner(["output_format", "output_filter"])
"""Integration tests for the Pulsar embedded runner."""
import os
from galaxy_test.driver import integration_util
SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
EMBEDDED_PULSAR_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY,
"embedded_pulsar_job_conf.xml")
class EmbeddedPulsarIntegrationInstance(integration_util.IntegrationInstance):
"""Describe a Galaxy test instance with embedded pulsar configured."""
framework_tool_and_types = True
@classmethod
def handle_galaxy_config_kwds(cls, config):
config["job_config_file"] = EMBEDDED_PULSAR_JOB_CONFIG_FILE
instance = integration_util.integration_module_instance(
EmbeddedPulsarIntegrationInstance)
test_tools = integration_util.integration_tool_runner([
"simple_constructs",
"multi_data_param",
"output_filter",
"vcf_bgzip_test",
])
"multi_output_recurse",
"tool_provided_metadata_1",
"tool_provided_metadata_2",
"tool_provided_metadata_3",
"tool_provided_metadata_4",
"tool_provided_metadata_5",
"tool_provided_metadata_6",
"tool_provided_metadata_7",
"tool_provided_metadata_8",
"tool_provided_metadata_9",
"tool_provided_metadata_10",
"tool_provided_metadata_11",
"tool_provided_metadata_12",
"composite_output",
"composite_output_tests",
"metadata",
"metadata_bam",
"output_format",
"output_auto_format",
]
class SwiftObjectStoreIntegrationTestCase(
BaseSwiftObjectStoreIntegrationTestCase):
pass
instance = integration_util.integration_module_instance(
SwiftObjectStoreIntegrationTestCase)
test_tools = integration_util.integration_tool_runner(TEST_TOOL_IDS)
"""Integration tests for the Pulsar embedded runner."""
from galaxy_test.driver import integration_util
from .test_extended_metadata import (
ExtendedMetadataIntegrationInstance,
TEST_TOOL_IDS
)
class ExtendedMetadataOutputsToWorkingDirIntegrationInstance(ExtendedMetadataIntegrationInstance):
"""Describe a Galaxy test instance with metadata_strategy set to extended and outputs_to_working_dir set."""
@classmethod
def handle_galaxy_config_kwds(cls, config):
config["metadata_strategy"] = "extended"
config["object_store_store_by"] = "uuid"
config["outpus_to_working_dir"] = True
config["retry_metadata_internally"] = False
instance = integration_util.integration_module_instance(ExtendedMetadataOutputsToWorkingDirIntegrationInstance)
test_tools = integration_util.integration_tool_runner(TEST_TOOL_IDS)
