import os
from galaxy_test.driver import integration_util
SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
EMBEDDED_PULSAR_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "embedded_pulsar_metadata_extended_job_conf.yml")
class EmbeddedAndExtendedMetadataPulsarIntegrationInstance(integration_util.IntegrationInstance):
"""Describe a Galaxy test instance with embedded pulsar configured."""
framework_tool_and_types = True
@classmethod
def handle_galaxy_config_kwds(cls, config):
config["job_config_file"] = EMBEDDED_PULSAR_JOB_CONFIG_FILE
config['object_store_store_by'] = 'uuid'
config['metadata_strategy'] = 'extended'
config['retry_metadata_internally'] = False
instance = integration_util.integration_module_instance(EmbeddedAndExtendedMetadataPulsarIntegrationInstance)
test_tools = integration_util.integration_tool_runner(
[
"simple_constructs",
"metadata_bam",
# "job_properties", # https://github.com/galaxyproject/galaxy/issues/11813
]
)
}
}
new_dataset = self.dataset_populator.fetch(
payload, assert_ok=True).json()["outputs"][0]
self.dataset_populator.wait_for_history(history_id, assert_ok=True)
content = self.dataset_populator.get_history_dataset_content(
history_id=history_id,
dataset_id=new_dataset['id'],
)
assert content == upload_content
class ExtendedMetadataIntegrationInstance(integration_util.IntegrationInstance
):
"""Describe a Galaxy test instance with embedded pulsar configured."""
framework_tool_and_types = True
@classmethod
def handle_galaxy_config_kwds(cls, config):
config["metadata_strategy"] = "extended"
config["tool_evaluation_strategy"] = "remote"
config["object_store_store_by"] = "uuid"
config["retry_metadata_internally"] = False
instance = integration_util.integration_module_instance(
ExtendedMetadataIntegrationInstance)
test_tools = integration_util.integration_tool_runner(TEST_TOOL_IDS)
framework_tool_and_types = True
@classmethod
def handle_galaxy_config_kwds(cls, config):
amqp_url = os.environ.get("GALAXY_TEST_AMQP_URL", None)
if amqp_url is None:
pytest.skip("External AMQP URL not configured for test")
jobs_directory = os.path.join(cls._test_driver.mkdtemp(),
"pulsar_staging")
safe_makedirs(jobs_directory)
job_conf_template = string.Template(JOB_CONF_TEMPLATE)
job_conf_str = job_conf_template.substitute(
amqp_url=AMQP_URL,
jobs_directory=jobs_directory,
galaxy_home=os.path.join(SCRIPT_DIRECTORY, os.pardir))
with tempfile.NamedTemporaryFile(suffix="_mq_job_conf.yml",
mode="w",
delete=False) as job_conf:
job_conf.write(job_conf_str)
config["job_config_file"] = job_conf.name
infrastructure_url = "http://localhost:$GALAXY_WEB_PORT"
config["galaxy_infrastructure_url"] = infrastructure_url
instance = integration_util.integration_module_instance(
EmbeddedMessageQueuePulsarIntegrationInstance)
test_tools = integration_util.integration_tool_runner(["simple_constructs"])
"""Run various framework tool tests with outputs_to_working_directory."""
from galaxy_test.driver import integration_util
class JobOutputsToWorkingDirectoryIntegrationInstance(
integration_util.IntegrationInstance):
"""Describe a Galaxy test instance with outputs_to_working_directory enabled."""
framework_tool_and_types = True
@classmethod
def handle_galaxy_config_kwds(cls, config):
config["outputs_to_working_directory"] = True
instance = integration_util.integration_module_instance(
JobOutputsToWorkingDirectoryIntegrationInstance)
test_tools = integration_util.integration_tool_runner([
"output_format", "output_empty_work_dir",
"collection_creates_pair_from_work_dir"
])
"""Integration tests for the flush_per_n_datasets setting."""
from galaxy_test.driver import integration_util
class FlushPerNDatasetsTestCase(integration_util.IntegrationInstance):
"""Describe a Galaxy test instance with embedded pulsar configured."""
framework_tool_and_types = True
@classmethod
def handle_galaxy_config_kwds(cls, config):
config["flush_per_n_datasets"] = 1
instance = integration_util.integration_module_instance(FlushPerNDatasetsTestCase)
test_tools = integration_util.integration_tool_runner([
'collection_creates_dynamic_nested',
'collection_creates_dynamic_list_of_pairs'
])
super().setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.history_id = self.dataset_populator.new_history()
@classmethod
def setUpClass(cls):
skip_if_container_type_unavailable(cls)
super().setUpClass()
class EmbeddedSingularityPulsarIntegrationTestCase(
BaseEmbeddedPulsarContainerIntegrationTestCase, MulledJobTestCases):
# singularity passes $HOME by default
default_container_home_dir = os.environ.get('HOME', '/')
job_config_file = EMBEDDED_PULSAR_JOB_CONFIG_FILE_SINGULARITY
container_type = 'singularity'
class EmbeddedDockerPulsarIntegrationTestCase(
BaseEmbeddedPulsarContainerIntegrationTestCase, MulledJobTestCases):
job_config_file = EMBEDDED_PULSAR_JOB_CONFIG_FILE_DOCKER
container_type = 'docker'
instance = integration_util.integration_module_instance(
EmbeddedSingularityPulsarIntegrationTestCase)
test_tools = integration_util.integration_tool_runner([
"tool_directory_docker",
])
"""Integration tests for the Pulsar embedded runner with outputs to working directory."""
import os
from galaxy_test.driver import integration_util
SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
EMBEDDED_PULSAR_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "embedded_pulsar_job_conf.yml")
class EmbeddedCopyWorkingPulsarIntegrationInstance(integration_util.IntegrationInstance):
"""Describe a Galaxy test instance with embedded pulsar configured."""
framework_tool_and_types = True
@classmethod
def handle_galaxy_config_kwds(cls, config):
config["job_config_file"] = EMBEDDED_PULSAR_JOB_CONFIG_FILE
config["outputs_to_working_directory"] = True
instance = integration_util.integration_module_instance(EmbeddedCopyWorkingPulsarIntegrationInstance)
test_tools = integration_util.integration_tool_runner(["output_format", "output_filter"])
"""Integration tests for the Pulsar embedded runner."""
import os
from galaxy_test.driver import integration_util
SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
EMBEDDED_PULSAR_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY,
"embedded_pulsar_job_conf.xml")
class EmbeddedPulsarIntegrationInstance(integration_util.IntegrationInstance):
"""Describe a Galaxy test instance with embedded pulsar configured."""
framework_tool_and_types = True
@classmethod
def handle_galaxy_config_kwds(cls, config):
config["job_config_file"] = EMBEDDED_PULSAR_JOB_CONFIG_FILE
instance = integration_util.integration_module_instance(
EmbeddedPulsarIntegrationInstance)
test_tools = integration_util.integration_tool_runner([
"simple_constructs",
"multi_data_param",
"output_filter",
"vcf_bgzip_test",
])
SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
EMBEDDED_PULSAR_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY,
"embedded_pulsar_job_conf.xml")
class EmbeddedPulsarIntegrationInstance(integration_util.IntegrationInstance):
"""Describe a Galaxy test instance with embedded pulsar configured."""
framework_tool_and_types = True
@classmethod
def handle_galaxy_config_kwds(cls, config):
config["job_config_file"] = EMBEDDED_PULSAR_JOB_CONFIG_FILE
instance = integration_util.integration_module_instance(
EmbeddedPulsarIntegrationInstance)
test_tools = integration_util.integration_tool_runner([
"collection_creates_dynamic_nested_from_json",
"composite",
"simple_constructs",
"multi_data_param",
"output_filter",
"vcf_bgzip_test",
"environment_variables",
"multi_output_assign_primary_ext_dbkey",
"strict_shell",
"tool_provided_metadata_9",
])
import os
from galaxy_test.driver import integration_util
SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
EMBEDDED_PULSAR_JOB_CONFIG_FILE = os.path.join(
SCRIPT_DIRECTORY, "embedded_pulsar_metadata_job_conf.yml")
class EmbeddedMetadataPulsarIntegrationInstance(
integration_util.IntegrationInstance):
"""Describe a Galaxy test instance with embedded pulsar configured."""
framework_tool_and_types = True
@classmethod
def handle_galaxy_config_kwds(cls, config):
config["job_config_file"] = EMBEDDED_PULSAR_JOB_CONFIG_FILE
config['object_store_store_by'] = 'uuid'
# We set the global metadata_strategy to `extended, but`metadata_strategy is
# being overridden in embedded_pulsar_metadata_job_conf.yml, since extended_metadata does not yet work on pulsar
config['metadata_strategy'] = 'extended'
config['retry_metadata_internally'] = False
instance = integration_util.integration_module_instance(
EmbeddedMetadataPulsarIntegrationInstance)
test_tools = integration_util.integration_tool_runner(
["simple_constructs", "metadata_bam"])
