def setUpClass(cls): cls.proj_name = "Y.Mom_15_01" cls.proj_id = "P1155" cls.libprep_id = "P1155_prepA" cls.seqrun_id = "P1155_seqrunA" cls.sample_name = "P1155_101" cls.engine_name = "piper_ngi" cls.proj_basepath = tempfile.mkdtemp() cls.workflow_name = "merge_process_variantcall" cls.xml_path = os.path.join(cls.proj_basepath, "some_config.xml") cls.exit_file = os.path.join(cls.proj_basepath, "some_file.exit") cls.config = load_yaml_config(locate_ngi_config()) cls.project_obj = NGIProject(name=cls.proj_name, dirname=cls.proj_name, project_id=cls.proj_id, base_path=cls.proj_basepath) cls.sample_obj = cls.project_obj.add_sample(name=cls.sample_name, dirname=cls.sample_name) # create a mock that can replace calls to charon cls.charon_mock = mock.Mock() cls.charon_mock.sample_get_libpreps = mock.Mock(return_value={ 'libpreps': [{ 'qc': 'PASSED', 'libprepid': cls.libprep_id }] }) cls.charon_mock.libprep_get_seqruns = mock.Mock( return_value={'seqruns': [{ 'seqrunid': cls.seqrun_id }]}) cls.charon_mock.seqrun_get = mock.Mock( return_value={'alignment_status': ''}) cls.charon_mock.project_get = mock.Mock( return_value={'sequencing_facility': 'Unknown'})
def setUpClass(cls): cls.proj_name = "Y.Mom_15_01" cls.proj_id = "P1155" cls.libprep_id = "P1155_prepA" cls.seqrun_id = "P1155_seqrunA" cls.sample_name = "P1155_101" cls.engine_name = "piper_ngi" cls.proj_basepath = tempfile.mkdtemp() cls.workflow_name = "merge_process_variantcall" cls.xml_path = os.path.join(cls.proj_basepath, "some_config.xml") cls.exit_file = os.path.join(cls.proj_basepath, "some_file.exit") cls.config = load_yaml_config(locate_ngi_config()) cls.project_obj = NGIProject(name=cls.proj_name, dirname=cls.proj_name, project_id=cls.proj_id, base_path=cls.proj_basepath) cls.sample_obj = cls.project_obj.add_sample(name=cls.sample_name, dirname=cls.sample_name) # create a mock that can replace calls to charon cls.charon_mock = mock.Mock() cls.charon_mock.sample_get_libpreps = mock.Mock(return_value = { 'libpreps': [{'qc': 'PASSED', 'libprepid': cls.libprep_id}]}) cls.charon_mock.libprep_get_seqruns = mock.Mock(return_value = { 'seqruns': [{'seqrunid': cls.seqrun_id}]}) cls.charon_mock.seqrun_get = mock.Mock(return_value = { 'alignment_status': ''}) cls.charon_mock.project_get = mock.Mock(return_value={ 'sequencing_facility': 'Unknown'})
def test_load_yaml_config(self): config_file_path = os.path.join(self.tmp_dir, 'config.yaml') with open(config_file_path, 'w') as config_file: config_file.write( yaml.dump(self.config_dict, default_flow_style=False)) self.assertEqual(self.config_dict, load_yaml_config(config_file_path))
def run_server(config_file, queues=None, task_module=None): """ Loads configuration and launches the server """ config = load_yaml_config(config_file) cl = ["celery"] tasks = task_module if task_module else "ngi_pipeline.distributed.tasks" cl += ["-A", tasks, "worker", "-n", "ngi_server"] check_call(cl)
def __call__(self, *args, **kwargs): # Assign positional args to relevant parameters # An alternative way to do this is with inspect.getargspec(self.f) # instead of self.f.func_code.co_varnames, but it's an additional import kwargs.update(dict(zip(self.f.func_code.co_varnames, args))) if not kwargs.get("config"): if not kwargs.get("config_file_path"): kwargs["config_file_path"] = locate_ngi_config() kwargs["config"] = load_yaml_config(kwargs["config_file_path"]) return self.f(**kwargs)
def test_workflows(self): config_file_path = locate_ngi_config() config = load_yaml_config(config_file_path) for workflow_name, workflow_dict in config.get("test_data", {}).get( "workflows", {}).iteritems(): # Load and rewrite config file as needed customize_config_dict = workflow_dict.get("customize_config") if customize_config_dict: config = update_dict(config, customize_config_dict) #self._install_test_files(workflow_dict) LOG.info( 'Starting test analysis pipeline for workflow "{}"'.format( workflow_name)) try: local_files = workflow_dict["local_files"] except KeyError: raise ValueError( "Required paths to input files for testing do not" "exist in config file (test_data.workflows." "{}.local_files); cannot proceed.".format(workflow_name)) try: flowcell_path = local_files["flowcell"] except KeyError: raise ValueError( "Path to flowcell is required and not specified " "in configuration file (test_data.workflows." "{}.local_files.flowcell); cannot proceed.".format( workflow_name)) try: test_project = workflow_dict["test_project"] test_proj_id = test_project["project_id"] test_proj_name = test_project["project_name"] test_proj_bpa = test_project["bpa"] except KeyError as e: raise ValueError( "Test project information is missing from config " "file (under test_data.workflows.{}.test_project " "({}); cannot proceed.".format(workflow_name, e.msg)) charon_session = CharonSession(config=config) try: charon_session.project_delete(projectid=test_proj_id) except CharonError: pass charon_session.project_create(projectid=test_proj_id, name=test_proj_name, status="OPEN", best_practice_analysis=test_proj_bpa) process_demultiplexed_flowcells([flowcell_path], fallback_libprep="A", config=config)
def __call__(self, *args, **kwargs): # Assign positional args to relevant parameters # An alternative way to do this is with inspect.getargspec(self.f) # instead of self.f.func_code.co_varnames, but it's an additional import kwargs.update(dict(list(zip(self.f.__code__.co_varnames, args)))) if not kwargs.get("config"): if not kwargs.get("config_file_path"): kwargs["config_file_path"] = locate_ngi_config() kwargs["config"] = load_yaml_config(kwargs["config_file_path"]) if kwargs.get("quiet"): kwargs["config"]["quiet"] = True if kwargs.get("manual"): kwargs["config"]["manual"] = True return self.f(**kwargs)
def test_workflows(self): config_file_path = locate_ngi_config() config = load_yaml_config(config_file_path) for workflow_name, workflow_dict in config.get("test_data", {}).get("workflows", {}).iteritems(): # Load and rewrite config file as needed customize_config_dict = workflow_dict.get("customize_config") if customize_config_dict: config = update_dict(config, customize_config_dict) #self._install_test_files(workflow_dict) LOG.info('Starting test analysis pipeline for workflow "{}"'.format(workflow_name)) try: local_files = workflow_dict["local_files"] except KeyError: raise ValueError("Required paths to input files for testing do not" "exist in config file (test_data.workflows." "{}.local_files); cannot proceed.".format(workflow_name)) try: flowcell_path = local_files["flowcell"] except KeyError: raise ValueError("Path to flowcell is required and not specified " "in configuration file (test_data.workflows." "{}.local_files.flowcell); cannot proceed.".format(workflow_name)) try: test_project = workflow_dict["test_project"] test_proj_id = test_project["project_id"] test_proj_name = test_project["project_name"] test_proj_bpa = test_project["bpa"] except KeyError as e: raise ValueError("Test project information is missing from config " "file (under test_data.workflows.{}.test_project " "({}); cannot proceed.".format(workflow_name, e.msg)) charon_session = CharonSession(config=config) try: charon_session.project_delete(projectid=test_proj_id) except CharonError: pass charon_session.project_create(projectid=test_proj_id, name=test_proj_name, status="OPEN", best_practice_analysis=test_proj_bpa) process_demultiplexed_flowcells([flowcell_path], fallback_libprep="A", config=config)
def test_load_yaml_config(self): config_file_path = os.path.join(self.tmp_dir, "config.yaml") with open(config_file_path, 'w') as config_file: config_file.write(yaml.dump(self.config_dict, default_flow_style=False)) self.assertEqual(self.config_dict, load_yaml_config(config_file_path))
def main(config_file): config = load_yaml_config(config_file) messenger = CeleryMessenger(config.get('celery'), 'ngi_pipeline') messenger.send_message('launch_main_analysis', '/Users/guillem/archive/test_data_empty_files')