def setUpClass(cls):
cls.proj_name = "Y.Mom_15_01"
cls.proj_id = "P1155"
cls.libprep_id = "P1155_prepA"
cls.seqrun_id = "P1155_seqrunA"
cls.sample_name = "P1155_101"
cls.engine_name = "piper_ngi"
cls.proj_basepath = tempfile.mkdtemp()
cls.workflow_name = "merge_process_variantcall"
cls.xml_path = os.path.join(cls.proj_basepath, "some_config.xml")
cls.exit_file = os.path.join(cls.proj_basepath, "some_file.exit")
cls.config = load_yaml_config(locate_ngi_config())
cls.project_obj = NGIProject(name=cls.proj_name,
dirname=cls.proj_name,
project_id=cls.proj_id,
base_path=cls.proj_basepath)
cls.sample_obj = cls.project_obj.add_sample(name=cls.sample_name,
dirname=cls.sample_name)
# create a mock that can replace calls to charon
cls.charon_mock = mock.Mock()
cls.charon_mock.sample_get_libpreps = mock.Mock(return_value={
'libpreps': [{
'qc': 'PASSED',
'libprepid': cls.libprep_id
}]
})
cls.charon_mock.libprep_get_seqruns = mock.Mock(
return_value={'seqruns': [{
'seqrunid': cls.seqrun_id
}]})
cls.charon_mock.seqrun_get = mock.Mock(
return_value={'alignment_status': ''})
cls.charon_mock.project_get = mock.Mock(
return_value={'sequencing_facility': 'Unknown'})
def setUpClass(cls):
cls.proj_name = "Y.Mom_15_01"
cls.proj_id = "P1155"
cls.libprep_id = "P1155_prepA"
cls.seqrun_id = "P1155_seqrunA"
cls.sample_name = "P1155_101"
cls.engine_name = "piper_ngi"
cls.proj_basepath = tempfile.mkdtemp()
cls.workflow_name = "merge_process_variantcall"
cls.xml_path = os.path.join(cls.proj_basepath, "some_config.xml")
cls.exit_file = os.path.join(cls.proj_basepath, "some_file.exit")
cls.config = load_yaml_config(locate_ngi_config())
cls.project_obj = NGIProject(name=cls.proj_name,
dirname=cls.proj_name,
project_id=cls.proj_id,
base_path=cls.proj_basepath)
cls.sample_obj = cls.project_obj.add_sample(name=cls.sample_name,
dirname=cls.sample_name)
# create a mock that can replace calls to charon
cls.charon_mock = mock.Mock()
cls.charon_mock.sample_get_libpreps = mock.Mock(return_value = {
'libpreps': [{'qc': 'PASSED', 'libprepid': cls.libprep_id}]})
cls.charon_mock.libprep_get_seqruns = mock.Mock(return_value = {
'seqruns': [{'seqrunid': cls.seqrun_id}]})
cls.charon_mock.seqrun_get = mock.Mock(return_value = {
'alignment_status': ''})
cls.charon_mock.project_get = mock.Mock(return_value={
'sequencing_facility': 'Unknown'})
def test_load_yaml_config(self):
config_file_path = os.path.join(self.tmp_dir, 'config.yaml')
with open(config_file_path, 'w') as config_file:
config_file.write(
yaml.dump(self.config_dict, default_flow_style=False))
self.assertEqual(self.config_dict, load_yaml_config(config_file_path))
def run_server(config_file, queues=None, task_module=None):
""" Loads configuration and launches the server
"""
config = load_yaml_config(config_file)
cl = ["celery"]
tasks = task_module if task_module else "ngi_pipeline.distributed.tasks"
cl += ["-A", tasks, "worker", "-n", "ngi_server"]
check_call(cl)
def __call__(self, *args, **kwargs):
# Assign positional args to relevant parameters
# An alternative way to do this is with inspect.getargspec(self.f)
# instead of self.f.func_code.co_varnames, but it's an additional import
kwargs.update(dict(zip(self.f.func_code.co_varnames, args)))
if not kwargs.get("config"):
if not kwargs.get("config_file_path"):
kwargs["config_file_path"] = locate_ngi_config()
kwargs["config"] = load_yaml_config(kwargs["config_file_path"])
return self.f(**kwargs)
def test_workflows(self):
config_file_path = locate_ngi_config()
config = load_yaml_config(config_file_path)
for workflow_name, workflow_dict in config.get("test_data", {}).get(
"workflows", {}).iteritems():
# Load and rewrite config file as needed
customize_config_dict = workflow_dict.get("customize_config")
if customize_config_dict:
config = update_dict(config, customize_config_dict)
#self._install_test_files(workflow_dict)
LOG.info(
'Starting test analysis pipeline for workflow "{}"'.format(
workflow_name))
try:
local_files = workflow_dict["local_files"]
except KeyError:
raise ValueError(
"Required paths to input files for testing do not"
"exist in config file (test_data.workflows."
"{}.local_files); cannot proceed.".format(workflow_name))
try:
flowcell_path = local_files["flowcell"]
except KeyError:
raise ValueError(
"Path to flowcell is required and not specified "
"in configuration file (test_data.workflows."
"{}.local_files.flowcell); cannot proceed.".format(
workflow_name))
try:
test_project = workflow_dict["test_project"]
test_proj_id = test_project["project_id"]
test_proj_name = test_project["project_name"]
test_proj_bpa = test_project["bpa"]
except KeyError as e:
raise ValueError(
"Test project information is missing from config "
"file (under test_data.workflows.{}.test_project "
"({}); cannot proceed.".format(workflow_name, e.msg))
charon_session = CharonSession(config=config)
try:
charon_session.project_delete(projectid=test_proj_id)
except CharonError:
pass
charon_session.project_create(projectid=test_proj_id,
name=test_proj_name,
status="OPEN",
best_practice_analysis=test_proj_bpa)
process_demultiplexed_flowcells([flowcell_path],
fallback_libprep="A",
config=config)
def __call__(self, *args, **kwargs):
# Assign positional args to relevant parameters
# An alternative way to do this is with inspect.getargspec(self.f)
# instead of self.f.func_code.co_varnames, but it's an additional import
kwargs.update(dict(list(zip(self.f.__code__.co_varnames, args))))
if not kwargs.get("config"):
if not kwargs.get("config_file_path"):
kwargs["config_file_path"] = locate_ngi_config()
kwargs["config"] = load_yaml_config(kwargs["config_file_path"])
if kwargs.get("quiet"):
kwargs["config"]["quiet"] = True
if kwargs.get("manual"):
kwargs["config"]["manual"] = True
return self.f(**kwargs)
def test_workflows(self):
config_file_path = locate_ngi_config()
config = load_yaml_config(config_file_path)
for workflow_name, workflow_dict in config.get("test_data", {}).get("workflows", {}).iteritems():
# Load and rewrite config file as needed
customize_config_dict = workflow_dict.get("customize_config")
if customize_config_dict:
config = update_dict(config, customize_config_dict)
#self._install_test_files(workflow_dict)
LOG.info('Starting test analysis pipeline for workflow "{}"'.format(workflow_name))
try:
local_files = workflow_dict["local_files"]
except KeyError:
raise ValueError("Required paths to input files for testing do not"
"exist in config file (test_data.workflows."
"{}.local_files); cannot proceed.".format(workflow_name))
try:
flowcell_path = local_files["flowcell"]
except KeyError:
raise ValueError("Path to flowcell is required and not specified "
"in configuration file (test_data.workflows."
"{}.local_files.flowcell); cannot proceed.".format(workflow_name))
try:
test_project = workflow_dict["test_project"]
test_proj_id = test_project["project_id"]
test_proj_name = test_project["project_name"]
test_proj_bpa = test_project["bpa"]
except KeyError as e:
raise ValueError("Test project information is missing from config "
"file (under test_data.workflows.{}.test_project "
"({}); cannot proceed.".format(workflow_name, e.msg))
charon_session = CharonSession(config=config)
try:
charon_session.project_delete(projectid=test_proj_id)
except CharonError:
pass
charon_session.project_create(projectid=test_proj_id, name=test_proj_name,
status="OPEN", best_practice_analysis=test_proj_bpa)
process_demultiplexed_flowcells([flowcell_path], fallback_libprep="A",
config=config)
def test_load_yaml_config(self):
config_file_path = os.path.join(self.tmp_dir, "config.yaml")
with open(config_file_path, 'w') as config_file:
config_file.write(yaml.dump(self.config_dict, default_flow_style=False))
self.assertEqual(self.config_dict, load_yaml_config(config_file_path))
def main(config_file):
config = load_yaml_config(config_file)
messenger = CeleryMessenger(config.get('celery'), 'ngi_pipeline')
messenger.send_message('launch_main_analysis',
'/Users/guillem/archive/test_data_empty_files')
