def get_NGIAnalysis(best_practice_analysis="sarek_germline_grch38", config=None, log=None):
project = TestLaunchers.get_NGIProject(1)
with mock.patch("ngi_pipeline.conductor.classes.CharonSession", autospec=True) as CharonSessionMock:
charon_mock = CharonSessionMock.return_value
charon_mock.project_get.return_value = {"best_practice_analysis": best_practice_analysis}
return NGIAnalysis(
project,
config=(config or TestLaunchers.CONFIG),
log=(log or minimal_logger(__name__, to_file=False, debug=True)))
def setUp(self):
self.log = minimal_logger(__name__, to_file=False, debug=True)
self.config = TestCharonConnector.CONFIG
self.project_id = "this-is-a-project-id"
self.sample_id = "this-is-a-sample-id"
self.libprep_id = "this-is-a-libprep-id"
self.libpreps = [
{"libprepid": "this-is-a-libprep-1"},
{"libprepid": "this-is-a-libprep-2"},
{"libprepid": "this-is-a-libprep-3"}]
self.seqruns = [
{"seqrunid": "this-is-a-seqrun-1"},
{"seqrunid": "this-is-a-seqrun-2"},
{"seqrunid": "this-is-a-seqrun-3"}]
def update_charon_with_local_jobs_status(
config=None, log=None, tracking_connector=None, charon_connector=None, **kwargs):
"""
Update Charon with the local changes in the SQLite tracking database.
:param config: optional dict with configuration options. If not specified, the global configuration will be used
instead
:param log: optional log instance. If not specified, a new log instance will be created
:param tracking_connector: optional connector to the tracking database. If not specified,
a new connector will be created
:param charon_connector: optional connector to the charon database. If not specified, a new connector will be
created
:param kwargs: placeholder for additional, unused options
:return: None
"""
log = log or minimal_logger(__name__, debug=True)
tracking_connector = tracking_connector or TrackingConnector(config, log)
charon_connector = charon_connector or CharonConnector(config, log)
log.debug("updating Charon status for locally tracked jobs")
# iterate over the analysis processes tracked in the local database
for analysis in tracking_connector.tracked_analyses():
try:
log.debug("checking status for analysis of {}:{} with {}:{}, having {}".format(
analysis.project_id,
analysis.sample_id,
analysis.engine,
analysis.workflow,
"pid {}".format(analysis.process_id) if analysis.process_id is not None else
"sbatch job id {}".format(analysis.slurm_job_id)))
# create an AnalysisTracker instance for the analysis
analysis_tracker = AnalysisTracker(analysis, charon_connector, tracking_connector, log, config)
# recreate the analysis_sample from disk/analysis
analysis_tracker.recreate_analysis_sample()
# poll the system for the analysis status
analysis_tracker.get_analysis_status()
# set the analysis status
analysis_tracker.report_analysis_status()
# set the analysis results
analysis_tracker.report_analysis_results()
# remove the analysis entry from the local db
analysis_tracker.remove_analysis()
# do cleanup
analysis_tracker.cleanup()
except Exception as e:
log.error("exception raised when processing sample {} in project {}, please review: {}".format(
analysis.project_id, analysis.sample_id, e))
def __init__(self, project, restart_failed_jobs=None,
restart_finished_jobs=False, restart_running_jobs=False,
keep_existing_data=False, no_qc=False, exec_mode="sbatch",
quiet=False, manual=False, config=None, config_file_path=None,
generate_bqsr_bam=False, log=None, sample=None):
self.project=project
self.sample=sample
self.restart_failed_jobs=restart_failed_jobs
self.restart_finished_jobs=restart_finished_jobs
self.restart_running_jobs=restart_running_jobs
self.keep_existing_data=keep_existing_data
self.no_qc=no_qc
self.exec_mode=exec_mode
self.quiet=quiet
self.manual=manual
self.config=config
self.config_file_path=config_file_path
self.generate_bqsr_bam=generate_bqsr_bam
self.log=log
if not log:
self.log=minimal_logger(__name__)
self.engine=self.get_engine()
from ngi_pipeline.conductor.classes import NGIProject
from ngi_pipeline.conductor.launchers import launch_analysis
from ngi_pipeline.database.classes import CharonSession, CharonError
from ngi_pipeline.database.communicate import get_project_id_from_name
from ngi_pipeline.database.filesystem import create_charon_entries_from_project
from ngi_pipeline.log.loggers import minimal_logger
from ngi_pipeline.utils.classes import with_ngi_config
from ngi_pipeline.utils.communication import mail_analysis
from ngi_pipeline.utils.filesystem import do_rsync, do_symlink, \
locate_flowcell, safe_makedir
from ngi_pipeline.utils.parsers import determine_library_prep_from_fcid, \
determine_library_prep_from_samplesheet, \
parse_lane_from_filename
LOG = minimal_logger(__name__)
UPPSALA_PROJECT_RE = re.compile(r'(\w{2}-\d{4}|\w{2}\d{2,3})')
STHLM_PROJECT_RE = re.compile(r'[A-z]{1,2}[_.][A-z0-9]+_\d{2}_\d{2}')
STHLM_X_PROJECT_RE = re.compile(r'[A-z]{1,2}_[A-z0-9]+_\d{2}_\d{2}')
## TODO we should just remove this function
def process_demultiplexed_flowcell(demux_fcid_dir_path, restrict_to_projects=None,
restrict_to_samples=None, restart_failed_jobs=False,
restart_finished_jobs=False, restart_running_jobs=False,
keep_existing_data=False, no_qc=False, quiet=False,
manual=False, config=None, config_file_path=None, generate_bqsr_bam=False):
"""Call process_demultiplexed_flowcells, restricting to a single flowcell.
Essentially a restrictive wrapper.
import gzip
import os
import re
import shlex
import subprocess
import time
import xml.etree.cElementTree as ET
import xml.parsers.expat
import json
from ngi_pipeline.database.classes import CharonSession, CharonError
from ngi_pipeline.log.loggers import minimal_logger
from ngi_pipeline.utils.classes import memoized
from six.moves import filter
LOG = minimal_logger(__name__)
## e.g. A.Wedell_13_03_P567_102_A_140528_D00415_0049_BC423WACXX <-- sthlm
## or ND-0522_NA10860_PCR-free_SX398_NA10860_PCR-free_140821_D00458_0029_AC45JGANXX <-- uusnp
STHLM_UUSNP_SEQRUN_RE = re.compile(r'(?P<project_name>\w\.\w+_\d+_\d+|\w{2}-\d+)_(?P<sample_id>[\w-]+)_(?P<libprep_id>\w|\w{2}\d{3}_\2)_(?P<seqrun_id>\d{6}_\w+_\d{4}_.{10})')
STHLM_UUSNP_SAMPLE_RE = re.compile(r'(?P<project_name>\w\.\w+_\d+_\d+|\w{2}-\d+)_(?P<sample_id>[\w-]+)')
def determine_library_prep_from_fcid(project_id, sample_name, fcid):
"""Use the information in the database to get the library prep id
from the project name, sample name, and flowcell id.
:param str project_id: The ID of the project
:param str sample_name: The name of the sample
:param str fcid: The flowcell ID
def setUp(self):
self.config = TestLaunchers.CONFIG
self.log = minimal_logger(__name__, to_file=False, debug=True)
self.analysis_object = TestLaunchers.get_NGIAnalysis(config=self.config, log=self.log)
import sys
import os
import re
import subprocess
import shutil
import click
import yaml
import vcf
import pyexcel_xlsx
from ngi_pipeline.database.classes import CharonSession, CharonError
from ngi_pipeline.log.loggers import minimal_logger
from ngi_pipeline.utils.classes import with_ngi_config
log = minimal_logger(os.path.basename(__file__))
@click.group()
@with_ngi_config
@click.pass_context
@click.option('--config',
'-c',
'custom_config',
help='Path to a config file',
type=click.Path())
def cli(context, config_file_path, config, custom_config=None):
# check first if config file is specified
if custom_config is not None:
log.info('Using custom config file: {}'.format(
os.path.abspath(custom_config)))
import sys
import os
import re
import subprocess
import shutil
import click
import yaml
import vcf
import pyexcel_xlsx
from ngi_pipeline.database.classes import CharonSession, CharonError
from ngi_pipeline.log.loggers import minimal_logger
from ngi_pipeline.utils.classes import with_ngi_config
log = minimal_logger(os.path.basename(__file__))
@click.group()
@with_ngi_config
@click.pass_context
@click.option('--config', '-c', 'custom_config', help='Path to a config file', type=click.Path())
def cli(context, config_file_path, config, custom_config=None):
# check first if config file is specified
if custom_config is not None:
log.info('Using custom config file: {}'.format(os.path.abspath(custom_config)))
if not os.path.exists(custom_config):
log.error('Config file does not exist!')
exit(1)
with open(custom_config, 'r') as config_file:
config = yaml.load(config_file) or {}
config = config.get('gt_concordance') or {}
def setUp(self):
self.log = minimal_logger(__name__, to_file=False, debug=True)
self.config = TestSarekAnalysis.CONFIG
self.analysis_obj = TestLaunchers.get_NGIAnalysis(log=self.log)
def setUpClass(cls):
cls.log = minimal_logger(__name__, debug=True)
def setUpClass(cls):
cls.log = minimal_logger(__name__, debug=True)
#!/bin/env python
import argparse
import os
from ngi_pipeline.engines.sarek import local_process_tracking
from ngi_pipeline.engines.sarek.database import TrackingConnector
from ngi_pipeline.log.loggers import minimal_logger
from ngi_pipeline.utils.classes import with_ngi_config
LOG = minimal_logger(__name__, debug=True)
class DiskTrackingSession(object):
"""
This is an object to replace the SQLAlchemy object injected into the TrackingConnector, in order to replace the
database connections
"""
def __init__(self, analyses=None):
self.analyses = analyses or list()
def add(self, db_obj):
self.analyses.append(db_obj)
def all(self, *args, **kwargs):
for analysis in self.analyses:
yield analysis
def commit(self, *args, **kwargs):
pass
"""
from __future__ import print_function
import argparse
import os
import sys
from ngi_pipeline.conductor import flowcell
from ngi_pipeline.conductor import launchers
from ngi_pipeline.conductor.flowcell import setup_analysis_directory_structure
from ngi_pipeline.database.filesystem import create_charon_entries_from_project
from ngi_pipeline.log.loggers import minimal_logger
from ngi_pipeline.server import main as server_main
from ngi_pipeline.utils.filesystem import recreate_project_from_filesystem
LOG = minimal_logger("ngi_pipeline_start")
def validate_force_update():
return _validate_dangerous_user_thing("OVERWRITE EXISTING DATA in CHARON")
def validate_delete_existing():
return _validate_dangerous_user_thing("DELETE EXISTING DATA in CHARON")
def _validate_dangerous_user_thing(action="do SOMETHING that Mario thinks you should BE WARNED about"):
print("DANGER WILL ROBINSON you have told this script to {action}!! Do you in fact want do to this?!?".format(action=action), file=sys.stderr)
attempts = 0
while True:
if attempts < 3:
attempts += 1
user_input = raw_input("Confirm overwrite by typing 'yes' or 'no' ({}): ".format(attempts)).lower()