def process(db, scout_db, mip_root):
"""Process all Scout uploads."""
cases = scout_db.cases()
total = cases.count()
with click.progressbar(cases, label='scout | cases', length=total) as bar:
for scout_case in bar:
scout_status = scout_case.status
case_id = scout_case.case_id
(cust_id, family_id) = case_id.split('_')
if cust_id == 'cust000':
continue
try:
family_dir = _get_family_dir(cust_id, family_id, mip_root)
except TooManyFamilyDirsFound:
logging.error("Too many family dirs for case {}".format(case_id))
continue
except FamilyDirNotFound:
logging.error("Couldn't find family dir for case {}".format(case_id))
continue
mip_analysis = load_analysis(family_dir)
is_new_case = scout_case.created_at > CHECKED_CUTOFF
is_checked_case = scout_case.analysis_checked == 1
is_mip_archived_case = mip_analysis.is_archived
if not is_mip_archived_case:
if scout_status in ('archived', 'solved'):
if is_new_case or is_checked_case:
dispatch(db, case_id, family_dir, cust_id)
logger.info("queued %s for archiving", case_id)
def parse_payload(db, payload, config, force=False):
"""Purge an analysis."""
family_dir = payload['family_dir']
customer_id = payload['customer_id']
mip_analysis = load_analysis(family_dir)
family_id = mip_analysis.family_id
if mip_analysis.status in ('Archived'):
targets = []
cust_dir = '/mnt/hds/proj/{}/INBOX/'.format(customer_id)
for sample_name in mip_analysis.sample_ids:
# find the file it links to in DEMUX
for mip_fastq_file in glob(family_dir + '/../' + sample_name + '/fastq/*fastq.gz'):
fastq_file = os.path.realpath(mip_fastq_file) # the actual fastq file
targets.append(mip_fastq_file)
targets.extend(_find_shackles(mip_fastq_file))
# find the sample dir in INBOX
same_files = _find_same_file(fastq_file, cust_dir)
sample_dirs = _get_sample_dir(same_files)
targets.extend(sample_dirs)
# find original fastq files if trimmed
targets.extend(_find_untrimmed_file(fastq_file))
script = _create_rm_script(targets)
script_name = "rm-{}-{}.bash".format(customer_id, family_id)
outfile = os.path.join(config['START_SCRIPTS_DIR'], script_name)
with open(outfile, 'w') as handle:
# overwrite if necessary
handle.write(script)
def analysis(family_dir):
"""Load data on an analysis.
Args:
family_dir (path): path to wrapping case folder
"""
try:
return load_analysis(family_dir)
except OSError as error:
logger.error('qc sample info file not found')
raise error
def parse_payload(db, payload, config, force=False):
"""Archive an analysis."""
hum_ref = config['MIP_HUMAN_REF_PATH']
project = config['MIP_PROJECT_ACCOUNT']
tmp_dir = config['MIP_TMP_DIR']
email = config['LOG_EMAIL']
family_dir = payload['family_dir']
mip_analysis = load_analysis(family_dir)
if mip_analysis.status not in ('Archived', 'Archiving'):
archive_mod.auto_archive(mip_analysis, hum_ref, project, tmp_dir,
email=email, force=force)
else:
logger.warn("analysis (%s) already '%s'", family_dir,
mip_analysis.status)
def traverse(roots):
"""Crawl analysis output directories.
Args:
roots (List[path]): list of customer root directories
"""
case_paths = case_roots(*roots)
for case_path in case_paths:
mip_analysis = load_analysis(case_path, insanity=True)
if mip_analysis.is_complete:
logger.debug("completed case: %s", case_path)
yield mip_analysis
else:
logger.debug("skipping non-complete case: %s", case_path)
def parse_payload(db, payload, config, force=False):
"""Upload variants for visualization."""
scout_db = ScoutDatabase(config["SCOUT_AUTH"])
mip_analysis = load_analysis(payload["family_dir"])
variant_type = payload["variant_type"]
threshold = config.get("SCOUT_VARIANT_THRESHOLD", 5000)
config_data = do_config(mip_analysis, variant_type)
del config_data["madeline"]
owner = mip_analysis.instance_tags[0]
case_loaded = scout_db.case_is_loaded(owner, mip_analysis.family_id, mip_analysis.analyzed_at, vtype=variant_type)
if force or not case_loaded:
scout_db.upload(config_data, mip_analysis.family_id, threshold=threshold)
else:
logger.info("case already uploaded, skipping")
def parse_payload(db, payload, config, force=False):
"""Upload frequency to the database."""
loqusdb_auth = config['LOQUSDB_AUTH']
adapter = LoqusdbAdapter()
adapter.connect(**loqusdb_auth)
case_id = payload['case_id']
mip_analysis = load_analysis(payload['family_dir'])
if mip_analysis.is_old:
logger.warn("unsupported MIP version for frequency: %s",
mip_analysis.version)
else:
vcf_path, affected = parse_mip(mip_analysis)
if affected:
upload(adapter, vcf_path, affected, case_id, force=force)
def parse_payload(db, payload, config, force=False):
"""Process payload for coverage."""
mip_analysis = load_analysis(payload['family_dir'])
chanjo_db = Store(config['CHANJO_URI'], base=TXBASE)
group_id = payload['case_id']
if force:
remove(chanjo_db, group_id)
if not is_loaded(group_id):
samples = parse_mip(mip_analysis)
for sample_id, bed_path in samples:
logger.info("load coverage for %s", sample_id)
upload(chanjo_db, sample_id, group_id, bed_path)
else:
logger.info("samples already loaded: %s", group_id)
def process(db, cust_roots):
"""Process all MIP output."""
family_dirs = list(automate.case_roots(*cust_roots))
analyses = (load_analysis(family_dir, insanity=True) for family_dir in
family_dirs)
total = len(family_dirs)
with click.progressbar(analyses, label='mip | analyses',
length=total) as bar:
for analysis in bar:
process_one(db, analysis)
try:
db.commit()
except (OperationalError, IntegrityError):
logger.exception('error in saving mip analysis')
db.session.rollback()
def parse_payload(db, payload, config, force=False):
"""Process an action to upload genotypes."""
taboo_db = TabooDatabase(config['TABOO_URI'])
mip_analysis = load_analysis(payload['family_dir'])
with open(config['TABOO_RSNUMBERS']) as rs_stream:
if mip_analysis.bcf_path and mip_analysis.bcf_path.exists():
analyses = load_bcf(taboo_db, mip_analysis.bcf_path, rs_stream,
force=force)
else:
analyses = load_vcf(taboo_db, mip_analysis.ready_vcf, rs_stream,
force=force)
for analysis in analyses:
logger.info("added analysis: %s", analysis.sample.sample_id)
sample_id = analysis.sample.sample_id
ped_sex = mip_analysis.ped.individuals[sample_id].sex
analysis.sample.expected_sex = SEX[ped_sex]
taboo_db.save()
def mip_analysis(family_dir):
"""Return an initialized version of a mip analysis object."""
analysis = load_analysis(family_dir)
return analysis
def running_analysis():
family_dir = ("tests/fixtures/analysis/customers/cust001/family-running/"
"exomes/family-running")
_analysis = load_analysis(family_dir)
yield _analysis
pass
def analysis():
family_dir = ("tests/fixtures/analysis/customers/cust000/15035-miptest/"
"exomes/15035-miptest")
_analysis = load_analysis(family_dir)
yield _analysis
pass
