def process(db, scout_db, mip_root): """Process all Scout uploads.""" cases = scout_db.cases() total = cases.count() with click.progressbar(cases, label='scout | cases', length=total) as bar: for scout_case in bar: scout_status = scout_case.status case_id = scout_case.case_id (cust_id, family_id) = case_id.split('_') if cust_id == 'cust000': continue try: family_dir = _get_family_dir(cust_id, family_id, mip_root) except TooManyFamilyDirsFound: logging.error("Too many family dirs for case {}".format(case_id)) continue except FamilyDirNotFound: logging.error("Couldn't find family dir for case {}".format(case_id)) continue mip_analysis = load_analysis(family_dir) is_new_case = scout_case.created_at > CHECKED_CUTOFF is_checked_case = scout_case.analysis_checked == 1 is_mip_archived_case = mip_analysis.is_archived if not is_mip_archived_case: if scout_status in ('archived', 'solved'): if is_new_case or is_checked_case: dispatch(db, case_id, family_dir, cust_id) logger.info("queued %s for archiving", case_id)
def parse_payload(db, payload, config, force=False): """Purge an analysis.""" family_dir = payload['family_dir'] customer_id = payload['customer_id'] mip_analysis = load_analysis(family_dir) family_id = mip_analysis.family_id if mip_analysis.status in ('Archived'): targets = [] cust_dir = '/mnt/hds/proj/{}/INBOX/'.format(customer_id) for sample_name in mip_analysis.sample_ids: # find the file it links to in DEMUX for mip_fastq_file in glob(family_dir + '/../' + sample_name + '/fastq/*fastq.gz'): fastq_file = os.path.realpath(mip_fastq_file) # the actual fastq file targets.append(mip_fastq_file) targets.extend(_find_shackles(mip_fastq_file)) # find the sample dir in INBOX same_files = _find_same_file(fastq_file, cust_dir) sample_dirs = _get_sample_dir(same_files) targets.extend(sample_dirs) # find original fastq files if trimmed targets.extend(_find_untrimmed_file(fastq_file)) script = _create_rm_script(targets) script_name = "rm-{}-{}.bash".format(customer_id, family_id) outfile = os.path.join(config['START_SCRIPTS_DIR'], script_name) with open(outfile, 'w') as handle: # overwrite if necessary handle.write(script)
def analysis(family_dir): """Load data on an analysis. Args: family_dir (path): path to wrapping case folder """ try: return load_analysis(family_dir) except OSError as error: logger.error('qc sample info file not found') raise error
def parse_payload(db, payload, config, force=False): """Archive an analysis.""" hum_ref = config['MIP_HUMAN_REF_PATH'] project = config['MIP_PROJECT_ACCOUNT'] tmp_dir = config['MIP_TMP_DIR'] email = config['LOG_EMAIL'] family_dir = payload['family_dir'] mip_analysis = load_analysis(family_dir) if mip_analysis.status not in ('Archived', 'Archiving'): archive_mod.auto_archive(mip_analysis, hum_ref, project, tmp_dir, email=email, force=force) else: logger.warn("analysis (%s) already '%s'", family_dir, mip_analysis.status)
def traverse(roots): """Crawl analysis output directories. Args: roots (List[path]): list of customer root directories """ case_paths = case_roots(*roots) for case_path in case_paths: mip_analysis = load_analysis(case_path, insanity=True) if mip_analysis.is_complete: logger.debug("completed case: %s", case_path) yield mip_analysis else: logger.debug("skipping non-complete case: %s", case_path)
def parse_payload(db, payload, config, force=False): """Upload variants for visualization.""" scout_db = ScoutDatabase(config["SCOUT_AUTH"]) mip_analysis = load_analysis(payload["family_dir"]) variant_type = payload["variant_type"] threshold = config.get("SCOUT_VARIANT_THRESHOLD", 5000) config_data = do_config(mip_analysis, variant_type) del config_data["madeline"] owner = mip_analysis.instance_tags[0] case_loaded = scout_db.case_is_loaded(owner, mip_analysis.family_id, mip_analysis.analyzed_at, vtype=variant_type) if force or not case_loaded: scout_db.upload(config_data, mip_analysis.family_id, threshold=threshold) else: logger.info("case already uploaded, skipping")
def parse_payload(db, payload, config, force=False): """Upload frequency to the database.""" loqusdb_auth = config['LOQUSDB_AUTH'] adapter = LoqusdbAdapter() adapter.connect(**loqusdb_auth) case_id = payload['case_id'] mip_analysis = load_analysis(payload['family_dir']) if mip_analysis.is_old: logger.warn("unsupported MIP version for frequency: %s", mip_analysis.version) else: vcf_path, affected = parse_mip(mip_analysis) if affected: upload(adapter, vcf_path, affected, case_id, force=force)
def parse_payload(db, payload, config, force=False): """Process payload for coverage.""" mip_analysis = load_analysis(payload['family_dir']) chanjo_db = Store(config['CHANJO_URI'], base=TXBASE) group_id = payload['case_id'] if force: remove(chanjo_db, group_id) if not is_loaded(group_id): samples = parse_mip(mip_analysis) for sample_id, bed_path in samples: logger.info("load coverage for %s", sample_id) upload(chanjo_db, sample_id, group_id, bed_path) else: logger.info("samples already loaded: %s", group_id)
def process(db, cust_roots): """Process all MIP output.""" family_dirs = list(automate.case_roots(*cust_roots)) analyses = (load_analysis(family_dir, insanity=True) for family_dir in family_dirs) total = len(family_dirs) with click.progressbar(analyses, label='mip | analyses', length=total) as bar: for analysis in bar: process_one(db, analysis) try: db.commit() except (OperationalError, IntegrityError): logger.exception('error in saving mip analysis') db.session.rollback()
def parse_payload(db, payload, config, force=False): """Process an action to upload genotypes.""" taboo_db = TabooDatabase(config['TABOO_URI']) mip_analysis = load_analysis(payload['family_dir']) with open(config['TABOO_RSNUMBERS']) as rs_stream: if mip_analysis.bcf_path and mip_analysis.bcf_path.exists(): analyses = load_bcf(taboo_db, mip_analysis.bcf_path, rs_stream, force=force) else: analyses = load_vcf(taboo_db, mip_analysis.ready_vcf, rs_stream, force=force) for analysis in analyses: logger.info("added analysis: %s", analysis.sample.sample_id) sample_id = analysis.sample.sample_id ped_sex = mip_analysis.ped.individuals[sample_id].sex analysis.sample.expected_sex = SEX[ped_sex] taboo_db.save()
def mip_analysis(family_dir): """Return an initialized version of a mip analysis object.""" analysis = load_analysis(family_dir) return analysis
def running_analysis(): family_dir = ("tests/fixtures/analysis/customers/cust001/family-running/" "exomes/family-running") _analysis = load_analysis(family_dir) yield _analysis pass
def analysis(): family_dir = ("tests/fixtures/analysis/customers/cust000/15035-miptest/" "exomes/15035-miptest") _analysis = load_analysis(family_dir) yield _analysis pass