def files_data(files_raw): """Get some data files""" return { "config": mip_dna_files_api.parse_config(files_raw["config"]), "sampleinfo": mip_dna_files_api.parse_sampleinfo(files_raw["sampleinfo"]), "qcmetrics": mip_dna_files_api.parse_qcmetrics(files_raw["qcmetrics"]), "rna_config": mip_dna_files_api.parse_config(files_raw["rna_config"]), "rna_sampleinfo": mip_rna_files_api.parse_sampleinfo_rna(files_raw["rna_sampleinfo"]), "rna_config_store": store_mip.parse_config(files_raw["rna_config_store"]), "rna_sampleinfo_store": store_mip.parse_sampleinfo(files_raw["rna_sampleinfo_store"]), }
def files_data(files_raw): return { 'config': files_api.parse_config(files_raw['config']), 'sampleinfo': files_api.parse_sampleinfo(files_raw['sampleinfo']), 'qcmetrics': files_api.parse_qcmetrics(files_raw['qcmetrics']), 'sacct': sacct.parse_sacct(files_raw['sacct']), }
def cancel(context, jobs, analysis_id): """Cancel all jobs in a run.""" analysis_obj = context.obj['store'].analysis(analysis_id) if analysis_obj is None: click.echo('analysis not found') context.abort() elif analysis_obj.status != 'running': click.echo(f"analysis not running: {analysis_obj.status}") context.abort() config_path = Path(analysis_obj.config_path) with config_path.open() as config_stream: config_raw = ruamel.yaml.safe_load(config_stream) config_data = parse_config(config_raw) log_path = Path(f"{config_data['log_path']}") if not log_path.exists(): click.echo(f"missing MIP log file: {log_path}") context.abort() with log_path.open() as log_stream: all_jobs = job_ids(log_stream) if jobs: for job_id in all_jobs: click.echo(job_id) else: for job_id in all_jobs: LOG.debug(f"cancelling job: {job_id}") process = subprocess.Popen(['scancel', job_id]) process.wait() analysis_obj.status = 'canceled' context.obj['store'].commit() click.echo('cancelled analysis successfully!')
def __call__(self, config_stream: List[str], sampleinfo: str = None, sacct: str = None): """Add a new analysis log.""" config_raw = ruamel.yaml.safe_load(config_stream) config_data = files_api.parse_config(config_raw) sampleinfo_path = Path(sampleinfo or config_data['sampleinfo_path']) if not sampleinfo_path.exists(): raise MissingFileError(sampleinfo_path) with sampleinfo_path.open() as stream: sampleinfo_raw = ruamel.yaml.safe_load(stream) sampleinfo_data = files_api.parse_sampleinfo_light(sampleinfo_raw) sacct_path = Path( sacct if sacct else f"{config_data['log_path']}.status") if not sacct_path.exists(): raise MissingFileError(sacct_path) with sacct_path.open() as stream: sacct_jobs = sacct_api.parse_sacct(stream) with Path(config_data['log_path']).open() as stream: jobs = len(miplog.job_ids(stream)) run_data = self.parse(config_data, sampleinfo_data, sacct_jobs, jobs=jobs) self._delete_temp_logs(run_data['case']) new_run = self.build(run_data) if new_run: self.store.add_commit(new_run) return new_run
def files_data(files_raw): return { 'config': files_api.parse_config(files_raw['config']), 'sampleinfo': files_api.parse_sampleinfo(files_raw['sampleinfo']), 'qcmetrics': files_api.parse_qcmetrics(files_raw['qcmetrics']), 'sacct': sacct.parse_sacct(files_raw['sacct']), }
def test_parse_config(files_raw) -> dict: """ Args: files_raw (dict): With dicts from files """ # GIVEN config data of a "sharp" run (not dry run) config_raw = files_raw['config'] # WHEN parsing the MIP output config config_data = files.parse_config(config_raw) # THEN it should work assert isinstance(config_data, dict) # THEN it should work and the dry run check should be "yes" assert config_data['is_dryrun'] is True # ... and should include all samples assert len(config_raw['analysis_type']) == len(config_data['samples']) # Build dict for return data config_test_data = { 'config_path': '/path_to/cases/case/analysis/case_config.yaml', 'email': '*****@*****.**', 'case': 'case', 'log_path': 'tests/fixtures/case/mip_2019-07-04T10:47:15.log', 'sampleinfo_path': '/path_to/cases/case/analysis/case_qc_sample_info.yaml', 'out_dir': '/path_to/cases/case/analysis', 'priority': 'normal', } # Check returns from def for key, value in config_test_data.items(): assert config_data[key] == value
def add_analysis(cls, config_stream): """Gather information from MIP analysis to store.""" config_raw = ruamel.yaml.safe_load(config_stream) config_data = mip_files.parse_config(config_raw) sampleinfo_raw = ruamel.yaml.safe_load( Path(config_data['sampleinfo_path']).open()) sampleinfo_data = mip_files.parse_sampleinfo(sampleinfo_raw) if sampleinfo_data['is_finished'] is False: raise AnalysisNotFinishedError('analysis not finished') new_bundle = cls._build_bundle(config_data, sampleinfo_data) return new_bundle
def test_parse_config(files_raw) -> dict: """ Args: files_raw (dict): With dicts from files """ # GIVEN config data of a "sharp" run (not dry run) config_raw = files_raw['config'] # WHEN parsing the MIP output config config_data = files.parse_config(config_raw) # THEN it should work assert isinstance(config_data, dict) # THEN it should work and the dry run check should be "no" assert config_data['is_dryrun'] is False # ... and should include all samples assert len(config_raw['analysis_type']) == len(config_data['samples']) # Build dict for return data config_test_data = { 'config_path': 'tests/fixtures/family/family_config.yaml', 'email': '*****@*****.**', 'family': 'family', 'log_path': 'tests/fixtures/family/mip.pl_2017-06-17T12:11:42.log', 'sampleinfo_path': ('/mnt/hds/proj/bioinfo/MIP_ANALYSIS/' + 'customers/cust003/family/analysis/' + 'family_qc_sample_info.yaml'), 'out_dir': ('/mnt/hds/proj/bioinfo/MIP_ANALYSIS/customers/' + 'cust003/family/analysis'), 'priority': 'normal', } # Check returns from def for key, value in config_test_data.items(): assert config_data[key] == value
def test_parse_config(files_raw) -> dict: """ Args: files_raw (dict): With dicts from files """ # GIVEN config data of a "sharp" run (not dry run) config_raw = files_raw['config'] # WHEN parsing the MIP output config config_data = files.parse_config(config_raw) # THEN it should work assert isinstance(config_data, dict) # THEN it should work and the dry run check should be "no" assert config_data['is_dryrun'] is False # ... and should include all samples assert len(config_raw['analysis_type']) == len(config_data['samples']) # Build dict for return data config_test_data = { 'config_path': 'tests/fixtures/family/family_config.yaml', 'email': '*****@*****.**', 'family': 'family', 'log_path': 'tests/fixtures/family/mip.pl_2017-06-17T12:11:42.log', 'sampleinfo_path': ('/mnt/hds/proj/bioinfo/MIP_ANALYSIS/' + 'customers/cust003/family/analysis/' + 'family_qc_sample_info.yaml'), 'out_dir': ('/mnt/hds/proj/bioinfo/MIP_ANALYSIS/customers/' + 'cust003/family/analysis'), 'priority': 'normal', } # Check returns from def for key, value in config_test_data.items(): assert config_data[key] == value
def __call__(self, config_stream: List[str], sampleinfo: str=None, sacct: str=None): """Add a new analysis log.""" config_raw = ruamel.yaml.safe_load(config_stream) config_data = files_api.parse_config(config_raw) sampleinfo_path = Path(sampleinfo or config_data['sampleinfo_path']) if not sampleinfo_path.exists(): raise MissingFileError(sampleinfo_path) with sampleinfo_path.open() as stream: sampleinfo_raw = ruamel.yaml.safe_load(stream) sampleinfo_data = files_api.parse_sampleinfo(sampleinfo_raw) sacct_path = Path(sacct if sacct else f"{config_data['log_path']}.status") if not sacct_path.exists(): raise MissingFileError(sacct_path) with sacct_path.open() as stream: sacct_jobs = sacct_api.parse_sacct(stream) with Path(config_data['log_path']).open() as stream: jobs = len(miplog.job_ids(stream)) run_data = self.parse(config_data, sampleinfo_data, sacct_jobs, jobs=jobs) self._delete_temp_logs(run_data['family']) new_run = self.build(run_data) if new_run: self.store.add_commit(new_run) return new_run
def get_sampleinfo(analysis: models.Analysis) -> str: """Get the sample info path for an analysis.""" raw_data = ruamel.yaml.safe_load(Path(analysis.config_path).open()) data = files.parse_config(raw_data) return data["sampleinfo_path"]
def check(context: click.Context, family: str): """Delete an analysis log from the database.""" analysis_obj = context.obj['store'].analyses(family=family).first() if analysis_obj is None: LOG.error('no analysis found') context.abort() config_path = Path(analysis_obj.config_path) if not config_path.exists(): LOG.error(f"analysis config not found: {config_path}") context.abort() config_raw = ruamel.yaml.safe_load(config_path.open()) config_data = files.parse_config(config_raw) sampleinfo_raw = ruamel.yaml.safe_load( Path(config_data['sampleinfo_path']).open()) sampleinfo_data = files.parse_sampleinfo(sampleinfo_raw) qcmetrics_path = Path(sampleinfo_data['qcmetrics_path']) if not qcmetrics_path.exists(): LOG.error(f"qc metrics not found: {str(qcmetrics_path)}") context.abort() qcmetrics_raw = ruamel.yaml.safe_load(qcmetrics_path.open()) qcmetrics_data = files.parse_qcmetrics(qcmetrics_raw) samples = { 'sample': [], 'type': [], 'ped': [], 'chanjo': [], 'peddy': [], 'plink': [], 'duplicates': [], } for sample_data in config_data['samples']: LOG.debug(f"{sample_data['id']}: parse analysis config") samples['sample'].append(sample_data['id']) samples['type'].append(sample_data['type']) for sample_data in sampleinfo_data['samples']: LOG.debug(f"{sample_data['id']}: parse sample info") samples['ped'].append(sample_data['sex']) with Path(sample_data['chanjo_sexcheck']).open() as chanjo_handle: sexcheck_data = files.parse_chanjo_sexcheck(chanjo_handle) predicted_sex = sexcheck_data['predicted_sex'] xy_ratio = sexcheck_data['y_coverage'] / sexcheck_data['x_coverage'] samples['chanjo'].append(f"{predicted_sex} ({xy_ratio:.3f})") for sample_data in qcmetrics_data['samples']: LOG.debug(f"{sample_data['id']}: parse qc metrics") samples['plink'].append(sample_data['plink_sex']) duplicates_percent = sample_data['duplicates'] * 100 samples['duplicates'].append(f"{duplicates_percent:.3f}%") peddy_path = Path(sampleinfo_data['peddy']['sex_check']) if peddy_path.exists(): with peddy_path.open() as sexcheck_handle: peddy_data = files.parse_peddy_sexcheck(sexcheck_handle) for sample_id in samples['sample']: LOG.debug(f"{sample_id}: parse peddy") predicted_sex = peddy_data[sample_id]['predicted_sex'] het_ratio = peddy_data[sample_id]['het_ratio'] samples['peddy'].append(f"{predicted_sex} ({het_ratio})") else: LOG.warning(f"missing peddy output: {peddy_path}") print(tabulate(samples, headers='keys', tablefmt='psql'))
def _parse_raw_mip_config_into_dict(mip_config_raw): config_data = files.parse_config(mip_config_raw) return config_data
def check(context: click.Context, family: str): """Delete an analysis log from the database.""" analysis_obj = context.obj['store'].analyses(family=family).first() if analysis_obj is None: LOG.error('no analysis found') context.abort() config_path = Path(analysis_obj.config_path) if not config_path.exists(): LOG.error(f"analysis config not found: {config_path}") context.abort() config_raw = ruamel.yaml.safe_load(config_path.open()) config_data = files.parse_config(config_raw) sampleinfo_raw = ruamel.yaml.safe_load(Path(config_data['sampleinfo_path']).open()) sampleinfo_data = files.parse_sampleinfo(sampleinfo_raw) qcmetrics_path = Path(sampleinfo_data['qcmetrics_path']) if not qcmetrics_path.exists(): LOG.error(f"qc metrics not found: {str(qcmetrics_path)}") context.abort() qcmetrics_raw = ruamel.yaml.safe_load(qcmetrics_path.open()) qcmetrics_data = files.parse_qcmetrics(qcmetrics_raw) samples = { 'sample': [], 'type': [], 'ped': [], 'chanjo': [], 'peddy': [], 'plink': [], 'duplicates': [], } for sample_data in config_data['samples']: LOG.debug(f"{sample_data['id']}: parse analysis config") samples['sample'].append(sample_data['id']) samples['type'].append(sample_data['type']) for sample_data in sampleinfo_data['samples']: LOG.debug(f"{sample_data['id']}: parse sample info") samples['ped'].append(sample_data['sex']) with Path(sample_data['chanjo_sexcheck']).open() as chanjo_handle: sexcheck_data = files.parse_chanjo_sexcheck(chanjo_handle) predicted_sex = sexcheck_data['predicted_sex'] xy_ratio = sexcheck_data['y_coverage'] / sexcheck_data['x_coverage'] samples['chanjo'].append(f"{predicted_sex} ({xy_ratio:.3f})") for sample_data in qcmetrics_data['samples']: LOG.debug(f"{sample_data['id']}: parse qc metrics") samples['plink'].append(sample_data['plink_sex']) duplicates_percent = sample_data['duplicates'] * 100 samples['duplicates'].append(f"{duplicates_percent:.3f}%") peddy_path = Path(sampleinfo_data['peddy']['sex_check']) if peddy_path.exists(): with peddy_path.open() as sexcheck_handle: peddy_data = files.parse_peddy_sexcheck(sexcheck_handle) for sample_id in samples['sample']: LOG.debug(f"{sample_id}: parse peddy") predicted_sex = peddy_data[sample_id]['predicted_sex'] het_ratio = peddy_data[sample_id]['het_ratio'] samples['peddy'].append(f"{predicted_sex} ({het_ratio})") else: LOG.warning(f"missing peddy output: {peddy_path}") print(tabulate(samples, headers='keys', tablefmt='psql'))
def _parse_raw_mip_config_into_dict(mip_config_raw): config_data = files.parse_config(mip_config_raw) return config_data