def generate_krona():
"""Generate Krona plot with all the results via Krona 2.0 XML spec"""
print(gray('\nBuilding the taxonomy multiple tree... '), end='')
sys.stdout.flush()
krona: KronaTree = KronaTree(
samples,
num_raw_samples=len(raw_samples),
stats=stats,
min_score=Score(
min([
min(scores[sample].values()) for sample in samples
if len(scores[sample])
])),
max_score=Score(
max([
max(scores[sample].values()) for sample in samples
if len(scores[sample])
])),
scoring=scoring,
)
polytree.grow(ontology=ncbi,
abundances=counts,
accs=accs,
scores=scores)
print(green('OK!'))
print(gray('Generating final plot (') + magenta(htmlfile) +
gray(')... '),
end='')
sys.stdout.flush()
polytree.toxml(ontology=ncbi, krona=krona)
krona.tohtml(htmlfile, pretty=False)
print(green('OK!'))
def check_debug():
"""Check debugging mode"""
if args.debug:
print(blue('INFO:'), gray('Debugging mode activated'))
print(blue('INFO:'), gray('Active parameters:'))
for key, value in vars(args).items():
if value:
print(gray(f'\t{key} ='), f'{value}')
def check_controls():
"""Check and info about the control samples"""
if args.controls:
if args.controls > len(input_files):
print(red(' ERROR!'), gray('More controls than samples'))
exit(1)
print(gray('Control(s) sample(s) for subtractions:'))
for i in range(args.controls):
print(blue(f'\t{input_files[i]}'))
def summarize_analysis(
*args,
**kwargs) -> Tuple[Optional[Sample], Counter[Id], Counter[Id], Scores]:
"""
Summarize for a cross-analysis (to be usually called in parallel!).
"""
# Recover input and parameters
analysis: str = args[0]
ontology: Ontology = kwargs['ontology']
# TODO: Delete the following comment lines in a future release
# including = ontology.including # See comment below for the reason
# excluding = ontology.excluding # in/excluding are not used anymore
counts: Dict[Sample, Counter[Id]] = kwargs['counts']
scores: Dict[Sample, Dict[Id, Score]] = kwargs['scores']
samples: List[Sample] = kwargs['samples']
output: io.StringIO = io.StringIO(newline='')
# Declare/define variables
summary_counts: Counter[Id] = col.Counter()
summary_acc: Counter[Id] = col.Counter()
summary_score: Scores = Scores({})
summary: Optional[Sample] = None
output.write(gray('Summary for ') + analysis + gray('... '))
target_samples: List[Sample] = [
smpl for smpl in samples if smpl.startswith(analysis)
]
assert len(target_samples) >= 1, \
red('ERROR! ') + analysis + gray(' has no samples to summarize!')
for smpl in target_samples:
summary_counts += counts[smpl]
summary_score.update(scores[smpl])
tree = TaxTree()
tree.grow(ontology=ontology, counts=summary_counts, scores=summary_score)
tree.subtract()
tree.shape()
summary_counts.clear()
summary_score.clear()
# Avoid including/excluding here as get_taxa is not as 'clever' as allin1
# and taxa are already included/excluded in the derived samples
tree.get_taxa(counts=summary_counts,
accs=summary_acc,
scores=summary_score)
summary_counts = +summary_counts # remove counts <= 0
if summary_counts: # Avoid returning empty sample (summary would be None)
summary = Sample(f'{analysis}_{STR_SUMMARY}')
output.write(
gray('(') + cyan(f'{len(target_samples)}') + gray(' samples)') +
green(' OK!\n'))
else:
output.write(yellow(' VOID\n'))
# Print output and return
print(output.getvalue(), end='')
sys.stdout.flush()
return summary, summary_counts, summary_acc, summary_score
def summarize_analysis(
*args, **kwargs) -> Tuple[Sample, Counter[Id], Counter[Id], Scores]:
"""
Summarize for a cross-analysis (to be usually called in parallel!).
"""
# Recover input and parameters
analysis: str = args[0]
ontology: Ontology = kwargs['ontology']
including = ontology.including
excluding = ontology.excluding
counts: Dict[Sample, Counter[Id]] = kwargs['counts']
scores: Dict[Sample, Dict[Id, Score]] = kwargs['scores']
samples: List[Sample] = kwargs['samples']
output: io.StringIO = io.StringIO(newline='')
# Declare/define variables
summary_counts: Counter[Id] = Counter()
summary_acc: Counter[Id] = Counter()
summary_score: Scores = Scores({})
summary: Sample = None
output.write(gray('Summary for ') + analysis + gray('... '))
target_samples: List[Sample] = [
smpl for smpl in samples if smpl.startswith(analysis)
]
assert len(target_samples) >= 1, \
red('ERROR! ') + analysis + gray(' has no samples to summarize!')
for smpl in target_samples:
summary_counts += counts[smpl]
summary_score.update(scores[smpl])
tree = TaxTree()
tree.grow(ontology=ontology, counts=summary_counts, scores=summary_score)
tree.subtract()
tree.shape()
summary_counts.clear()
summary_score.clear()
tree.get_taxa(counts=summary_counts,
accs=summary_acc,
scores=summary_score,
include=including,
exclude=excluding)
summary_counts = +summary_counts # remove counts <= 0
if summary_counts: # Avoid returning empty sample (summary would be None)
summary = Sample(f'{analysis}_{STR_SUMMARY}')
output.write(
gray('(') + cyan(f'{len(target_samples)}') + gray(' samples)') +
green(' OK!\n'))
else:
output.write(yellow(' VOID\n'))
# Print output and return
print(output.getvalue(), end='')
sys.stdout.flush()
return summary, summary_counts, summary_acc, summary_score
def summarize_samples():
"""Summary of samples in parallel by type of cross-analysis"""
# timing initialization
summ_start_time: float = time.perf_counter()
print(gray('Please, wait. Generating summaries in parallel...'))
# Update kwargs with more parameters for the followings func calls
kwargs.update({'samples': samples})
# Get list of set of samples to summarize (note pylint bug #776)
# pylint: disable=unsubscriptable-object
target_analysis: col.OrderedDict[str, None] = col.OrderedDict({
f'{raw}_{study}': None
for study in [STR_EXCLUSIVE, STR_CONTROL] for raw in raw_samples
for smpl in samples if smpl.startswith(f'{raw}_{study}')
})
# pylint: enable=unsubscriptable-object
# Add shared and control_shared analysis if they exist (are not void)
for study in [STR_SHARED, STR_CONTROL_SHARED]:
for smpl in samples:
if smpl.startswith(study):
target_analysis[study] = None
break
if platform.system() and not args.sequential: # Only for known systems
mpctx = mp.get_context('fork')
with mpctx.Pool(
processes=min(os.cpu_count(), len(input_files))) as pool:
async_results = [
pool.apply_async(summarize_analysis,
args=[analysis],
kwds=kwargs)
for analysis in target_analysis
]
for analysis, (summary, abund, acc,
score) in zip(target_analysis,
[r.get() for r in async_results]):
if summary: # Avoid adding empty samples
summaries.append(summary)
counts[summary] = abund
accs[summary] = acc
scores[summary] = score
else: # sequential processing of each selected rank
for analysis in target_analysis:
(summary, abund, acc,
score) = summarize_analysis(analysis, **kwargs)
if summary: # Avoid adding empty samples
summaries.append(summary)
counts[summary] = abund
accs[summary] = acc
scores[summary] = score
# Timing results
print(gray('Summary elapsed time:'),
f'{time.perf_counter() - summ_start_time:.3g}', gray('sec'))
def read_mock_files(mock: Filename) -> Counter[Id]:
"""Read a mock layout (.mck) file"""
mock_layout: Counter[Id] = col.Counter()
with open(mock, 'r') as file:
vprint(gray('\nProcessing'), blue(mock), gray('file:\n'))
for line in file:
if line.startswith('#'):
continue
_tid, _num = line.split('\t')
tid = Id(_tid)
num = int(_num)
mock_layout[tid] = num
vprint(num, gray('\treads for taxid\t'), tid, '\t(',
cyan(ncbi.get_name(tid)), ')\n')
return mock_layout
def _debug_dummy_plot(
taxonomy: Taxonomy,
htmlfile: Filename,
scoring: Scoring = Scoring.SHEL,
):
"""
Generate dummy Krona plot via Krona 2.0 XML spec and exit
"""
print(gray(f'Generating dummy Krona plot {htmlfile}...'), end='')
sys.stdout.flush()
samples: List[Sample] = [
Sample('SINGLE'),
]
krona: KronaTree = KronaTree(
samples,
min_score=Score(35),
max_score=Score(100),
scoring=scoring,
)
polytree: MultiTree = MultiTree(samples=samples)
polytree.grow(ontology=taxonomy)
polytree.toxml(ontology=taxonomy, krona=krona)
krona.tohtml(htmlfile, pretty=True)
print(green('OK!'))
def read_report(report_file: str) -> Tuple[str, Counter[Id], Dict[Id, Rank]]:
"""
Read Centrifuge/Kraken report file
Args:
report_file: report file name
Returns:
log string, abundances counter, taxlevel dict
"""
# TODO: Discontinued method, to be erased in a future release
output: io.StringIO = io.StringIO(newline='')
abundances: Counter[Id] = col.Counter()
level_dic = {}
output.write(f'\033[90mLoading report file {report_file}...\033[0m')
try:
with open(report_file, 'r') as file:
for report_line in file:
_, _, taxnum, taxlev, _tid, _ = report_line.split('\t')
tid = Id(_tid)
abundances[tid] = int(taxnum)
level_dic[tid] = Rank.centrifuge(taxlev)
except KeyboardInterrupt:
print(gray(' User'), yellow('interrupted!'))
raise
except Exception:
print(red('ERROR!'), 'Cannot read "' + report_file + '"')
raise
else:
output.write('\033[92m OK! \033[0m\n')
return output.getvalue(), abundances, level_dic
def shared_ctrl_analysis():
"""Perform last steps of shared taxa analysis"""
shared_ctrl_tree: TaxTree = TaxTree()
shared_ctrl_out: SampleDataById = SampleDataById(
['shared', 'accs'])
shared_ctrl_tree.allin1(ontology=ontology,
counts=shared_ctrl_counts,
scores=shared_ctrl_score,
min_taxa=get_shared_mintaxa(),
include=including,
exclude=(exclude_candidates -
shared_crossover),
out=shared_ctrl_out)
shared_ctrl_out.purge_counters()
out_counts: SharedCounter = shared_ctrl_out.get_shared_counts()
output.write(
gray(f' Ctrl-shared: Including {len(out_counts)}'
' shared taxa. Generating sample... '))
if out_counts:
sample = Sample(f'{STR_CONTROL_SHARED}_{rank.name.lower()}')
samples.append(sample)
counts[Sample(sample)] = out_counts
accs[Sample(sample)] = shared_ctrl_out.get_accs()
scores[sample] = shared_ctrl_out.get_shared_scores()
output.write(green('OK!\n'))
else:
output.write(yellow('VOID\n'))
def analyze_samples():
"""Cross analysis of samples in parallel by taxlevel"""
print(gray('Please, wait. Performing cross analysis in parallel...\n'))
# Update kwargs with more parameters for the followings func calls
kwargs.update({
'taxids': taxids,
'counts': counts,
'scores': scores,
'accs': accs,
'raw_samples': raw_samples
})
if platform.system() and not args.sequential: # Only for known systems
mpctx = mp.get_context('fork') # Important for OSX&Win
with mpctx.Pool(processes=min(os.cpu_count(),
len(Rank.selected_ranks))) as pool:
async_results = [
pool.apply_async(process_rank, args=[level], kwds=kwargs)
for level in Rank.selected_ranks
]
for level, (smpls, abunds, accumulators,
score) in zip(Rank.selected_ranks,
[r.get() for r in async_results]):
samples.extend(smpls)
counts.update(abunds)
accs.update(accumulators)
scores.update(score)
else: # sequential processing of each selected rank
for level in Rank.selected_ranks:
(smpls, abunds, accumulators,
score) = process_rank(level, **kwargs)
samples.extend(smpls)
counts.update(abunds)
accs.update(accumulators)
scores.update(score)
def mock_from_scratch(out: Filename, mock_layout: Counter[TaxId]) -> None:
"""Generate a mock Centrifuge output file from scratch"""
with open(out, 'w') as fout:
vprint(gray('Generating'), blue(out), gray('file... '))
fout.write('readID\tseqID\ttaxID\tscore\t2ndBestScore\t'
'hitLength\tqueryLength\tnumMatches\n')
reads_writen: int = 0
for tid in mock_layout:
maxhl: int = random.randint(args.random + 1, MAX_HIT_LENGTH)
rank: str = str(ncbi.get_rank(tid)).lower()
for _ in range(mock_layout[tid]):
hit_length = random.randint(args.random + 1, maxhl)
fout.write(f'test{reads_writen}\t{rank}\t'
f'{tid}\t{(hit_length-15)**2}\t'
f'0\t{hit_length}\t{MAX_HIT_LENGTH}\t1\n')
reads_writen += 1
vprint(reads_writen, 'reads', green('OK!\n'))
def select_lmat_inputs(lmats: List[Filename]) -> None:
""""LMAT files processing specific stuff"""
if lmats == ['.']:
lmats.clear()
with os.scandir() as dir_entry:
for entry in dir_entry:
if not entry.name.startswith('.') and entry.is_dir():
if entry.name != os.path.basename(TAXDUMP_PATH):
lmats.append(Filename(entry.name))
lmats.sort()
print(gray('LMAT subdirs to analyze:'), lmats)
def mock_from_scratch(out: Filename, mock_layout: Counter[Id]) -> None:
"""Generate a mock Centrifuge output file from scratch"""
with open(out, 'w') as fout:
vprint(gray('Generating'), blue(out), gray('file... '))
fout.write('readID\tseqID\ttaxID\tscore\t2ndBestScore\t'
'hitLength\tqueryLength\tnumMatches\n')
reads_writen: int = 0
for numtid in mock_layout:
tid = Id(numtid) # Convert to Id the excel integer
maxhl: int = random.randint(rnd + 1, MAX_HIT_LENGTH)
rank: str = str(ncbi.get_rank(tid)).lower()
for _ in range(int(mock_layout[numtid])):
hit_length = random.randint(rnd + 1, maxhl)
fout.write(f'test{reads_writen}\t{rank}\t'
f'{tid}\t{(hit_length - 15) ** 2}\t'
f'0\t{hit_length}\t{MAX_HIT_LENGTH}\t1\n')
reads_writen += 1
vprint(reads_writen, 'reads', green('OK!\n'))
if out == TEST_REXT_SMPL: # Test mode: create mock FASTQ for smpl
mock_fastq(reads_writen)
def mock_fastq(num_reads: int) -> None:
"""Do the job in case of Excel file with all the details"""
def fastq_seqs(alphabet=single_letter_alphabet):
"""Generator function that creates mock fastq sequences
"""
for seq in range(num_reads):
yield SeqRecord(Seq('AGTC', alphabet),
id=f'test{seq}',
name=f'test{seq}',
description=f'test{seq}',
annotations={'quality': '@@@@'})
print(gray('Writing'),
magenta(f'{num_reads}'),
gray('reads in'),
TEST_REXT_FSTQ,
gray('...'),
end='',
flush=True)
SeqIO.write((sq for sq in fastq_seqs()), TEST_REXT_FSTQ, 'quickfastq')
print(green(' OK!'))
def mock_from_source(out: Filename, mock_layout: Counter[Id]) -> None:
"""Generate a mock Centrifuge output file from source file"""
with open(out, 'w') as fout, open(args.file) as fcfg:
vprint(gray('Generating'), blue(out), gray('file... '))
fout.write(fcfg.readline()) # copy cfg output file header
reads_writen: int = 0
for line in fcfg:
tid = Id(line.split('\t')[2])
if mock_layout[tid]:
fout.write(line)
mock_layout[tid] -= 1
reads_writen += 1
if not sum(mock_layout.values()):
vprint(reads_writen, 'reads', green('OK!\n'))
break
if sum(mock_layout.values()):
print(red('ERROR!\n'))
print(gray('Incomplete read copy by taxid:'))
mock_layout = +mock_layout # Delete zero counts elements
for tid in mock_layout:
print(yellow(mock_layout[tid]), gray('reads missing for tid'),
tid, '(', cyan(ncbi.get_name(tid)), ')\n')
def select_kraken_inputs(krakens: List[Filename], ext: str = '.krk') -> None:
"""Search for Kraken files to analyze"""
dir_name = krakens[0]
krakens.clear()
with os.scandir(dir_name) as dir_entry:
for fil in dir_entry:
if not fil.name.startswith('.') and fil.name.endswith(ext):
if dir_name != '.':
krakens.append(Filename(os.path.join(dir_name, fil.name)))
else: # Avoid sample names starting with just the dot
krakens.append(Filename(fil.name))
krakens.sort()
print(gray(f'Kraken {ext} files to analyze:'), krakens)
def select_clark_inputs(clarks: List[Filename], ext: str = '.csv') -> None:
"""Search for CLARK, CLARK-l, CLARK-S files to analyze"""
dir_name = clarks[0]
clarks.clear()
with os.scandir(dir_name) as dir_entry:
for fil in dir_entry:
if not fil.name.startswith('.') and fil.name.endswith(ext):
if dir_name != '.':
clarks.append(Filename(os.path.join(dir_name, fil.name)))
else: # Avoid sample names starting with just the dot
clarks.append(Filename(fil.name))
clarks.sort()
print(gray(f'CLARK {ext} files to analyze:'), clarks)
def select_centrifuge_inputs(outputs: List[Filename],
ext: str = '.out') -> None:
"""Centrifuge output files processing specific stuff"""
dir_name = outputs[0]
outputs.clear()
with os.scandir(dir_name) as dir_entry:
for fil in dir_entry:
if not fil.name.startswith('.') and fil.name.endswith(ext):
if dir_name != '.':
outputs.append(Filename(os.path.join(dir_name, fil.name)))
else: # Avoid sample names starting with just the dot
outputs.append(Filename(fil.name))
outputs.sort()
print(gray(f'Centrifuge {ext} files to analyze:'), outputs)
def read_samples():
"""Read samples"""
print(gray('\nPlease, wait, processing files in parallel...\n'))
# Enable parallelization with 'spawn' under known platforms
if platform.system() and not args.sequential: # Only for known systems
mpctx = mp.get_context('fork')
with mpctx.Pool(
processes=min(os.cpu_count(), len(input_files))) as pool:
async_results = [
pool.apply_async(
process,
args=[
input_files[num], # file name
True if num < args.controls else False
], # is ctrl?
kwds=kwargs) for num in range(len(input_files))
]
for file, (sample, tree, out, stat,
err) in zip(input_files,
[r.get() for r in async_results]):
if err is Err.NO_ERROR:
samples.append(sample)
trees[sample] = tree
taxids[sample] = out.get_taxlevels()
counts[sample] = out.counts
accs[sample] = out.accs
scores[sample] = out.scores
stats[sample] = stat
elif err is Err.VOID_CTRL:
print('There were void controls.', red('Aborting!'))
exit(1)
else: # sequential processing of each sample
for num, file in enumerate(input_files):
(sample, tree, out, stat,
err) = process(file, True if num < args.controls else False,
**kwargs)
if err is Err.NO_ERROR:
samples.append(sample)
trees[sample] = tree
taxids[sample] = out.get_taxlevels()
counts[sample] = out.counts
accs[sample] = out.accs
scores[sample] = out.scores
stats[sample] = stat
elif err is Err.VOID_CTRL:
print('There were void controls.', red('Aborting!'))
exit(1)
raw_samples.extend(samples) # Store raw sample names
def by_excel_file() -> None:
"""Do the job in case of Excel file with all the details"""
dirname = os.path.dirname(args.xcel)
# Expected index (taxids) in column after taxa name, and last row will
# be removed (reserved for sum of reads in Excel file)
mock_df = pd.read_excel(args.xcel, index_col=1, skip_footer=1)
del mock_df['RECENTRIFUGE MOCK']
vprint(gray('Layout to generate the mock files:\n'), mock_df, '\n')
for name, series in mock_df.iteritems():
mock_layout: Counter[TaxId] = col.Counter(series.to_dict(dict))
# In prev, series.to_dict(col.Counter) fails, so this is workaround
test: Filename = Filename(os.path.join(dirname, name + '.out'))
if args.file:
mock_from_source(test, mock_layout)
else:
mock_from_scratch(test, mock_layout)
def robust_contamination_removal():
"""Implement robust contamination removal algorithm."""
nonlocal exclude_sets, shared_crossover
def compute_qn(data: List[float], dist: str = "Gauss") -> float:
"""Compute Qn robust estimator of scale (Rousseeuw, 1993)"""
c_d: float # Select d parameter depending on the distribution
if dist == "Gauss":
c_d = 2.2219
elif dist == "Cauchy": # Heavy-tailed distribution
c_d = 1.2071
elif dist == "NegExp": # Negative exponential (asymetric)
c_d = 3.4760
else:
raise Exception(red('\nERROR! ') + 'Unknown distribution')
num: int = len(data)
sort_data = sorted(data)
pairwisedifs: List[float] = []
for (i, x_val) in enumerate(sort_data):
for y_val in sort_data[i + 1:]:
pairwisedifs.append(abs(x_val - y_val))
k: int = int(num * (num / 2 + 1) / 4)
return c_d * sorted(pairwisedifs)[k - 1]
exclude_sets = {smpl: set() for smpl in raws[controls:]}
vwrite(
gray('Robust contamination removal: '
'Searching for contaminants...\n'))
for tid in exclude_candidates:
relfreq_ctrl: List[float] = [
accs[ctrl][tid] / accs[ctrl][ontology.ROOT]
for ctrl in raws[:controls]
]
relfreq_smpl: List[float] = [
accs[smpl][tid] / accs[smpl][ontology.ROOT]
for smpl in raws[controls:]
]
relfreq: List[float] = relfreq_ctrl + relfreq_smpl
crossover: List[bool] # Crossover source (yes/no)
# Just-controls contamination check
if all([rf < EPS for rf in relfreq_smpl]):
vwrite(cyan('just-ctrl:\t'), tid, ontology.get_name(tid),
gray('relfreq:'),
fltlst2str(relfreq_ctrl) + fltlst2str(relfreq_smpl),
'\n')
continue # Go for next candidate
# Critical contamination check
if all([rf > SEVR_CONTM_MIN_RELFREQ for rf in relfreq_ctrl]):
vwrite(red('critical:\t'), tid, ontology.get_name(tid),
gray('relfreq:'),
fltlst2str(relfreq_ctrl) + fltlst2str(relfreq_smpl),
'\n')
for exclude_set in exclude_sets.values():
exclude_set.add(tid)
continue # Go for next candidate
# Severe contamination check
if any([rf > SEVR_CONTM_MIN_RELFREQ for rf in relfreq_ctrl]):
vwrite(yellow('severe: \t'), tid, ontology.get_name(tid),
gray('relfreq:'),
fltlst2str(relfreq_ctrl) + fltlst2str(relfreq_smpl),
'\n')
for exclude_set in exclude_sets.values():
exclude_set.add(tid)
continue # Go for next candidate
# Mild contamination check
if all([rf > MILD_CONTM_MIN_RELFREQ for rf in relfreq_ctrl]):
vwrite(blue('mild cont:\t'), tid, ontology.get_name(tid),
gray('relfreq:'),
fltlst2str(relfreq_ctrl) + fltlst2str(relfreq_smpl),
'\n')
for exclude_set in exclude_sets.values():
exclude_set.add(tid)
continue # Go for next candidate
# Calculate median and MAD median but including controls
mdn: float = statistics.median(relfreq)
# mad:float=statistics.mean([abs(mdn - rf) for rf in relfreq])
q_n: float = compute_qn(relfreq, dist="NegExp")
# Calculate crossover in samples
outlier_lim: float = mdn + ROBUST_XOVER_OUTLIER * q_n
ordomag_lim: float = max(
relfreq_ctrl) * 10**ROBUST_XOVER_ORD_MAG
crossover = [
rf > outlier_lim and rf > ordomag_lim
for rf in relfreq[controls:]
]
# Crossover contamination check
if any(crossover):
vwrite(
magenta('crossover:\t'), tid, ontology.get_name(tid),
green(f'lims: [{outlier_lim:.1g}]' +
('<' if outlier_lim < ordomag_lim else '>') +
f'[{ordomag_lim:.1g}]'), gray('relfreq:'),
fltlst2str(relfreq_ctrl) + fltlst2str(relfreq_smpl),
gray('crossover:'), blst2str(crossover), '\n')
# Exclude just for contaminated samples (not the source)
vwrite(magenta('\t->'), gray(f'Include {tid} just in:'))
for i in range(len(raws[controls:])):
if not crossover[i]:
exclude_sets[raws[i + controls]].add(tid)
else:
vwrite(f' {raws[i + controls]}')
if all(crossover): # Shared taxon contaminating control(s)
vwrite(' (', yellow('Shared crossover taxon!'), ')')
shared_crossover.add(tid)
vwrite('\n')
continue
# Other contamination: remove from all samples
vwrite(
gray('other cont:\t'), tid, ontology.get_name(tid),
green(f'lims: [{outlier_lim:.1g}]' +
('<' if outlier_lim < ordomag_lim else '>') +
f'[{ordomag_lim:.1g}]'), gray('relfreq:'),
fltlst2str(relfreq_ctrl) + fltlst2str(relfreq_smpl), '\n')
for exclude_set in exclude_sets.values():
exclude_set.add(tid)
def read_clark_output(
output_file: Filename,
scoring: Scoring = Scoring.CLARK_C,
minscore: Score = None,
) -> Tuple[str, SampleStats, Counter[Id], Dict[Id, Score]]:
"""
Read CLARK(-l)(-S) full mode output file
Args:
output_file: output file name
scoring: type of scoring to be applied (see Scoring class)
minscore: minimum confidence level for the classification
Returns:
log string, statistics, abundances counter, scores dict
"""
output: io.StringIO = io.StringIO(newline='')
all_scores: Dict[Id, List[Score]] = {}
all_confs: Dict[Id, List[Score]] = {}
all_gammas: Dict[Id, List[Score]] = {}
all_length: Dict[Id, List[int]] = {}
taxids: Set[Id] = set()
num_read: int = 0
nt_read: int = 0
num_uncl: int = 0
last_error_read: int = -1 # Number of read of the last error
num_errors: int = 0 # Number or reads discarded due to error
output.write(gray(f'Loading output file {output_file}... '))
try:
with open(output_file, 'r') as file:
# Check number of cols in header
header = file.readline().split(',')
if len(header) != 8:
print(
red('\nERROR! ') + 'CLARK output format of ',
yellow(f'"{output_file}"'), 'not supported.')
print(magenta('Expected:'),
'ID,Length,Gamma,1st,score1,2nd,score2,conf')
print(magenta('Found:'), ','.join(header), end='')
print(blue('HINT:'), 'Use CLARK, CLARK-l, or CLARK-S '
'with full mode (', blue('-m 0'), ')')
raise Exception('Unsupported file format. Aborting.')
for raw_line in file:
try:
output_line = raw_line.strip()
(_label, _length, _gamma, _tid1, _score1, _tid2, _score2,
_conf) = output_line.split(',')
except ValueError:
print(
yellow('Failure'), 'parsing line elements:'
f' {output_line} in {output_file}'
'. Ignoring line!')
last_error_read = num_read + 1
num_errors += 1
continue
try:
length: int = int(_length)
gamma: Score = Score(float(_gamma))
tid1: Id = Id(_tid1)
score1: Score = Score(float(_score1))
tid2: Id = Id(_tid2)
score2: Score = Score(float(_score2))
conf: Score = Score(float(_conf))
except ValueError:
print(
yellow('Failure'), 'parsing line elements:'
f' {output_line} in {output_file}'
'. Ignoring line!')
last_error_read = num_read + 1
num_errors += 1
continue
num_read += 1
nt_read += length
# Select tid and score between CLARK assignments 1 and 2
tid: Id = tid1
score: Score = score1
if tid1 == UNCLASSIFIED:
if tid2 == UNCLASSIFIED: # Just count unclassified reads
num_uncl += 1
continue
else: # Majority of read unclassified
tid = tid2
score = score2
conf = Score(1 - conf) # Get CLARK's h2/(h1+h2)
# From CLARK_C(S) score get "single hit equivalent length"
shel: Score = Score(score + K_MER_SIZE)
taxids.add(tid) # Save all the selected tids (tid1 or tid2)
if minscore is not None: # Decide if ignore read if low score
if scoring is Scoring.CLARK_C:
if conf < minscore:
continue
elif scoring is Scoring.CLARK_G:
if gamma < minscore:
continue
else:
if shel < minscore:
continue
try:
all_scores[tid].append(shel)
except KeyError:
all_scores[tid] = [
shel,
]
try:
all_confs[tid].append(conf)
except KeyError:
all_confs[tid] = [
conf,
]
try:
all_gammas[tid].append(gamma)
except KeyError:
all_gammas[tid] = [
gamma,
]
try:
all_length[tid].append(length)
except KeyError:
all_length[tid] = [
length,
]
except FileNotFoundError:
raise Exception(red('\nERROR! ') + f'Cannot read "{output_file}"')
if last_error_read == num_read + 1: # Check error in last line: truncated!
print(yellow('Warning!'), f'{output_file} seems truncated!')
counts: Counter[Id] = col.Counter(
{tid: len(all_scores[tid])
for tid in all_scores})
output.write(green('OK!\n'))
if num_read == 0:
raise Exception(
red('\nERROR! ') +
f'Cannot read any sequence from "{output_file}"')
filt_seqs: int = sum([len(scores) for scores in all_scores.values()])
if filt_seqs == 0:
raise Exception(red('\nERROR! ') + 'No sequence passed the filter!')
# Get statistics
stat: SampleStats = SampleStats(minscore=minscore,
nt_read=nt_read,
lens=all_length,
scores=all_scores,
scores2=all_confs,
scores3=all_gammas,
seq_read=num_read,
seq_unclas=num_uncl,
seq_filt=filt_seqs,
tid_clas=len(taxids))
# Output statistics
if num_errors:
output.write(
gray(' Seqs fail: ') + red(f'{num_errors:_d}\t') +
gray('(Last error in read ') + red(f'{last_error_read}') +
gray(')\n'))
output.write(
gray(' Seqs read: ') + f'{stat.seq.read:_d}\t' + gray('[') +
f'{stat.nt_read}' + gray(']\n'))
output.write(
gray(' Seqs clas: ') + f'{stat.seq.clas:_d}\t' + gray('(') +
f'{stat.get_unclas_ratio():.2%}' + gray(' unclassified)\n'))
output.write(
gray(' Seqs pass: '******'{stat.seq.filt:_d}\t' + gray('(') +
f'{stat.get_reject_ratio():.2%}' + gray(' rejected)\n'))
output.write(
gray(' Hit (score): min = ') + f'{stat.sco.mini:.1f},' +
gray(' max = ') + f'{stat.sco.maxi:.1f},' + gray(' avr = ') +
f'{stat.sco.mean:.1f}\n')
output.write(
gray(' Conf. score: min = ') + f'{stat.sco2.mini:.1f},' +
gray(' max = ') + f'{stat.sco2.maxi:.1f},' + gray(' avr = ') +
f'{stat.sco2.mean:.1f}\n')
output.write(
gray(' Gamma score: min = ') + f'{stat.sco3.mini:.1f},' +
gray(' max = ') + f'{stat.sco3.maxi:.1f},' + gray(' avr = ') +
f'{stat.sco3.mean:.1f}\n')
output.write(
gray(' Read length: min = ') + f'{stat.len.mini},' + gray(' max = ') +
f'{stat.len.maxi},' + gray(' avr = ') + f'{stat.len.mean}\n')
output.write(
gray(' TaxIds: by classifier = ') + f'{stat.tid.clas}' +
gray(', by filter = ') + f'{stat.tid.filt}\n')
# Select score output
out_scores: Dict[Id, Score]
if scoring is Scoring.SHEL:
out_scores = {tid: Score(mean(all_scores[tid])) for tid in all_scores}
elif scoring is Scoring.CLARK_C:
out_scores = {
tid: Score(mean(all_confs[tid]) * 100)
for tid in all_confs
}
elif scoring is Scoring.CLARK_G:
out_scores = {tid: Score(mean(all_gammas[tid])) for tid in all_gammas}
elif scoring is Scoring.LENGTH:
out_scores = {tid: Score(mean(all_length[tid])) for tid in all_length}
elif scoring is Scoring.LOGLENGTH:
out_scores = {
tid: Score(log10(mean(all_length[tid])))
for tid in all_length
}
elif scoring is Scoring.NORMA:
scores: Dict[Id, Score] = {
tid: Score(mean(all_scores[tid]))
for tid in all_scores
}
lengths: Dict[Id, Score] = {
tid: Score(mean(all_length[tid]))
for tid in all_length
}
out_scores = {
tid: Score(scores[tid] / lengths[tid] * 100)
for tid in scores
}
else:
print(red('ERROR!'), f'clark: Unsupported Scoring "{scoring}"')
raise Exception('Unsupported scoring')
# Return
return output.getvalue(), stat, counts, out_scores
def read_lmat_output(
output_file: Filename,
scoring: Scoring = Scoring.LMAT,
minscore: Score = None,
) -> Tuple[str, SampleStats, Counter[Id], Dict[Id, Score]]:
"""
Read LMAT output (iterate over all the output files)
Args:
output_file: output file name (prefix)
scoring: type of scoring to be applied (see Scoring class)
minscore: minimum confidence level for the classification
Returns:
log string, abundances counter, scores dict
"""
output: io.StringIO = io.StringIO(newline='')
all_scores: Dict[Id, List[Score]] = {}
all_length: Dict[Id, List[int]] = {}
nt_read: int = 0
matchings: Counter[Match] = Counter()
output_files: List[Filename] = []
# Select files to process depending on if the output files are explicitly
# given or directory name is provided (all the output files there)
if os.path.isdir(output_file): # Just the directory name is provided
dirname = os.path.normpath(output_file)
for file in os.listdir(dirname): # Add all LMAT output files in dir
if ('_output' in file and file.endswith('.out')
and 'canVfin' not in file and 'pyLCA' not in file):
output_files.append(Filename(file))
else: # Explicit path and file name prefix is given
dirname, basename = os.path.split(output_file)
for file in os.listdir(dirname): # Add selected output files in dir
if (file.startswith(basename) and file.endswith('.out')
and 'canVfin' not in file and 'pyLCA' not in file):
output_files.append(Filename(file))
if not output_files:
raise Exception(
f'\n\033[91mERROR!\033[0m Cannot read from "{output_file}"')
# Read LMAT output files
for output_name in output_files:
path: Filename = Filename(os.path.join(dirname, output_name))
output.write(f'\033[90mLoading output file {path}...\033[0m')
try:
with open(path, 'r') as io_file:
for seq in SeqIO.parse(io_file, "lmat"):
tid: Id = seq.annotations['final_taxid']
score: Score = seq.annotations['final_score']
match: Match = Match.lmat(seq.annotations['final_match'])
matchings[match] += 1
length: int = len(seq)
nt_read += length
if minscore is not None:
if score < minscore: # Ignore read if low score
continue
if match in [Match.DIRECTMATCH, Match.MULTIMATCH]:
try:
all_scores[tid].append(score)
except KeyError:
all_scores[tid] = [
score,
]
try:
all_length[tid].append(length)
except KeyError:
all_length[tid] = [
length,
]
except FileNotFoundError:
raise Exception(red('\nERROR!') + f'Cannot read "{path}"')
output.write(green('OK!\n'))
abundances: Counter[Id] = Counter(
{tid: len(all_scores[tid])
for tid in all_scores})
# Basic output statistics
read_seqs: int = sum(matchings.values())
if read_seqs == 0:
raise Exception(
red('\nERROR! ') + f'Cannot read any sequence from"{output_file}"')
filt_seqs: int = sum([len(scores) for scores in all_scores.values()])
if filt_seqs == 0:
raise Exception(red('\nERROR! ') + 'No sequence passed the filter!')
stat: SampleStats = SampleStats(minscore=minscore,
nt_read=nt_read,
scores=all_scores,
lens=all_length,
seq_read=read_seqs,
seq_filt=filt_seqs,
seq_clas=matchings[Match.DIRECT] +
matchings[Match.MULTI])
output.write(
gray(' Seqs read: ') + f'{stat.seq.read:_d}\t' + gray('[') +
f'{stat.nt_read}' + gray(']\n'))
output.write(
gray(' Seqs clas: ') + f'{stat.seq.clas:_d}\t' + gray('(') +
f'{stat.get_unclas_ratio():.2%}' + gray(' unclassified)\n'))
output.write(
gray(' Seqs pass: '******'{stat.seq.filt:_d}\t' + gray('(') +
f'{stat.get_reject_ratio():.2%}' + gray(' rejected)\n'))
multi_rel: float = matchings[Match.MULTI] / read_seqs
direct_rel: float = matchings[Match.DIRECT] / read_seqs
nodbhits_rel: float = matchings[Match.NODBHITS] / read_seqs
tooshort_rel: float = matchings[Match.READTOOSHORT] / read_seqs
lowscore_rel: float = matchings[Match.LOWSCORE] / read_seqs
output.write(f'\033[90m DB Matching: '
f'Multi =\033[0m {multi_rel:.1%}\033[90m '
f'Direct =\033[0m {direct_rel:.1%}\033[90m '
f'ReadTooShort =\033[0m {tooshort_rel:.1%}\033[90m '
f'LowScore =\033[0m {lowscore_rel:.1%}\033[90m '
f'NoDbHits =\033[0m {nodbhits_rel:.1%}\033[90m\n')
output.write(
gray(' Scores: min = ') + f'{stat.sco.mini:.1f},' + gray(' max = ') +
f'{stat.sco.maxi:.1f},' + gray(' avr = ') + f'{stat.sco.mean:.1f}\n')
output.write(
gray(' Length: min = ') + f'{stat.len.mini},' + gray(' max = ') +
f'{stat.len.maxi},' + gray(' avr = ') + f'{stat.len.mean}\n')
output.write(f' {stat.num_taxa}' + gray(f' taxa with assigned reads\n'))
# Select score output
out_scores: Dict[Id, Score]
if scoring is Scoring.LMAT:
out_scores = {tid: Score(mean(all_scores[tid])) for tid in all_scores}
else:
print(red('ERROR!'), f' LMAT: Unsupported Scoring "{scoring}"')
raise Exception('Unsupported scoring') # Return
return output.getvalue(), stat, abundances, out_scores
def fltlst2str(lst: List[float]) -> str:
"""Convert a list of floats into a nice string"""
return '[' + gray((', '.join(f'{elm:.1g}' for elm in lst))) + ']'
def process_rank(
*args, **kwargs
) -> Tuple[List[Sample], Dict[Sample, UnionCounter], Dict[Sample, Counter[Id]],
Dict[Sample, UnionScores]]:
"""
Process results for a taxlevel (to be usually called in parallel!).
"""
# Recover input and parameters
rank: Rank = args[0]
controls: int = kwargs['controls']
mintaxas: Dict[Sample, int] = kwargs['mintaxas']
ontology: Ontology = kwargs['ontology']
including = ontology.including
excluding = ontology.excluding
taxids: Dict[Sample, TaxLevels] = kwargs['taxids']
counts: Dict[Sample, UnionCounter] = kwargs['counts']
accs: Dict[Sample, Counter[Id]] = kwargs['accs']
scores: Dict[Sample, UnionScores] = kwargs['scores']
raws: List[Sample] = kwargs['raw_samples']
output: io.StringIO = io.StringIO(newline='')
def vwrite(*args) -> None:
"""Print only if verbose/debug mode is enabled"""
if kwargs['debug']:
output.write(' '.join(str(item) for item in args))
def fltlst2str(lst: List[float]) -> str:
"""Convert a list of floats into a nice string"""
return '[' + gray((', '.join(f'{elm:.1g}' for elm in lst))) + ']'
def blst2str(lst: List[bool]) -> str:
"""Convert a list of booleans into a nice string"""
return ('[' + (', '.join(magenta('T') if elm else 'F'
for elm in lst)) + ']')
def get_shared_mintaxa() -> int:
"""Give a value of mintaxa for shared derived samples
This value is currently the minimum of the mintaxa of all the
(non control) raw samples.
"""
return min([mintaxas[smpl] for smpl in raws[controls:]])
# Declare/define variables
samples: List[Sample] = []
# pylint: disable = unused-variable
shared_counts: SharedCounter = SharedCounter()
shared_score: SharedCounter = SharedCounter()
shared_ctrl_counts: SharedCounter = SharedCounter()
shared_ctrl_score: SharedCounter = SharedCounter()
# pylint: enable = unused-variable
output.write(f'\033[90mAnalysis for taxonomic rank "'
f'\033[95m{rank.name.lower()}\033[90m":\033[0m\n')
def cross_analysis(iteration, raw):
"""Cross analysis: exclusive and part of shared&ctrl"""
nonlocal shared_counts, shared_score
nonlocal shared_ctrl_counts, shared_ctrl_score
def partial_shared_update(i):
"""Perform shared and shared-control taxa partial evaluations"""
nonlocal shared_counts, shared_score
nonlocal shared_ctrl_counts, shared_ctrl_score
if i == 0: # 1st iteration: Initialize shared abundance and score
shared_counts.update(sub_shared_counts)
shared_score.update(sub_shared_score)
elif i < controls: # Just update shared abundance and score
shared_counts &= sub_shared_counts
shared_score &= sub_shared_score
elif i == controls: # Initialize shared-control counters
shared_counts &= sub_shared_counts
shared_score &= sub_shared_score
shared_ctrl_counts.update(sub_shared_counts)
shared_ctrl_score.update(sub_shared_score)
elif controls: # Both: Accumulate shared abundance and score
shared_counts &= sub_shared_counts
shared_score &= sub_shared_score
shared_ctrl_counts &= sub_shared_counts
shared_ctrl_score &= sub_shared_score
else: # Both: Accumulate shared abundance and score (no controls)
shared_counts &= sub_shared_counts
shared_score &= sub_shared_score
exclude: Set[Id] = set()
# Get taxids at this rank that are present in the other samples
for sample in (smpl for smpl in raws if smpl != raw):
exclude.update(taxids[sample][rank])
exclude.update(excluding) # Add explicit excluding taxa if any
output.write(f' \033[90mExclusive: From \033[0m{raw}\033[90m '
f'excluding {len(exclude)} taxa. '
f'Generating sample...\033[0m')
exclude_tree = TaxTree()
exclude_out = SampleDataById(['counts', 'scores', 'accs'])
exclude_tree.allin1(ontology=ontology,
counts=counts[raw],
scores=scores[raw],
min_taxa=mintaxas[raw],
min_rank=rank,
just_min_rank=True,
include=including,
exclude=exclude,
out=exclude_out)
exclude_out.purge_counters()
if exclude_out.counts: # Avoid adding empty samples
sample = Sample(f'{raw}_{STR_EXCLUSIVE}_{rank.name.lower()}')
samples.append(sample)
counts[sample] = exclude_out.get_counts()
accs[sample] = exclude_out.get_accs()
scores[sample] = exclude_out.get_scores()
output.write('\033[92m OK! \033[0m\n')
else:
output.write('\033[93m VOID \033[0m\n')
# Get partial abundance and score for the shared analysis
sub_shared_tree = TaxTree()
sub_shared_out = SampleDataById(['shared', 'accs'])
sub_shared_tree.allin1(ontology=ontology,
counts=counts[raw],
scores=scores[raw],
min_taxa=mintaxas[raw],
min_rank=rank,
just_min_rank=True,
include=including,
exclude=excluding,
out=sub_shared_out)
sub_shared_out.purge_counters()
# Scale scores by abundance
sub_shared_counts: SharedCounter = sub_shared_out.get_shared_counts()
sub_shared_score: SharedCounter = sub_shared_out.get_shared_scores()
sub_shared_score *= sub_shared_counts
partial_shared_update(iteration)
def shared_analysis():
"""Perform last steps of shared taxa analysis"""
shared_tree: TaxTree = TaxTree()
shared_out: SampleDataById = SampleDataById(['shared', 'accs'])
shared_tree.allin1(ontology=ontology,
counts=shared_counts,
scores=shared_score,
min_taxa=get_shared_mintaxa(),
include=including,
exclude=excluding,
out=shared_out)
shared_out.purge_counters()
out_counts: SharedCounter = shared_out.get_shared_counts()
output.write(
gray(f' Shared: Including {len(out_counts)}'
' shared taxa. Generating sample... '))
if out_counts:
sample = Sample(f'{STR_SHARED}_{rank.name.lower()}')
samples.append(sample)
counts[Sample(sample)] = out_counts
accs[Sample(sample)] = shared_out.get_accs()
scores[sample] = shared_out.get_shared_scores()
output.write(green('OK!\n'))
else:
output.write(yellow('VOID\n'))
def control_analysis():
"""Perform last steps of control and shared controls analysis"""
nonlocal shared_ctrl_counts, shared_ctrl_score
def robust_contamination_removal():
"""Implement robust contamination removal algorithm."""
nonlocal exclude_sets, shared_crossover
def compute_qn(data: List[float], dist: str = "Gauss") -> float:
"""Compute Qn robust estimator of scale (Rousseeuw, 1993)"""
c_d: float # Select d parameter depending on the distribution
if dist == "Gauss":
c_d = 2.2219
elif dist == "Cauchy": # Heavy-tailed distribution
c_d = 1.2071
elif dist == "NegExp": # Negative exponential (asymetric)
c_d = 3.4760
else:
raise Exception(red('\nERROR! ') + 'Unknown distribution')
num: int = len(data)
sort_data = sorted(data)
pairwisedifs: List[float] = []
for (i, x_val) in enumerate(sort_data):
for y_val in sort_data[i + 1:]:
pairwisedifs.append(abs(x_val - y_val))
k: int = int(num * (num / 2 + 1) / 4)
return c_d * sorted(pairwisedifs)[k - 1]
exclude_sets = {smpl: set() for smpl in raws[controls:]}
vwrite(
gray('Robust contamination removal: '
'Searching for contaminants...\n'))
for tid in exclude_candidates:
relfreq_ctrl: List[float] = [
accs[ctrl][tid] / accs[ctrl][ontology.ROOT]
for ctrl in raws[:controls]
]
relfreq_smpl: List[float] = [
accs[smpl][tid] / accs[smpl][ontology.ROOT]
for smpl in raws[controls:]
]
relfreq: List[float] = relfreq_ctrl + relfreq_smpl
crossover: List[bool] # Crossover source (yes/no)
# Just-controls contamination check
if all([rf < EPS for rf in relfreq_smpl]):
vwrite(cyan('just-ctrl:\t'), tid, ontology.get_name(tid),
gray('relfreq:'),
fltlst2str(relfreq_ctrl) + fltlst2str(relfreq_smpl),
'\n')
continue # Go for next candidate
# Critical contamination check
if all([rf > SEVR_CONTM_MIN_RELFREQ for rf in relfreq_ctrl]):
vwrite(red('critical:\t'), tid, ontology.get_name(tid),
gray('relfreq:'),
fltlst2str(relfreq_ctrl) + fltlst2str(relfreq_smpl),
'\n')
for exclude_set in exclude_sets.values():
exclude_set.add(tid)
continue # Go for next candidate
# Severe contamination check
if any([rf > SEVR_CONTM_MIN_RELFREQ for rf in relfreq_ctrl]):
vwrite(yellow('severe: \t'), tid, ontology.get_name(tid),
gray('relfreq:'),
fltlst2str(relfreq_ctrl) + fltlst2str(relfreq_smpl),
'\n')
for exclude_set in exclude_sets.values():
exclude_set.add(tid)
continue # Go for next candidate
# Mild contamination check
if all([rf > MILD_CONTM_MIN_RELFREQ for rf in relfreq_ctrl]):
vwrite(blue('mild cont:\t'), tid, ontology.get_name(tid),
gray('relfreq:'),
fltlst2str(relfreq_ctrl) + fltlst2str(relfreq_smpl),
'\n')
for exclude_set in exclude_sets.values():
exclude_set.add(tid)
continue # Go for next candidate
# Calculate median and MAD median but including controls
mdn: float = statistics.median(relfreq)
# mad:float=statistics.mean([abs(mdn - rf) for rf in relfreq])
q_n: float = compute_qn(relfreq, dist="NegExp")
# Calculate crossover in samples
outlier_lim: float = mdn + ROBUST_XOVER_OUTLIER * q_n
ordomag_lim: float = max(
relfreq_ctrl) * 10**ROBUST_XOVER_ORD_MAG
crossover = [
rf > outlier_lim and rf > ordomag_lim
for rf in relfreq[controls:]
]
# Crossover contamination check
if any(crossover):
vwrite(
magenta('crossover:\t'), tid, ontology.get_name(tid),
green(f'lims: [{outlier_lim:.1g}]' +
('<' if outlier_lim < ordomag_lim else '>') +
f'[{ordomag_lim:.1g}]'), gray('relfreq:'),
fltlst2str(relfreq_ctrl) + fltlst2str(relfreq_smpl),
gray('crossover:'), blst2str(crossover), '\n')
# Exclude just for contaminated samples (not the source)
vwrite(magenta('\t->'), gray(f'Include {tid} just in:'))
for i in range(len(raws[controls:])):
if not crossover[i]:
exclude_sets[raws[i + controls]].add(tid)
else:
vwrite(f' {raws[i + controls]}')
if all(crossover): # Shared taxon contaminating control(s)
vwrite(' (', yellow('Shared crossover taxon!'), ')')
shared_crossover.add(tid)
vwrite('\n')
continue
# Other contamination: remove from all samples
vwrite(
gray('other cont:\t'), tid, ontology.get_name(tid),
green(f'lims: [{outlier_lim:.1g}]' +
('<' if outlier_lim < ordomag_lim else '>') +
f'[{ordomag_lim:.1g}]'), gray('relfreq:'),
fltlst2str(relfreq_ctrl) + fltlst2str(relfreq_smpl), '\n')
for exclude_set in exclude_sets.values():
exclude_set.add(tid)
# Get taxids at this rank that are present in the control samples
exclude_candidates: Set[Id] = set()
for i in range(controls):
exclude_candidates.update(taxids[raws[i]][rank])
exclude_sets: Dict[Sample, Set[Id]]
shared_crossover: Set[Id] = set() # Shared taxa contaminating controls
if controls and (len(raws) - controls >= ROBUST_MIN_SAMPLES):
robust_contamination_removal()
else: # If this case, just apply strict control
exclude_sets = {
file: exclude_candidates
for file in raws[controls::]
}
# Add explicit excluding taxa (if any) to exclude sets
for exclude_set in exclude_sets.values():
exclude_set.update(excluding)
exclude_candidates.update(excluding)
# Process each sample excluding control taxa
for raw in raws[controls:]:
output.write(
gray(' Ctrl: From') + f' {raw} ' +
gray(f'excluding {len(exclude_sets[raw])} ctrl taxa. '
f'Generating sample... '))
ctrl_tree = TaxTree()
ctrl_out = SampleDataById(['counts', 'scores', 'accs'])
ctrl_tree.allin1(ontology=ontology,
counts=counts[raw],
scores=scores[raw],
min_taxa=mintaxas[raw],
min_rank=rank,
just_min_rank=True,
include=including,
exclude=exclude_sets[raw],
out=ctrl_out)
ctrl_out.purge_counters()
if ctrl_out.counts: # Avoid adding empty samples
sample = Sample(f'{raw}_{STR_CONTROL}_{rank.name.lower()}')
samples.append(sample)
counts[sample] = ctrl_out.get_counts()
accs[sample] = ctrl_out.get_accs()
scores[sample] = ctrl_out.get_scores()
output.write(green('OK!\n'))
else:
output.write(yellow('VOID\n'))
def shared_ctrl_analysis():
"""Perform last steps of shared taxa analysis"""
shared_ctrl_tree: TaxTree = TaxTree()
shared_ctrl_out: SampleDataById = SampleDataById(
['shared', 'accs'])
shared_ctrl_tree.allin1(ontology=ontology,
counts=shared_ctrl_counts,
scores=shared_ctrl_score,
min_taxa=get_shared_mintaxa(),
include=including,
exclude=(exclude_candidates -
shared_crossover),
out=shared_ctrl_out)
shared_ctrl_out.purge_counters()
out_counts: SharedCounter = shared_ctrl_out.get_shared_counts()
output.write(
gray(f' Ctrl-shared: Including {len(out_counts)}'
' shared taxa. Generating sample... '))
if out_counts:
sample = Sample(f'{STR_CONTROL_SHARED}_{rank.name.lower()}')
samples.append(sample)
counts[Sample(sample)] = out_counts
accs[Sample(sample)] = shared_ctrl_out.get_accs()
scores[sample] = shared_ctrl_out.get_shared_scores()
output.write(green('OK!\n'))
else:
output.write(yellow('VOID\n'))
# Shared-control taxa final analysis
if shared_ctrl_counts:
# Normalize scaled scores by total abundance
shared_ctrl_score /= (+shared_ctrl_counts)
# Get averaged abundance by number of samples minus ctrl samples
shared_ctrl_counts //= (len(raws) - controls)
shared_ctrl_analysis()
else:
output.write(
gray(' Ctrl-shared: No taxa! ') + yellow('VOID') +
gray(' sample.\n'))
# Cross analysis iterating by output: exclusive and part of shared&ctrl
for num_file, raw_sample_name in enumerate(raws):
cross_analysis(num_file, raw_sample_name)
# Shared taxa final analysis
shared_counts = +shared_counts # remove counts <= 0
if shared_counts:
# Normalize scaled scores by total abundance (after eliminating zeros)
shared_score /= (+shared_counts)
# Get averaged abundance by number of samples
shared_counts //= len(raws)
shared_analysis()
else:
output.write(
gray(' Shared: No shared taxa! ') + yellow('VOID') +
gray(' sample.\n'))
# Control sample subtraction
if controls:
control_analysis()
# Print output and return
print(output.getvalue())
sys.stdout.flush()
return samples, counts, accs, scores
def control_analysis():
"""Perform last steps of control and shared controls analysis"""
nonlocal shared_ctrl_counts, shared_ctrl_score
def robust_contamination_removal():
"""Implement robust contamination removal algorithm."""
nonlocal exclude_sets, shared_crossover
def compute_qn(data: List[float], dist: str = "Gauss") -> float:
"""Compute Qn robust estimator of scale (Rousseeuw, 1993)"""
c_d: float # Select d parameter depending on the distribution
if dist == "Gauss":
c_d = 2.2219
elif dist == "Cauchy": # Heavy-tailed distribution
c_d = 1.2071
elif dist == "NegExp": # Negative exponential (asymetric)
c_d = 3.4760
else:
raise Exception(red('\nERROR! ') + 'Unknown distribution')
num: int = len(data)
sort_data = sorted(data)
pairwisedifs: List[float] = []
for (i, x_val) in enumerate(sort_data):
for y_val in sort_data[i + 1:]:
pairwisedifs.append(abs(x_val - y_val))
k: int = int(num * (num / 2 + 1) / 4)
return c_d * sorted(pairwisedifs)[k - 1]
exclude_sets = {smpl: set() for smpl in raws[controls:]}
vwrite(
gray('Robust contamination removal: '
'Searching for contaminants...\n'))
for tid in exclude_candidates:
relfreq_ctrl: List[float] = [
accs[ctrl][tid] / accs[ctrl][ontology.ROOT]
for ctrl in raws[:controls]
]
relfreq_smpl: List[float] = [
accs[smpl][tid] / accs[smpl][ontology.ROOT]
for smpl in raws[controls:]
]
relfreq: List[float] = relfreq_ctrl + relfreq_smpl
crossover: List[bool] # Crossover source (yes/no)
# Just-controls contamination check
if all([rf < EPS for rf in relfreq_smpl]):
vwrite(cyan('just-ctrl:\t'), tid, ontology.get_name(tid),
gray('relfreq:'),
fltlst2str(relfreq_ctrl) + fltlst2str(relfreq_smpl),
'\n')
continue # Go for next candidate
# Critical contamination check
if all([rf > SEVR_CONTM_MIN_RELFREQ for rf in relfreq_ctrl]):
vwrite(red('critical:\t'), tid, ontology.get_name(tid),
gray('relfreq:'),
fltlst2str(relfreq_ctrl) + fltlst2str(relfreq_smpl),
'\n')
for exclude_set in exclude_sets.values():
exclude_set.add(tid)
continue # Go for next candidate
# Severe contamination check
if any([rf > SEVR_CONTM_MIN_RELFREQ for rf in relfreq_ctrl]):
vwrite(yellow('severe: \t'), tid, ontology.get_name(tid),
gray('relfreq:'),
fltlst2str(relfreq_ctrl) + fltlst2str(relfreq_smpl),
'\n')
for exclude_set in exclude_sets.values():
exclude_set.add(tid)
continue # Go for next candidate
# Mild contamination check
if all([rf > MILD_CONTM_MIN_RELFREQ for rf in relfreq_ctrl]):
vwrite(blue('mild cont:\t'), tid, ontology.get_name(tid),
gray('relfreq:'),
fltlst2str(relfreq_ctrl) + fltlst2str(relfreq_smpl),
'\n')
for exclude_set in exclude_sets.values():
exclude_set.add(tid)
continue # Go for next candidate
# Calculate median and MAD median but including controls
mdn: float = statistics.median(relfreq)
# mad:float=statistics.mean([abs(mdn - rf) for rf in relfreq])
q_n: float = compute_qn(relfreq, dist="NegExp")
# Calculate crossover in samples
outlier_lim: float = mdn + ROBUST_XOVER_OUTLIER * q_n
ordomag_lim: float = max(
relfreq_ctrl) * 10**ROBUST_XOVER_ORD_MAG
crossover = [
rf > outlier_lim and rf > ordomag_lim
for rf in relfreq[controls:]
]
# Crossover contamination check
if any(crossover):
vwrite(
magenta('crossover:\t'), tid, ontology.get_name(tid),
green(f'lims: [{outlier_lim:.1g}]' +
('<' if outlier_lim < ordomag_lim else '>') +
f'[{ordomag_lim:.1g}]'), gray('relfreq:'),
fltlst2str(relfreq_ctrl) + fltlst2str(relfreq_smpl),
gray('crossover:'), blst2str(crossover), '\n')
# Exclude just for contaminated samples (not the source)
vwrite(magenta('\t->'), gray(f'Include {tid} just in:'))
for i in range(len(raws[controls:])):
if not crossover[i]:
exclude_sets[raws[i + controls]].add(tid)
else:
vwrite(f' {raws[i + controls]}')
if all(crossover): # Shared taxon contaminating control(s)
vwrite(' (', yellow('Shared crossover taxon!'), ')')
shared_crossover.add(tid)
vwrite('\n')
continue
# Other contamination: remove from all samples
vwrite(
gray('other cont:\t'), tid, ontology.get_name(tid),
green(f'lims: [{outlier_lim:.1g}]' +
('<' if outlier_lim < ordomag_lim else '>') +
f'[{ordomag_lim:.1g}]'), gray('relfreq:'),
fltlst2str(relfreq_ctrl) + fltlst2str(relfreq_smpl), '\n')
for exclude_set in exclude_sets.values():
exclude_set.add(tid)
# Get taxids at this rank that are present in the control samples
exclude_candidates: Set[Id] = set()
for i in range(controls):
exclude_candidates.update(taxids[raws[i]][rank])
exclude_sets: Dict[Sample, Set[Id]]
shared_crossover: Set[Id] = set() # Shared taxa contaminating controls
if controls and (len(raws) - controls >= ROBUST_MIN_SAMPLES):
robust_contamination_removal()
else: # If this case, just apply strict control
exclude_sets = {
file: exclude_candidates
for file in raws[controls::]
}
# Add explicit excluding taxa (if any) to exclude sets
for exclude_set in exclude_sets.values():
exclude_set.update(excluding)
exclude_candidates.update(excluding)
# Process each sample excluding control taxa
for raw in raws[controls:]:
output.write(
gray(' Ctrl: From') + f' {raw} ' +
gray(f'excluding {len(exclude_sets[raw])} ctrl taxa. '
f'Generating sample... '))
ctrl_tree = TaxTree()
ctrl_out = SampleDataById(['counts', 'scores', 'accs'])
ctrl_tree.allin1(ontology=ontology,
counts=counts[raw],
scores=scores[raw],
min_taxa=mintaxas[raw],
min_rank=rank,
just_min_rank=True,
include=including,
exclude=exclude_sets[raw],
out=ctrl_out)
ctrl_out.purge_counters()
if ctrl_out.counts: # Avoid adding empty samples
sample = Sample(f'{raw}_{STR_CONTROL}_{rank.name.lower()}')
samples.append(sample)
counts[sample] = ctrl_out.get_counts()
accs[sample] = ctrl_out.get_accs()
scores[sample] = ctrl_out.get_scores()
output.write(green('OK!\n'))
else:
output.write(yellow('VOID\n'))
def shared_ctrl_analysis():
"""Perform last steps of shared taxa analysis"""
shared_ctrl_tree: TaxTree = TaxTree()
shared_ctrl_out: SampleDataById = SampleDataById(
['shared', 'accs'])
shared_ctrl_tree.allin1(ontology=ontology,
counts=shared_ctrl_counts,
scores=shared_ctrl_score,
min_taxa=get_shared_mintaxa(),
include=including,
exclude=(exclude_candidates -
shared_crossover),
out=shared_ctrl_out)
shared_ctrl_out.purge_counters()
out_counts: SharedCounter = shared_ctrl_out.get_shared_counts()
output.write(
gray(f' Ctrl-shared: Including {len(out_counts)}'
' shared taxa. Generating sample... '))
if out_counts:
sample = Sample(f'{STR_CONTROL_SHARED}_{rank.name.lower()}')
samples.append(sample)
counts[Sample(sample)] = out_counts
accs[Sample(sample)] = shared_ctrl_out.get_accs()
scores[sample] = shared_ctrl_out.get_shared_scores()
output.write(green('OK!\n'))
else:
output.write(yellow('VOID\n'))
# Shared-control taxa final analysis
if shared_ctrl_counts:
# Normalize scaled scores by total abundance
shared_ctrl_score /= (+shared_ctrl_counts)
# Get averaged abundance by number of samples minus ctrl samples
shared_ctrl_counts //= (len(raws) - controls)
shared_ctrl_analysis()
else:
output.write(
gray(' Ctrl-shared: No taxa! ') + yellow('VOID') +
gray(' sample.\n'))
def main():
"""Main entry point to script."""
# Argument Parser Configuration
parser = argparse.ArgumentParser(
description='Extract reads following Centrifuge/Kraken output',
epilog=f'%(prog)s - {__author__} - {__date__}')
parser.add_argument('-V',
'--version',
action='version',
version=f'%(prog)s release {__version__} ({__date__})')
parser.add_argument('-f',
'--file',
action='store',
metavar='FILE',
required=True,
help='Centrifuge output file.')
parser.add_argument('-l',
'--limit',
action='store',
metavar='NUMBER',
type=int,
default=None,
help=('Limit of FASTQ reads to extract. '
'Default: no limit'))
parser.add_argument(
'-m',
'--maxreads',
action='store',
metavar='NUMBER',
type=int,
default=None,
help=('Maximum number of FASTQ reads to search for the taxa. '
'Default: no maximum'))
parser.add_argument(
'-n',
'--nodespath',
action='store',
metavar='PATH',
default=TAXDUMP_PATH,
help=('path for the nodes information files (nodes.dmp and names.dmp' +
' from NCBI'))
parser.add_argument(
'-i',
'--include',
action='append',
metavar='TAXID',
type=TaxId,
default=[],
help=('NCBI taxid code to include a taxon and all underneath ' +
'(multiple -i is available to include several taxid). ' +
'By default all the taxa is considered for inclusion.'))
parser.add_argument(
'-x',
'--exclude',
action='append',
metavar='TAXID',
type=TaxId,
default=[],
help=('NCBI taxid code to exclude a taxon and all underneath ' +
'(multiple -x is available to exclude several taxid)'))
parser.add_argument(
'-y',
'--minscore',
action='store',
metavar='NUMBER',
type=lambda txt: Score(float(txt)),
default=None,
help=('minimum score/confidence of the classification of a read '
'to pass the quality filter; all pass by default'))
filein = parser.add_mutually_exclusive_group(required=True)
filein.add_argument('-q',
'--fastq',
action='store',
metavar='FILE',
default=None,
help='Single FASTQ file (no paired-ends)')
filein.add_argument('-1',
'--mate1',
action='store',
metavar='FILE',
default=None,
help='Paired-ends FASTQ file for mate 1s '
'(filename usually includes _1)')
parser.add_argument('-2',
'--mate2',
action='store',
metavar='FILE',
default=None,
help='Paired-ends FASTQ file for mate 2s '
'(filename usually includes _2)')
# timing initialization
start_time: float = time.time()
# Program header
print(f'\n=-= {sys.argv[0]} =-= v{__version__} =-= {__date__} =-=\n')
sys.stdout.flush()
# Parse arguments
args = parser.parse_args()
output_file = args.file
nodesfile: Filename = Filename(os.path.join(args.nodespath, NODES_FILE))
namesfile: Filename = Filename(os.path.join(args.nodespath, NAMES_FILE))
excluding: Set[TaxId] = set(args.exclude)
including: Set[TaxId] = set(args.include)
fastq_1: Filename
fastq_2: Filename = args.mate2
if not fastq_2:
fastq_1 = args.fastq
else:
fastq_1 = args.mate1
# Load NCBI nodes, names and build children
plasmidfile: Filename = None
ncbi: Taxonomy = Taxonomy(nodesfile, namesfile, plasmidfile, False,
excluding, including)
# Build taxonomy tree
print(gray('Building taxonomy tree...'), end='')
sys.stdout.flush()
tree = TaxTree()
tree.grow(taxonomy=ncbi, look_ancestors=False)
print(green(' OK!'))
# Get the taxa
print(gray('Filtering taxa...'), end='')
sys.stdout.flush()
ranks: Ranks = Ranks({})
tree.get_taxa(ranks=ranks, include=including, exclude=excluding)
print(green(' OK!'))
taxids: Set[TaxId] = set(ranks)
taxlevels: TaxLevels = Rank.ranks_to_taxlevels(ranks)
num_taxlevels = Counter({rank: len(taxlevels[rank]) for rank in taxlevels})
num_taxlevels = +num_taxlevels
# Statistics about including taxa
print(f' {len(taxids)}\033[90m taxid selected in \033[0m', end='')
print(f'{len(num_taxlevels)}\033[90m different taxonomical levels:\033[0m')
for rank in num_taxlevels:
print(f' Number of different {rank}: {num_taxlevels[rank]}')
assert taxids, red('ERROR! No taxids to search for!')
# Get the records
records: List[SeqRecord] = []
num_seqs: int = 0
# timing initialization
start_time_load: float = time.perf_counter()
print(gray(f'Loading output file {output_file}...'), end='')
sys.stdout.flush()
try:
with open(output_file, 'rU') as file:
file.readline() # discard header
for num_seqs, record in enumerate(SeqIO.parse(file, 'centrifuge')):
tid: TaxId = record.annotations['taxID']
if tid not in taxids:
continue # Ignore read if low confidence
score: Score = Score(record.annotations['score'])
if args.minscore is not None and score < args.minscore:
continue
records.append(record)
except FileNotFoundError:
raise Exception(red('ERROR!') + 'Cannot read "' + output_file + '"')
print(green(' OK!'))
# Basic records statistics
print(
gray(' Load elapsed time: ') +
f'{time.perf_counter() - start_time_load:.3g}' + gray(' sec'))
print(f' \033[90mMatching reads: \033[0m{len(records):_d} \033[90m\t'
f'(\033[0m{len(records)/num_seqs:.4%}\033[90m of sample)')
sys.stdout.flush()
# FASTQ sequence dealing
# records_ids: List[SeqRecord] = [record.id for record in records]
records_ids: Set[SeqRecord] = {record.id for record in records}
seqs1: List[SeqRecord] = []
seqs2: List[SeqRecord] = []
extracted: int = 0
i: int = 0
if fastq_2:
print(
f'\033[90mLoading FASTQ files {fastq_1} and {fastq_2}...\n'
f'Mseqs: \033[0m',
end='')
sys.stdout.flush()
try:
with open(fastq_1, 'rU') as file1, open(fastq_2, 'rU') as file2:
for i, (rec1, rec2) in enumerate(
zip(SeqIO.parse(file1, 'quickfastq'),
SeqIO.parse(file2, 'quickfastq'))):
if not records_ids or (args.maxreads and i >= args.maxreads
) or (args.limit
and extracted >= args.limit):
break
elif not i % 1000000:
print(f'{i//1000000:_d}', end='')
sys.stdout.flush()
elif not i % 100000:
print('.', end='')
sys.stdout.flush()
try:
records_ids.remove(rec1.id)
except KeyError:
pass
else:
seqs1.append(rec1)
seqs2.append(rec2)
extracted += 1
except FileNotFoundError:
raise Exception('\n\033[91mERROR!\033[0m Cannot read FASTQ files')
else:
print(f'\033[90mLoading FASTQ files {fastq_1}...\n'
f'Mseqs: \033[0m',
end='')
sys.stdout.flush()
try:
with open(fastq_1, 'rU') as file1:
for i, rec1 in enumerate(SeqIO.parse(file1, 'quickfastq')):
if not records_ids or (args.maxreads and i >= args.maxreads
) or (args.limit
and extracted >= args.limit):
break
elif not i % 1000000:
print(f'{i//1000000:_d}', end='')
sys.stdout.flush()
elif not i % 100000:
print('.', end='')
sys.stdout.flush()
try:
records_ids.remove(rec1.id)
except KeyError:
pass
else:
seqs1.append(rec1)
extracted += 1
except FileNotFoundError:
raise Exception('\n\033[91mERROR!\033[0m Cannot read FASTQ file')
print(cyan(f' {i/1e+6:.3g} Mseqs'), green('OK! '))
def format_filename(fastq: Filename) -> Filename:
"""Auxiliary function to properly format the output filenames.
Args:
fastq: Complete filename of the fastq input file
Returns: Filename of the rextracted fastq output file
"""
fastq_filename, _ = os.path.splitext(fastq)
output_list: List[str] = [fastq_filename, '_rxtr']
if including:
output_list.append('_incl')
output_list.extend('_'.join(including))
if excluding:
output_list.append('_excl')
output_list.extend('_'.join(excluding))
output_list.append('.fastq')
return Filename(''.join(output_list))
filename1: Filename = format_filename(fastq_1)
SeqIO.write(seqs1, filename1, 'quickfastq')
print(gray('Wrote'), magenta(f'{len(seqs1)}'), gray('reads in'), filename1)
if fastq_2:
filename2: Filename = format_filename(fastq_2)
SeqIO.write(seqs2, filename2, 'quickfastq')
print(gray('Wrote'), magenta(f'{len(seqs1)}'), gray('reads in'),
filename2)
# Timing results
print(gray('Total elapsed time:'),
time.strftime("%H:%M:%S", time.gmtime(time.time() - start_time)))
def read_output(
output_file: Filename,
scoring: Scoring = Scoring.SHEL,
minscore: Score = None,
) -> Tuple[str, SampleStats, Counter[Id], Dict[Id, Score]]:
"""
Read Centrifuge output file
Args:
output_file: output file name
scoring: type of scoring to be applied (see Scoring class)
minscore: minimum confidence level for the classification
Returns:
log string, statistics, abundances counter, scores dict
"""
output: io.StringIO = io.StringIO(newline='')
all_scores: Dict[Id, List[Score]] = {}
all_length: Dict[Id, List[int]] = {}
taxids: Set[Id] = set()
num_read: int = 0
nt_read: int = 0
num_uncl: int = 0
last_error_read: int = -1 # Number of read of the last error
num_errors: int = 0 # Number or reads discarded due to error
output.write(gray(f'Loading output file {output_file}... '))
try:
with open(output_file, 'r') as file:
file.readline() # discard header
for output_line in file:
try:
_, _, _tid, _score, _, _, _length, *_ = output_line.split(
'\t')
except ValueError:
print(
yellow('Failure'), 'parsing line elements:'
f' {output_line} in {output_file}'
'. Ignoring line!')
last_error_read = num_read + 1
num_errors += 1
continue
tid = Id(_tid)
try:
# From Centrifuge score get "single hit equivalent length"
shel = Score(float(_score)**0.5 + 15)
length = int(_length)
except ValueError:
print(yellow('Failure'), f'parsing score ({_score}) for ',
f'query length {_length} for taxid {_tid}',
f'in {output_file}. Ignoring line!')
last_error_read = num_read + 1
num_errors += 1
continue
num_read += 1
nt_read += length
if tid == UNCLASSIFIED: # Just count unclassified reads
num_uncl += 1
continue
else:
taxids.add(tid) # Save all the tids of classified reads
if minscore is not None and shel < minscore:
continue # Ignore read if low confidence
try:
all_scores[tid].append(shel)
except KeyError:
all_scores[tid] = [
shel,
]
try:
all_length[tid].append(length)
except KeyError:
all_length[tid] = [
length,
]
except FileNotFoundError:
raise Exception(red('\nERROR! ') + f'Cannot read "{output_file}"')
if last_error_read == num_read + 1: # Check error in last line: truncated!
print(yellow('Warning!'), f'{output_file} seems truncated!')
counts: Counter[Id] = col.Counter(
{tid: len(all_scores[tid])
for tid in all_scores})
output.write(green('OK!\n'))
if num_read == 0:
raise Exception(
red('\nERROR! ') +
f'Cannot read any sequence from "{output_file}"')
filt_seqs: int = sum([len(scores) for scores in all_scores.values()])
if filt_seqs == 0:
raise Exception(red('\nERROR! ') + 'No sequence passed the filter!')
# Get statistics
stat: SampleStats = SampleStats(minscore=minscore,
nt_read=nt_read,
scores=all_scores,
lens=all_length,
seq_read=num_read,
seq_unclas=num_uncl,
seq_filt=filt_seqs,
tid_clas=len(taxids))
# Output statistics
if num_errors:
output.write(
gray(' Seqs fail: ') + red(f'{num_errors:_d}\t') +
gray('(Last error in read ') + red(f'{last_error_read}') +
gray(')\n'))
output.write(
gray(' Seqs read: ') + f'{stat.seq.read:_d}\t' + gray('[') +
f'{stat.nt_read}' + gray(']\n'))
output.write(
gray(' Seqs clas: ') + f'{stat.seq.clas:_d}\t' + gray('(') +
f'{stat.get_unclas_ratio():.2%}' + gray(' unclassified)\n'))
output.write(
gray(' Seqs pass: '******'{stat.seq.filt:_d}\t' + gray('(') +
f'{stat.get_reject_ratio():.2%}' + gray(' rejected)\n'))
output.write(
gray(' Scores: min = ') + f'{stat.sco.mini:.1f}' + gray(', max = ') +
f'{stat.sco.maxi:.1f}' + gray(', avr = ') + f'{stat.sco.mean:.1f}\n')
output.write(
gray(' Length: min = ') + f'{stat.len.mini}' + gray(', max = ') +
f'{stat.len.maxi}' + gray(', avr = ') + f'{stat.len.mean}\n')
output.write(
gray(' TaxIds: by classifier = ') + f'{stat.tid.clas}' +
gray(', by filter = ') + f'{stat.tid.filt}\n')
# Select score output
out_scores: Dict[Id, Score]
if scoring is Scoring.SHEL:
out_scores = {tid: Score(mean(all_scores[tid])) for tid in all_scores}
elif scoring is Scoring.LENGTH:
out_scores = {tid: Score(mean(all_length[tid])) for tid in all_length}
elif scoring is Scoring.LOGLENGTH:
out_scores = {
tid: Score(log10(mean(all_length[tid])))
for tid in all_length
}
elif scoring is Scoring.NORMA:
scores: Dict[Id, Score] = {
tid: Score(mean(all_scores[tid]))
for tid in all_scores
}
lengths: Dict[Id, Score] = {
tid: Score(mean(all_length[tid]))
for tid in all_length
}
out_scores = {
tid: Score(scores[tid] / lengths[tid] * 100)
for tid in scores
}
else:
print(red('ERROR!'), f' Centrifuge: Unsupported Scoring "{scoring}"')
raise Exception('Unsupported scoring')
# Return
return output.getvalue(), stat, counts, out_scores
def process_report(
*args, **kwargs
) -> Tuple[Sample, TaxTree, SampleDataByTaxId, SampleStats, Err]:
"""
Process Centrifuge/Kraken report files (to be usually called in parallel!).
"""
# TODO: Full review to report support
# Recover input and parameters
filerep: Filename = args[0]
taxonomy: Taxonomy = kwargs['taxonomy']
mintaxa: int = kwargs['mintaxa']
collapse: bool = taxonomy.collapse
including: Set[TaxId] = taxonomy.including
excluding: Set[TaxId] = taxonomy.excluding
debug: bool = kwargs['debug']
output: io.StringIO = io.StringIO(newline='')
def vwrite(*args):
"""Print only if verbose/debug mode is enabled"""
if kwargs['debug']:
output.write(' '.join(str(item) for item in args))
sample: Sample = Sample(filerep)
# Read Centrifuge/Kraken report file to get abundances
log: str
abundances: Counter[TaxId]
log, abundances, _ = read_report(filerep)
output.write(log)
# Remove root counts, in case
if kwargs['root']:
vwrite(gray('Removing'), abundances[ROOT], gray('"ROOT" reads... '))
abundances[ROOT] = 0
vwrite(green('OK!'), '\n')
# Build taxonomy tree
output.write(' \033[90mBuilding taxonomy tree...\033[0m')
tree = TaxTree()
tree.grow(taxonomy=taxonomy,
counts=abundances) # Grow tax tree from root node
output.write('\033[92m OK! \033[0m\n')
# Prune the tree
output.write(' \033[90mPruning taxonomy tree...\033[0m')
tree.prune(mintaxa, None, collapse, debug)
tree.shape()
output.write('\033[92m OK! \033[0m\n')
# Get the taxa with their abundances and taxonomical levels
output.write(' \033[90mFiltering taxa...\033[0m')
new_abund: Counter[TaxId] = col.Counter()
new_accs: Counter[TaxId] = col.Counter()
ranks: Ranks = Ranks({})
tree.get_taxa(abundance=new_abund,
accs=new_accs,
ranks=ranks,
mindepth=0,
maxdepth=0,
include=including,
exclude=excluding)
new_abund = +new_abund # remove zero and negative counts
if including or excluding: # Recalculate accumulated counts
new_tree = TaxTree()
new_tree.grow(taxonomy, new_abund) # Grow tree with new abund
new_tree.shape()
new_abund = col.Counter() # Reset abundances
new_accs = col.Counter() # Reset accumulated
new_tree.get_taxa(new_abund, new_accs) # Get new accumulated counts
out: SampleDataByTaxId = SampleDataByTaxId()
out.set(counts=new_abund, ranks=ranks, accs=new_accs)
output.write('\033[92m OK! \033[0m\n')
print(output.getvalue())
sys.stdout.flush()
return sample, tree, out, SampleStats(), Err.NO_ERROR