def groupinfo(args): # open and read the first line of infile if args.fmt in ['pickle', 'npy']: from seqpy.core.bioio import naltparser from types import SimpleNamespace nalt_args = SimpleNamespace(infile=args.infile, fmt=args.fmt, n=-1) nalt_parser = naltparser.NAltLineParser(nalt_args, with_group=False, with_position=False) samples = nalt_parser.samples elif args.fmt == 'list': with gzopen(args.infile) as f: buf = f.read() samples = buf.split() else: with gzopen(args.infile) as f: samples = f.readline().strip().split() group_parser = grpparser.GroupParser(args) groups = group_parser.assign_groups(samples) total = 0 cout('Groups:') for g in sorted(groups.keys()): c = len(groups[g]) cout(' %3d - %s' % (c, g)) total += c cout('Total: %d samples' % total)
def ralt2nalt(args): ralt_parser = naltparser.NAltLineParser(args, datatype='ralt', with_group=False, with_position=False) region = ralt_parser.parse_whole() # convert to n_alt cerr('[I - converting to nalt format]') cerr('[ M dtype: {}]'.format(region.M.dtype)) region.ralt_to_nalt(hetratio=args.hetratio if not args.major else -1) cerr('[ M dtype: {}]'.format(region.M.dtype)) region.save(args.outfmt, prefixname=args.outfile, autofilename=args.autofilename, with_position=False) return # write to outfile with open(args.outfile, 'w') as outfile: # write header outfile.write(ralt_parser.get_sample_header()) outfile.write('\n') np.savetxt(outfile, region.M, fmt='%d', delimiter='\t') cerr('[I: finish writing to %s' % args.outfile)
def dist2clonalqc(args): # read distance matrix df = pd.read_csv(args.infile, sep='\t') samples = df.columns D = df.values # read quality file or pickled ralt/nalt file if args.datafile: nalt_args = SimpleNamespace(infile=args.datafile, fmt=args.fmt, n=-1) nalt_parser = naltparser.NAltLineParser(nalt_args, with_group=False, with_position=False) region = nalt_parser.parse_whole() qual = np.count_nonzero(region.M == -1, axis=0) else: cexit('ERR: other input file has not been defined') clonal_samples = clonal_index(D, qual.max() - qual, samples, args.threshold) cerr('[I - removing %d clonal samples]' % len(clonal_samples)) if args.outfile: np.savetxt(args.outfile, samples[clonal_samples], fmt='%s')
def nalt2dist(args): with_position = True if args.includepos else False nalt_parser = naltparser.NAltLineParser(args, with_group=False, with_position=with_position) whole_region = nalt_parser.parse_whole() if with_position: with open(args.includepos) as f_posline: poslines = [x.split() for x in f_posline] if poslines[0][0] == 'CHROM' and poslines[0][1] == 'POS': del poslines[0] whole_region.filter_poslines(poslines, inplace=True) # read whole genotype, and release all unused memory cerr('[I - converting to haplotypes]') haplotypes = whole_region.haplotypes() cerr('[I - calculating pairwise dxy for %d samples]' % len(haplotypes)) distm = fastdx.pwdistm(haplotypes, args.countmissing) cerr('[I - writing to %s]' % args.outfile) with open(args.outfile, 'w') as outfile: outfile.write('\t'.join(nalt_parser.parse_samples())) outfile.write('\n') # write the matrix np.savetxt(outfile, distm, delimiter='\t', fmt='%.6f')
def ralt2pickle(args): alt_parser = naltparser.NAltLineParser(args, datatype=args.type, with_group=False, with_position=False) whole_region = alt_parser.parse_whole() whole_region.df_M.to_pickle(args.outfile)
def simxval(args): start_time = time.monotonic() model = get_model(args) logfh = open(args.logfile, 'w') if args.logfile else None cerr('[I - repeats: %d, k-fold: %d, iteration: %d, k: %s]' % (args.repeats, args.fold, args.iter, args.k)) nalt_parser = naltparser.NAltLineParser( args, datatype='nalt') # we need group info nalt_parser.parse_grouping() group_keys = nalt_parser.group_parser.group_keys # remove groups whose samples are less than 3 suitable_groups = set( [ g for g in nalt_parser.group_parser.groups if len(nalt_parser.group_parser.groups[g]) > 2]) cerr('[I - reading %s samples]' % len(nalt_parser.samples)) mask = [False] * len(nalt_parser.samples) for i in range(len(nalt_parser.samples)): if group_keys[i] in suitable_groups: mask[i] = True samples = list(itertools.compress(nalt_parser.samples, mask)) group_keys = np.array(list(itertools.compress(group_keys, mask))) cerr('[I - masking to %d samples]' % len(samples)) region = nalt_parser.parse_whole(mask=mask) if args.mac > 0: cerr('[I - filtering for MAC = %d]' % args.mac) region.filter_mac(args.mac, inplace=True) cerr('[I - preparing haplotypes]') haplotypes = region.haplotypes() k_list = [ int(x) for x in args.k.split(',') ] results, snp_table = validate( model, haplotypes, group_keys, k_list = k_list, repeats = args.repeats, fold = args.fold, iteration = args.iter, procs = args.j, with_snp = (args.outsnp is not None ), logfh = logfh ) results.to_csv(args.outfile, sep='\t', index=False) cerr('[I - writing scores to %s]' % args.outfile) if args.outsnp: import yaml yaml.dump(snp_table, open(args.outsnp, 'w')) cerr('[I - writing SNP table to %s]' % args.outsnp ) if logfh: cerr('[I - writing log to %s]' % args.logfile) cerr('[I - finished in %d secs with %d results]' % (time.monotonic() - start_time, len(results)))
def train(args): model = get_model(args) cerr('[I - train model: %s iteration: %d') nalt_parser = naltparser.NAltLineParser( args, dataype='nalt') # we need group info nalt_parser.parse_grouping() group_keys = nalt_parser.group_parser.group_keys cerr('[I - masking to %d samples]' % len(samples)) region = nalt_parser.parse_whole(mask=mask) haplotypes = region.haplotypes() matrix_profiles = cross_train( haplotypes, group_keys, repeats, fold, )
def nalt2haplotypes(args): nalt_parser = naltparser.NAltLineParser(args, with_group=False) region = nalt_parser.parse_whole() samples = nalt_parser.parse_samples() if args.includepos: with open(args.includepos) as f: poslines = [x.strip().split() for x in f] if poslines[0][0] == 'CHROM' and poslines[0][1] == 'POS': del poslines[0] else: poslines = None if poslines: region.filter_poslines(poslines, inplace=True) # read whole genotype, and release all unused memory cerr('[I - converting to haplotypes]') haplotypes = region.haplotypes() positions = region.P lines = [] for i in range(len(haplotypes)): snps = [] for j in range(len(haplotypes[i])): allel = haplotypes[i][j] if allel == 0: snps.append(positions[j][2]) elif allel == 2: snps.append(positions[j][3]) elif allel < 0: snps.append('X') else: snps.append('N') lines.append((samples[i], ''.join(snps))) if args.outfile: with open(args.outfile, 'w') as f: if args.revfmt: for line in lines: f.write('{}\t{}\n'.format(line[1], line[0])) else: for line in lines: f.write('{}\t{}\n'.format(line[0], line[1]))
def read_samplefile(infile, fmt): if fmt in ['pickle', 'npy']: from seqpy.core.bioio import naltparser from types import SimpleNamespace nalt_args = SimpleNamespace(infile=infile, fmt=fmt, n=-1) nalt_parser = naltparser.NAltLineParser(nalt_args, with_group=False, with_position=False) samples = nalt_parser.samples elif fmt == 'list': with gzopen(infile) as f: buf = f.read() samples = buf.strip().split() else: # only read the first line with gzopen(infile) as f: samples = f.readline().strip().split() return samples
def train(args): # load profiles if exists nalt_parser = naltparser.NAltLineParser(args, datatype='nalt') nalt_parser.parse_grouping() group_keys = nalt_parser.group_parser.group_keys region = nalt_parser.parse_whole() samples = nalt_parser.parse_samples() poslines = [line.split() for line in open(args.includepos)][1:] region.filter_poslines(poslines, inplace=True, sort_position=False) haplotypes = region.haplotypes() #import IPython; IPython.embed() cerr('[I - fitting for {}]'.format(args.code)) classifier = lkest.SNPLikelihoodEstimator() classifier.fit(haplotypes, group_keys) profile = classifier.get_profile() profile.positions = region.P profile.code = args.code profile.remark = args.remark try: with open(args.profile, 'rb') as f: profiles = pickle.load(f) except FileNotFoundError: profiles = {} if args.code in profiles and not args.replace: cexit('ERR: cannot replace ') profiles[args.code] = profile.to_dict() with open(args.profile, 'wb') as f: pickle.dump(profiles, f) cerr('[I - profiles saved to {}]'.format(args.profile))
def nalt2qc(args): """ write to out.imiss & out.lmiss for each sample and SNPS, evaluate: N_MISS F_MISS N_HETS F_HETS out.imiss: SAMPLE N_SNP N_MISS F_MISS N_HETS F_HETS out.lmiss: CHR POS N_SAMPLE N_MISS F_MISS N_HETS F_HETS """ start_time = time.monotonic() nalt_parser = naltparser.NAltLineParser(args, datatype='nalt', with_group=False) samples = nalt_parser.parse_samples() whole = nalt_parser.parse_whole() cerr('[I - reading input file in %s secs]' % (time.monotonic() - start_time)) # create an array for N samples with column: # N_SNP N_MISS N_HETS asamples = np.zeros(shape=(len(samples), 3)) # container for lmiss output chr_pos = [] snps = [] # -1 indicate missing data, 1 indicate heterozygous SNP for pos, n_alt in whole.parse_positions(): # gather imiss data asamples[range(n_alt.size), 0] += 1 asamples[n_alt == -1, 1] += 1 asamples[n_alt == 1, 2] += 1 # gather lmiss data chromosome = pos[0] position = pos[1] n_sample = len(n_alt) n_miss = np.where(n_alt == -1)[0].size n_het = np.where(n_alt == 1)[0].size chr_pos.append((chromosome, position)) snps.append((n_sample, n_miss, n_het)) # create imiss stats imiss = np.zeros(shape=(len(samples), 5)) n_snps = asamples[:, 0] imiss[:, 0] = asamples[:, 0] imiss[:, 1] = asamples[:, 1] imiss[:, 2] = asamples[:, 1] / n_snps imiss[:, 3] = asamples[:, 2] imiss[:, 4] = asamples[:, 2] / n_snps # create lmiss stats snps = np.array(snps) n_samples = snps[:, 0] lmiss = np.zeros(shape=(len(snps), 5)) lmiss[:, 0] = snps[:, 0] lmiss[:, 1] = snps[:, 1] lmiss[:, 2] = snps[:, 1] / n_samples lmiss[:, 3] = snps[:, 2] lmiss[:, 4] = snps[:, 2] / n_samples # output imiss with open('out.imiss', 'w') as iout: iout.write('INDV\tN_SNP\tN_MISS\tF_MISS\tN_HETS\tF_HETS\n') for sample, i in zip(samples, imiss): iout.write('{}\t'.format(sample)) iout.write('{}\n'.format('\t'.join(map(str, i)))) with open('out.lmiss', 'w') as lout: lout.write('CHROM\tPOS\tN_SAMPLE\tN_MISS\tF_MISS\tN_HETS\tF_HETS\n') for (chrom, pos), l in zip(chr_pos, lmiss): lout.write('{}\t{}\t'.format(chrom, pos)) lout.write('{}\n'.format('\t'.join(map(str, l))))
def read_data( args ): nalt_parser = naltparser.NAltLineParser( args, with_group=False, with_position=False) region = nalt_parser.parse_whole() return region.M, np.arange(len(region.M[0])), np.arange(len(region.M))
def ralt2ralt( args ): cerr('[I - reading input files]') start_time = time.monotonic() alt_parser = naltparser.NAltLineParser(args, datatype=args.type) whole_region = alt_parser.parse_whole() samples = alt_parser.parse_samples() if args.posindex: pos_indexes = np.loadtxt(args.posindex, dtype=int) cerr('[I - filtering for %d SNP position]' % len(pos_indexes)) whole_region.filter_positions(pos_indexes) elif args.includepos: with gzopen(args.includepos) as f_posline: poslines = [ x.split() for x in f_posline ] if poslines[0][0] == 'CHROM' and poslines[0][1] == 'POS': del poslines[0] whole_region.filter_poslines(poslines, inplace=True) if args.indvindex: indv_indexes = np.loadtxt(args.indvindex, dtype=int) whole_region.filter_samples(indv_indexes) samples = samples[indv_indexes] if args.excludesample: excluded_samples = np.loadtxt(args.excludesample, dtype=str) excluded_indexes = np.where(np.array(samples) == excluded_samples[:,None])[1] indv_indexes = np.array( list(set( range(len(samples))) - set(excluded_indexes)) ) cerr('[I - excluding %d samples]' % len(excluded_indexes)) whole_region.filter_samples(indv_indexes) samples = samples[indv_indexes] if args.includesample: included_samples = np.loadtxt(args.includesample, dtype=str) included_indexes = np.where(np.array(samples) == included_samples[:,None])[1] cerr('[I - including {} | {} out of {} samples]'.format( len(included_samples), len(included_indexes), len(samples))) whole_region.filter_samples(included_indexes) samples = samples[included_indexes] if args.completesamples: # only output samples without missing SNPs indv_missing = whole_region.get_snp_missingness() complete_indv = np.where(indv_missing == 0.0)[0] #import IPython; IPython.embed() whole_region.filter_samples(complete_indv) samples = samples[complete_indv] if args.mac > 0: whole_region.filter_mac(args.mac) # save to outfile whole_region.save(args.outfmt, prefixname=args.outfile, autofilename=args.autofilename , with_position=True) return if args.autofilename: args.outfile = '%s-%d-%d' % ( 'r' if args.type == 'ralt' else 'n', len(samples), len(whole_region.M) ) cerr('[I - writing to outfiles]') outmatrix = args.outfile + ('.ralt' if args.type == 'ralt' else '.nalt') if args.outfmt == 'pickle': outmatrix = outmatrix + '.pickle.gz' whole_region.df_M.to_pickle(outmatrix) else: outmatrix = outmatrix + '.txt.gz' whole_region.df_M.to_csv(outmatrix, sep='\t', index=False) outpos = args.outfile + '.pos.txt.gz' whole_region.df_P.to_csv(outpos, sep='\t', index=False) cerr('[I - writing to file: %s and %s' % (outmatrix, outpos)) return with open(outmatrix, 'wt') as f_matrix, open(outpos, 'wt') as f_pos: f_matrix.write('\t'.join(samples)) f_matrix.write('\n') np.savetxt(f_matrix, whole_region.M, delimiter='\t', fmt='%4.3f' if args.type == 'ralt' else '%d') f_pos.write('\t'.join(alt_parser.position_parser.header)) np.savetxt(f_pos, whole_region.P, delimiter='\t') cerr('[I - writing to file: %s and %s' % (outmatrix, outpos))