parser.add_argument('-o', help='Output JSON bead file.') args = parser.parse_args() inputFile = args.i maxee = args.maxee outputFile = args.o mockRaw = [] with open(inputFile, 'r') as f: for line in f: mockRaw.append(json.loads(line)) print('Find {0} bead in the file'.format(len(mockRaw))) #%% For each bead, remove low quality read and duplicated reads # Then write to a new JSON file i = 0 for item in mockRaw: beadQC = bead.maxEE(item, maxee=maxee) beadDerep = bead.derep(beadQC) beadProcessed = bead.beadSequence(beadDerep) if len(beadProcessed.fragments) > 0: # Only save bead with fragments left. i += 1 beadProcessed.jsonWrite(outputFile, mode='a') print('{0} beads pass the QC and derep'.format(i)) #%% The JSON file can be read in by line # A single line can be converted to a bead Class ''' beadList = [] with open(outputFile, 'r') as f: for line in f: beadList.append(bead.beadSequence(json.loads(line))) '''
parser = argparse.ArgumentParser() parser.add_argument('-i', help='Input beadJson file') parser.add_argument('-o', help='Output short fragment count per bead') parser.add_argument('-d', help='Output distribution of count') args = parser.parse_args() inputFile = args.i outputFile = args.o outputDist = args.d count = {} dist = {} with open(inputFile, 'r') as f: for line in f: currentBead = bead.beadSequence(json.loads(line.strip('\n'))) fragmentCount = len(currentBead.fragments) count[currentBead.barcode] = fragmentCount try: dist[fragmentCount] += 1 except KeyError: dist[fragmentCount] = 1 # Output with open(outputDist, 'w') as f: f.write('{0}\t{1}\n'.format('FragmentCount', 'Frequency')) for key, value in dist.items(): f.write('{0}\t{1}\n'.format(str(key), str(value))) with open(outputFile, 'w') as f: f.write('{0}\t{1}\n'.format('Barcode', 'FragmentCount'))
parser.add_argument('-k', default=21, type=int, help='Kmer size default = 21') args = parser.parse_args() inputFile = args.i outputRaw = args.rawout outputThreshold = args.tout threshold = args.threshold kmerSize = args.k # Read in JSON-Bead file #Calculate kmer pools for all beads kmerPool = [] beadCount = 0 with open(inputFile, 'r') as f: for line in f: b = bead.beadSequence(json.loads(line)) kmerPool.append(kmer.kmerCount(b, kmerSize)) beadCount += 1 print('Found {0} beads.'.format(beadCount)) # Calculate kmer distance for all pairs edge = [] edgeThreshold = [] n1 = 0 n2 = 0 for pair in combinations(kmerPool, 2): D = kmer.kmerDistance((pair[0].set, pair[1].set)).mashDistance() edge.append((pair[0].barcode, pair[1].barcode, D)) if threshold[0] <= D <= threshold[1]: edgeThreshold.append((pair[0].barcode, pair[1].barcode, D)) n2 += 1
def main(): # Read in JSON-Bead file #Calculate kmer pools for all beads kmerPool = [] beadCount = 0 with open(inputFile, 'r') as f: for line in f: b = bead.beadSequence(json.loads(line)) kmerPool.append(kmer.kmerCount(b, kmerSize)) beadCount += 1 print('Found {0} beads.'.format(beadCount)) # Setup the parallel enviroment # Create shared list for store edge list and progress counter manager = Manager() edge = manager.list([[]] * job) # n list for edge list count = manager.list([0] * job) # n list for count print('Starting mash distance ...') # Divide the kmer pair pool pairPool = [] for pair in combinations(kmerPool, 2): pairPool.append(pair) size = len(pairPool) print('Total is {0} pairs.'.format(size)) step = size // job print('Step is {0}'.format(step)) start = 0 workers = [] print(len(pairPool)) for i in range(job): if i + 1 < job: # not the last job workers.append( Process(target=kmerDistanceWorker, args=(pairPool[start:start + step], edge, i, count))) start += step print('Start change to {0}'.format(start)) else: workers.append( Process(target=kmerDistanceWorker, args=(pairPool[start:], edge, i, count))) print('Starting %i jobs ...' % job) count_worker = 1 for j in workers: j.start() print('Starting thread No. %i ...' % count_worker) count_worker += 1 job_alive = True while job_alive: time.sleep(0.01) job_alive = False for j in workers: if j.is_alive(): job_alive = True progress = str(sum(count) / size * 100) + "\r" sys.stderr.write(progress) #print(len(edge[0])) for j in workers: j.join() print('Finished dereplicating.') with open(outputRaw, 'w') as f: f.write('Source\tTarget\tDistance\n') for item in edge: for line in item: f.write('{0}\t{1}\t{2}\n'.format(line[0], line[1], line[2]))
if item[2] > 0.02: f.write('{0}\t{1}\t{2}\n'.format(item[0], item[1], item[2])) ''' #%% ''' Extract bead sequences by module number ''' from metaSeq import io as seqIO from metaSeq import bead module = {} with open('kmer.jcd.0.02.module.txt', 'r') as f: f.readline() for line in f: line = line.strip('\n').split('\t') module[line[0]] = line[1] print(len(module)) cluster = {} for item in list(set(module.values())): cluster[item] = [] print(len(cluster)) beads = seqIO.beadJson('CL100077200_L01.json') for item in beads: b = bead.beadSequence(item) classNumber = module.get(b.barcode, False) if classNumber: cluster[classNumber] += b.fastaSequences() print(len(cluster)) for key, value in cluster.items(): seqIO.write_seqs(value, 'cluster/{0}.fa'.format(key), fastx='a', mode='w')
# -*- coding: utf-8 -*- """ Created on Mon Jul 30 16:47:24 2018 @author: Zewei Song @email: [email protected] """ #%% from __future__ import print_function from __future__ import division import json from metaSeq import io as seqIO from metaSeq import qc as seqQC from metaSeq import bead # Read in the JSON bead file inputFile = 'mock.qc.derep.json' beadList = [] with open(inputFile, 'r') as f: for line in f: beadList.append(bead.beadSequence(json.loads(line))) #%% # Write the first 10 bead into FASTA file i = 1 threshold = 100 for line in beadList: #outputFile = line.barcode + '.fa' line.fastaWrite() i += 1 if i > 10: break
[email protected] [email protected] ------------------------''')) parser.add_argument('-i', help='merged fasta file.') parser.add_argument('-d', help='Output directory.') parser.add_argument( '-z', help='save fa with missing tag(0000) into a seperated directory.') args = parser.parse_args() #%% Try to sample bead with more than 1 fragments inputBeadJson = args.i beadPool = [] with open(inputBeadJson, 'r') as f: for line in f: beadPool.append(bead.beadSequence(json.loads(line))) print('Find {0} beads.'.format(len(beadPool))) #%% Get beads with more than 1 fragment beadPool_1 = [] for item in beadPool: if len(item.fragments) > 1: beadPool_1.append(item) print('Find {0} bead with more than 1 fragments.'.format(len(beadPool_1))) #%% Write these bead to FASTA files outputFolder = args.d code0Folder = args.z for item in beadPool_1: if re.match("0000", item.barcode, 0):