def initialize_polypeptides( log_fh, fasta_file ):
'''
Reads a FASTA file of (presumably) polypeptide sequences and creates a dict of Polypeptide
objects, keyed by ID, with bioannotation.FunctionalAnnotation objects attached.
'''
seqs = biocodeutils.fasta_dict_from_file( fasta_file )
polypeptides = dict()
for seq_id in seqs:
polypeptide = biothings.Polypeptide( id=seq_id, length=len(seqs[seq_id]['s']), residues=seqs[seq_id]['s'] )
annotation = bioannotation.FunctionalAnnotation(product_name=DEFAULT_PRODUCT_NAME)
log_fh.write("INFO: {0}: Set initial product name to '{1}'\n".format(seq_id, DEFAULT_PRODUCT_NAME))
polypeptide.annotation = annotation
polypeptides[seq_id] = polypeptide
return polypeptides
def get_gff3_features(gff3_file, assemblies=None):
'''
Parses the passed GFF3 file and returns two dicts, loaded with biocode.biothings objects:
1. The first dict are the Assembly objects, keyed on assembly ID. Each Assembly has all of the
children populated, so you can fully recover gene, RNA, exon and CDS features iterating on
the assembly.
2. The second dist is a flat structure of all the descendent feature objects of the Assemblies
keyed by the feature IDs.
See the documentation for each feature type in biocode.biothings for more info
'''
if assemblies is None:
assemblies = dict()
features = dict()
# these are related to parsing any embedded FASTA
in_fasta_section = False
is_assembly_fasta = False
current_fasta_id = None
for line in open(gff3_file):
#print("INFO: processing line: {0}".format(line))
if in_fasta_section == True:
m = re.search('>(\S+)\s*(.*)', line)
if m:
current_fasta_id = m.group(1)
if current_fasta_id in assemblies:
is_assembly_fasta = True
else:
is_assembly_fasta = False
else:
if is_assembly_fasta == True:
# must be a sequence line for an assembly
# python 2.6+ makes string concatenation amortized O(n)
# http://stackoverflow.com/a/4435752/1368079
assemblies[current_fasta_id].residues += str(line.rstrip())
assemblies[current_fasta_id].length = len(
assemblies[current_fasta_id].residues)
continue
elif line.startswith("##FASTA"):
# all data to the end of the file must be FASTA
in_fasta_section = True
continue
# ignore all other comments
if line.startswith('#'):
continue
cols = line.split("\t")
if len(cols) != 9:
continue
mol_id = cols[0]
# initialize this assembly if we haven't seen it yet
if mol_id not in assemblies:
assemblies[mol_id] = biothings.Assembly(id=mol_id, residues='')
current_assembly = assemblies[mol_id]
rfmin = int(cols[3]) - 1
rfmax = int(cols[4])
rstrand = None
atts = column_9_dict(cols[8])
feat_id = atts.get('ID')
parent_id = atts.get('Parent')
parent_feat = None
# sanity check
if rfmin > rfmax:
raise Exception(
"ERROR: Coordinates in GFF for feature id {0} appear to be reversed and violate GFF3 specification: {1} > {2}"
.format(feat_id, cols[3], cols[4]))
if 'locus_tag' in atts:
locus_tag = atts['locus_tag']
else:
locus_tag = None
# shared features is not yet supported
if isinstance(parent_id, list):
raise Exception(
"This line contains a shared feature with multiple parents. This isn't yet supported:\n{0}"
.format(line))
if parent_id is not None:
if parent_id in features:
parent_feat = features[parent_id]
else:
raise Exception(
"Error in GFF3: Parent {0} referenced by a child feature before it was defined"
.format(parent_id))
if cols[6] == '-':
rstrand = -1
elif cols[6] == '+':
rstrand = 1
else:
rstrand = 0
if cols[2] == 'gene':
gene = biothings.Gene(id=feat_id, locus_tag=locus_tag)
gene.locate_on(target=current_assembly,
fmin=rfmin,
fmax=rfmax,
strand=rstrand)
features[feat_id] = gene
current_assembly.add_gene(gene)
elif cols[2] == 'mRNA':
mRNA = biothings.mRNA(id=feat_id,
parent=parent_feat,
locus_tag=locus_tag)
mRNA.locate_on(target=current_assembly,
fmin=rfmin,
fmax=rfmax,
strand=rstrand)
parent_feat.add_mRNA(mRNA)
features[feat_id] = mRNA
elif cols[2] == 'rRNA':
rRNA = biothings.rRNA(id=feat_id,
parent=parent_feat,
locus_tag=locus_tag)
rRNA.locate_on(target=current_assembly,
fmin=rfmin,
fmax=rfmax,
strand=rstrand)
parent_feat.add_rRNA(rRNA)
features[feat_id] = rRNA
elif cols[2] == 'tRNA':
tRNA = biothings.tRNA(id=feat_id,
parent=parent_feat,
locus_tag=locus_tag)
tRNA.locate_on(target=current_assembly,
fmin=rfmin,
fmax=rfmax,
strand=rstrand)
parent_feat.add_tRNA(tRNA)
features[feat_id] = tRNA
elif cols[2] == 'exon':
exon = biothings.Exon(id=feat_id, parent=parent_feat)
exon.locate_on(target=current_assembly,
fmin=rfmin,
fmax=rfmax,
strand=rstrand)
parent_feat.add_exon(exon)
features[feat_id] = exon
elif cols[2] == 'CDS':
CDS = biothings.CDS(id=feat_id, parent=parent_feat)
CDS.locate_on(target=current_assembly,
fmin=rfmin,
fmax=rfmax,
strand=rstrand)
parent_feat.add_CDS(CDS)
features[feat_id] = CDS
elif cols[2] == 'polypeptide':
polypeptide = biothings.Polypeptide(id=feat_id, parent=parent_feat)
polypeptide.locate_on(target=current_assembly,
fmin=rfmin,
fmax=rfmax,
strand=rstrand)
parent_feat.add_polypeptide(polypeptide)
polypeptide.annotation = parse_annotation_from_column_9(cols[8])
features[feat_id] = polypeptide
elif cols[2] == 'five_prime_UTR':
utr = biothings.FivePrimeUTR(id=feat_id, parent=parent_feat)
utr.locate_on(target=current_assembly,
fmin=rfmin,
fmax=rfmax,
strand=rstrand)
parent_feat.add_five_prime_UTR(utr)
features[feat_id] = utr
elif cols[2] == 'three_prime_UTR':
utr = biothings.ThreePrimeUTR(id=feat_id, parent=parent_feat)
utr.locate_on(target=current_assembly,
fmin=rfmin,
fmax=rfmax,
strand=rstrand)
parent_feat.add_three_prime_UTR(utr)
features[feat_id] = utr
else:
sys.stderr.write("Skipping feature {0} with type {1}\n".format(
feat_id, cols[2]))
continue
features[feat_id].length = rfmax - rfmin
return (assemblies, features)
def main():
parser = argparse.ArgumentParser(
description='Metagenemark GFF -> GFF3 conversion script')
## output file to be written
parser.add_argument('-i',
'--input',
type=str,
required=True,
help='Path to a GFF file created by Metagenemark')
parser.add_argument('-o',
'--output',
type=str,
required=True,
help='Path to an output file to be created')
parser.add_argument('-p',
'--prefix',
type=str,
required=True,
help='Prefix to use in ID generation')
parser.add_argument('-pf',
'--protein_fasta',
type=str,
required=False,
help='Optional protein FASTA to be written')
args = parser.parse_args()
assemblies = dict()
current_assembly = None
# key like 2 = SRS014890.polypeptide.2
polypeptide_lookup = dict()
writing_protein = False
gene = None
mRNAs = dict()
current_sequence = None
current_gene_comment_lines = list()
fout = open(args.output, mode='wt', encoding='utf-8')
fout.write("##gff-version 3\n")
if args.protein_fasta is not None:
protein_out = open(args.protein_fasta, mode='wt', encoding='utf-8')
for line in open(args.input):
if line.startswith("#"):
if line.startswith("##FASTA"):
current_gene_comment_lines.append("#{0}".format(line))
elif line.startswith("##end-Protein"):
writing_protein = False
current_gene_comment_lines.append(line)
# since we're already doing our own header, don't duplicate the old one
elif line.startswith("##gff-version"):
continue
else:
if line.startswith("##Protein "):
m = re.match("##Protein (\d+)", line)
if m:
writing_protein = True
protein_out.write(">{0}\n".format(
polypeptide_lookup[m.group(1)]))
else:
raise Exception(
"ERROR: Expected line to match: ##Protein N")
elif writing_protein == True:
protein_out.write(line[2:])
current_gene_comment_lines.append(line)
else:
cols = line.split("\t")
if len(cols) != 9:
continue
mol_id = cols[0]
mol_id_m = re.match('^(\S+) ', mol_id)
if mol_id_m:
print("MATCH!")
mol_id = mol_id_m.group(1)
feat_type = cols[2]
## we expect only gene types here
if feat_type not in ['gene', 'CDS']:
raise Exception(
"ERROR: expected only 'gene' or 'CDS' feature types as input (depending on metagenemark version)."
)
m_gene = re.match('gene_id[ =](\d+)', cols[8])
if m_gene:
gene_num = m_gene.group(1)
else:
raise Exception(
"ERROR: expected 9th column to have gene ids like: gene_id 5"
)
## initialize this assembly if we haven't seen it yet
if mol_id not in assemblies:
assemblies[mol_id] = biothings.Assembly(id=mol_id)
current_assembly = assemblies[mol_id]
gene = biothings.Gene(
id="{0}.gene.{1}".format(args.prefix, gene_num))
gene.locate_on(target=current_assembly,
fmin=int(cols[3]) - 1,
fmax=int(cols[4]),
strand=cols[6])
mRNA = biothings.mRNA(id="{0}.mRNA.{1}".format(
args.prefix, gene_num),
parent=gene.id)
mRNA.locate_on(target=current_assembly,
fmin=int(cols[3]) - 1,
fmax=int(cols[4]),
strand=cols[6])
gene.add_mRNA(mRNA)
CDS = biothings.CDS(id="{0}.CDS.{1}".format(args.prefix, gene_num),
parent=mRNA.id)
CDS.locate_on(target=current_assembly,
fmin=int(cols[3]) - 1,
fmax=int(cols[4]),
strand=cols[6],
phase=int(cols[7]))
mRNA.add_CDS(CDS)
exon = biothings.Exon(id="{0}.exon.{1}".format(
args.prefix, gene_num),
parent=mRNA.id)
exon.locate_on(target=current_assembly,
fmin=int(cols[3]) - 1,
fmax=int(cols[4]),
strand=cols[6])
mRNA.add_exon(exon)
polypeptide_id = "{0}.polypeptide.{1}".format(
args.prefix, gene_num)
polypeptide = biothings.Polypeptide(id=polypeptide_id,
parent=mRNA.id)
polypeptide.locate_on(target=current_assembly,
fmin=int(cols[3]) - 1,
fmax=int(cols[4]),
strand=cols[6])
mRNA.add_polypeptide(polypeptide)
polypeptide_lookup[gene_num] = polypeptide_id
gene.print_as(fh=fout, source='GeneMark.hmm', format='gff3')
fout.write("".join(current_gene_comment_lines))
current_gene_comment_lines = list()
def main():
flawed_gff_file = 'canonical.flawed.gff3'
ilri_gff = 'Theileria-all-Theileria1_ourids.gff'
source = 'GenBank'
out_gff = 'canonical.corrected.gff3'
fout = open(out_gff, mode='wt', encoding='utf-8')
fout.write("##gff-version 3\n")
(assemblies, features) = biocodegff.get_gff3_features(flawed_gff_file)
print("INFO: loaded {0} assemblies and {1} features".format(
len(assemblies), len(features)))
polypeptides = dict()
for line in open(ilri_gff):
cols = line.split("\t")
if len(cols) != 9 or cols[2] != 'polypeptide':
continue
id = biocodegff.column_9_value(cols[8], 'ID')
parent = biocodegff.column_9_value(cols[8], 'Parent')
polypeptides[parent] = biothings.Polypeptide(id=id, parent=parent)
polypeptides[parent].locate_on(target=assemblies[cols[0]],
fmin=int(cols[3]) - 1,
fmax=int(cols[4]),
strand=cols[6])
print("DEBUG: loaded {0} polypeptides from ILRI file".format(
len(polypeptides)))
for assembly_id in assemblies:
for gene in assemblies[assembly_id].genes():
for mRNA in gene.mRNAs():
if mRNA.id not in polypeptides:
print(
"DEBUG: {0} not found as a parent to any polypeptide".
format(mRNA.id))
else:
polypeptide = polypeptides[mRNA.id]
# pull this outside of the iteration since iterating might delete some
CDSs = mRNA.CDSs()
for CDS in CDSs:
keep = True
if CDS < polypeptide:
mRNA.delete_CDS(CDS)
elif CDS <= polypeptide:
CDS.location().fmin = polypeptide.location().fmin
if CDS > polypeptide:
mRNA.delete_CDS(CDS)
elif CDS >= polypeptide:
CDS.location().fmax = polypeptide.location().fmax
#print("WARN: found a CDS {0}:{1}-{2} outside the range of the polypeptide {3}:{4}-{5}".format( \
# CDS.id, CDS.locations[0].fmin, CDS.locations[0].fmax, \
# polypeptide.id, polypeptide.locations[0].fmin, polypeptide.locations[0].fmax))
gene.print_as(fh=fout, source=source, format='gff3')