def close(self,commitData=True): """Close method which performs updating the docstrings and creating the schema objects.""" print "# Finalizing mapping schema(s)..." if self._closed == True: return # UPDATE DOC STRING FOR forward MAPPING self.Mf.__doc__ = "Mapping resource (forward) between annotations %s and %s" % (self.Mf.sourceDB._persistent_id, self.Mf.targetDB._persistent_id) #UPDATE OUR METABASE WITH THE RESOURCE STRING FOR THE MAPPING worldbase.add_resource(self.resourceString+"_forward", self.Mf) # FOR forward MAPPING forward_bindAttrs = (self.forwardAttr, self.inverseAttr) # self.inverseAttr is either None or set to an appropriate inverse attribute relationF = metabase.OneToManyRelation(self.Mf.sourceDB, self.Mf.targetDB, bindAttrs=forward_bindAttrs) relationF.__doc__ = "Mapping schema (forward) between annotations %s and %s" % (self.Mf.sourceDB._persistent_id, self.Mf.targetDB._persistent_id) # UPDATE OUR SCHEMA WITH THE RESOURCE STRING FOR THE MAPPING worldbase.add_schema(self.resourceString+"_forward", relationF) # HANDLE REVERSE MAPPING AND SCHEMA relationR = None if self.Mr != None: # UPDATE DOC STRING FOR reverse MAPPING self.Mr.__doc__ = "Mapping resource (reverse) between annotations %s and %s" % (self.Mr.sourceDB._persistent_id, self.Mr.targetDB._persistent_id) #UPDATE OUR METABASE WITH THE RESOURCE STRING FOR THE MAPPING worldbase.add_resource(self.resourceString+"_reverse", self.Mr) # FOR reverse MAPPING reverse_bindAttrs = (self.reverseAttr, None) relationR = metabase.OneToManyRelation(self.Mr.sourceDB, self.Mr.targetDB, bindAttrs=reverse_bindAttrs) relationR.__doc__ = "Mapping schema (reverse) between annotations %s and %s" % (self.Mr.sourceDB._persistent_id, # Use self.Mf for consistent self.Mr.targetDB._persistent_id) # doc strings with forward mapping # UPDATE OUR SCHEMA WITH THE RESOURCE STRING FOR THE MAPPING worldbase.add_schema(self.resourceString+"_reverse", relationR) if(commitData==True): print "# Committing to worldbase: (1) %s (2) %s" % (str(self.Mf.__doc__),str(self.Mr.__doc__)) worldbase.commit() print "# Closing mapping object(s)" # FLUSH MAPPING(S) TO PERSISTENT STORAGE self.Mf.close() if self.Mr: self.Mr.close() # Set closed flag self._closed = True
def test_download(self): "Downloading of gzipped file using pygr.Data" url = SourceURL('http://www.doe-mbi.ucla.edu/~leec/test.gz') url.__doc__ = 'test download' worldbase.add_resource('Bio.Test.Download1', url) worldbase.commit() # performs the download fpath = worldbase.Bio.Test.Download1() h = testutil.get_file_md5(fpath) self.assertEqual(h.hexdigest(), 'f95656496c5182d6cff9a56153c9db73') os.remove(fpath)
def test_download(self): "Downloading of gzipped file using worldbase" url = SourceURL('http://www.doe-mbi.ucla.edu/~leec/test.gz') url.__doc__ = 'test download' worldbase.add_resource('Bio.Test.Download1', url) worldbase.commit() # performs the download fpath = worldbase.Bio.Test.Download1() h = testutil.get_file_md5(fpath) self.assertEqual(h.hexdigest(), 'f95656496c5182d6cff9a56153c9db73') os.remove(fpath)
def save_NLMSA_downloaders(url, fileFilter=lambda x: x.endswith(".txt.gz"), resourceStem='Bio.MSA.UCSC.', fileDocumenter=None, fileNamer=None): 'save NLMSA downloader / builder objects for a set of downloadable textdump files' if fileDocumenter is None: fileDocumenter = lambda x: 'NLMSA alignment '+x if fileNamer is None: # a default resource naming function fileNamer = lambda x:resourceStem+x[:-3] # remove .gz suffix from pygr.nlmsa_utils import NLMSABuilder from pygr.downloader import SourceURL d = catalog_downloads(url, fileFilter, fileNamer, fileDocumenter, SourceURL) for resID,o in d.items(): nlmsa = NLMSABuilder(o) nlmsa.__doc__ = fileDocumenter(resID) d[resID[:-4]] = nlmsa # remove .txt suffix from pygr import worldbase worldbase.add_resource(d) worldbase.commit() return d # just in case the user wants to see what was saved
def save_NLMSA_downloaders(url, fileFilter=lambda x: x.endswith(".txt.gz"), resourceStem='Bio.MSA.UCSC.', fileDocumenter=None, fileNamer=None): '''save NLMSA downloader / builder objects for a set of downloadable textdump files''' if fileDocumenter is None: fileDocumenter = lambda x: 'NLMSA alignment ' + x if fileNamer is None: # a default resource naming function fileNamer = lambda x: resourceStem + x[:-3] # remove .gz suffix from pygr.nlmsa_utils import NLMSABuilder from pygr.downloader import SourceURL d = catalog_downloads(url, fileFilter, fileNamer, fileDocumenter, SourceURL) for resID, o in d.items(): nlmsa = NLMSABuilder(o) nlmsa.__doc__ = fileDocumenter(resID) d[resID[:-4]] = nlmsa # remove .txt suffix from pygr import worldbase worldbase.add_resource(d) worldbase.commit() return d # just in case the user wants to see what was saved
def setUp(self): """Set up some testing sequences and features. """ print "# Setting annotation databases, nlmsa and committing to worldbase" tuple_attrdict = dict(id=0, start=1, stop=2, orientation=3) self.genome = worldbase("Bio.Seq.Genome.HUMAN.hg18") # annotation db1 self.annodb1 = annotation.AnnotationDB({}, self.genome, sliceAttrDict=tuple_attrdict) self.annodb1._persistent_id = 'foo1_db' # set up some test slices in an AnnotationDB self.seq_id = "chr1" self.annot1 = self.annodb1.new_annotation('A1', (self.seq_id, 200, 300, 1)) self.annot2 = self.annodb1.new_annotation('B1', (self.seq_id, 100, 150, 1)) self.annot3 = self.annodb1.new_annotation('C1', (self.seq_id, 50, 75, -1)) self.annot4 = self.annodb1.new_annotation('D1', (self.seq_id, 400, 500, 1)) self.annot5 = self.annodb1.new_annotation('E1', (self.seq_id, 600, 700, 1)) # create a nested list from our AnnotationDB # these are our "features" self.nlmsa1 = cnestedlist.NLMSA(pathstem='test.mapping.foo1', mode='w', pairwiseMode=True) for k in self.annodb1: self.nlmsa1.addAnnotation(self.annodb1[k]) self.nlmsa1.build() # annotation db2 self.annodb2 = annotation.AnnotationDB({}, self.genome, sliceAttrDict=tuple_attrdict) self.annodb2._persistent_id = 'foo2_db' # set up some test slices in an AnnotationDB self.seq_id2 = "chr2" self.annot6 = self.annodb2.new_annotation('A2', (self.seq_id2, 200, 300, 1)) self.annot7 = self.annodb2.new_annotation('B2', (self.seq_id2, 100, 150, 1)) self.annot8 = self.annodb2.new_annotation('C2', (self.seq_id2, 50, 75, -1)) self.annot9 = self.annodb2.new_annotation('D2', (self.seq_id2, 400, 500, 1)) self.annot10 = self.annodb2.new_annotation('E2', (self.seq_id2, 600, 700, 1)) # create a nested list from our AnnotationDB # these are our "features" self.nlmsa2 = cnestedlist.NLMSA(pathstem='test.mapping.foo2', mode='w', pairwiseMode=True) for k in self.annodb2: self.nlmsa2.addAnnotation(self.annodb2[k]) self.nlmsa2.build() # update WORLDBASEPATH self.annodb1.__doc__ = 'annodb1 db' self.nlmsa1.__doc__ = 'annodb1 nlmsa' self.annodb2.__doc__ = 'annodb2 db' self.nlmsa2.__doc__ = 'annodb2 nlmsa' worldbase.add_resource('Test.Annotations.annodb1_db',self.annodb1) worldbase.add_resource('Test.Annotations.annodb2_db',self.annodb2) worldbase.add_resource('Test.Annotations.annodb1',self.nlmsa1) worldbase.add_resource('Test.Annotations.annodb2',self.nlmsa2) worldbase.commit()
def main(): """Build an annotation from the given gff file """ usage = """Build and save the annotations defined in the given gff files Saves an annotationDB (representing the file itself) and creates a mapping in the form genome[chromosome][100:200].officialGenes""" parser = optparse.OptionParser("%prog [options] data1.gff [data2.gff ...]\n"+usage) parser.add_option("--genome_resource", '-g', dest="genome_resource", type="string", help="""The pygr resource for the genome, eg, 'Bio.Seq.Genome.TRICA.triCas3'""") #parser.add_option("--annotationDB_resource", '-a', dest="annotationDB_resource", type="string", #help="""Where to save the created annotationDB. eg, #Bio.Annotation.TRICA.triCas3.officialGenes""") parser.add_option("--sqlDB_resource", '-s', dest="sqlDB_resource", type="string", help="""Where to save the created sqlDB and a unique file name eg, Bio.Annotation.TRICA.triCas3.features_sqlDB,gffDB_v1""") parser.add_option("--save_pathstem", '-p', dest="pathstem", type="string", help="""The file to save the resource to, eg, '/home/baldig/projects/genomics/pygrdata/annotations/fly/triCas3_official_genes'""") parser.add_option("--map_resource", '-m', dest="map_resource", type="string", help="""the resource to save the annotationDB->Genome map, saved both to worldbase and to worldbase.schema, eg, 'Bio.Annotation.TRICA.triCas3.BeetleBase.officialGenesMap""") parser.add_option("--bind_attribute", '-b', dest="bind_attribute", type="string", help="""The attribute to access annotationDB from genome region, eg, 'officialGenes' would be accessible via triCas3['ChLG2'][100:200].officialGenes Default is not to bind an attribute to genome""") (opts, args) = parser.parse_args() if len(args) < 1: parser.print_help() print 'Please specify at least one gff file to read' sys.exit(-1) if None in [opts.genome_resource, opts.pathstem, opts.map_resource]: parser.print_help() print 'Required options: genome_resource, sqlDB_resource, pathstem, map_resource' sys.exit(-1) if opts.sqlDB_resource.count(',') != 1: parser.print_help() print 'Error: sqlDB_resource must be comma separated string with exactly one comma.' else: opts.sqlDB_resource = opts.sqlDB_resource.split(',') try : w = worldbase(opts.sqlDB_resource[0]) parser.print_help() print "Warning: sqlDB_resource already exists. Please select a new name." exit(-1) except WorldbaseNotFoundError: pass print '# Loading original genome db' genome = worldbase(opts.genome_resource) #annotDB = annotation.AnnotationDB(None, genome, opts.bind_attribute, #filename=opts.pathstem + '_annotDB', mode='c', verbose=False) sqlDB = sqlgraph.SQLiteServerInfo('%s/%s.sqlite' %(opts.pathstem,opts.sqlDB_resource[1])) gff2lite = simpleGFF2PygrSQLite(sqlDB) nlmsa = cnestedlist.NLMSA(opts.pathstem, 'w', pairwiseMode=True, bidirectional=False) for filename in args: print '# adding to sqlDB from %s' % filename gff2lite.update(filename) tableNames = gff2lite.getTableNames() for table in tableNames: #for row in read_for_pygr(fileIn): #curAnnot = annotDB.new_annotation(index, row) #nlmsa.addAnnotation(curAnnot) #index += 1 #annotDB.close() # Flush annotation data to disk print '# building NLMSA from all gff files' nlmsa.build(saveSeqDict=True) print '# saving annotationDB and NLMSA to worldbase as %s and %s' % (opts.annotationDB_resource, opts.map_resource) annotDB.__doc__ = 'Combined gff annotationDB from files %s on genome %s' % (', '.join(args), opts.genome_resource) nlmsa.__doc__ = 'Mapping of %s, from gff files %s onto genome %s' % (opts.annotationDB_resource, ', '.join(args), opts.genome_resource) worldbase.add_resource(opts.annotationDB_resource, annotDB) worldbase.add_resource(opts.map_resource, nlmsa) if opts.bind_attribute: print '# saving worldbase schema with bindAttrs=(%s)' % opts.bind_attribute genome_annotDB_relation = metabase.ManyToManyRelation(genome, annotDB, bindAttrs=(opts.bind_attribute,)) genome_annotDB_relation.__doc__ = 'GFF based mapping from %s to genome %s' % (opts.annotationDB_resource, opts.genome_resource) worldbase.add_schema('%s' % opts.map_resource, genome_annotDB_relation) print '# committing worldbase resources' worldbase.commit() if __name__ == "__main__": main()
def main(): """ Load the given csv file into an sqlite table, saving an annotationDB and an NLMSA version of the original file """ parser = optparse.OptionParser("%prog [options] infile.csv\n"+main.__doc__) parser.add_option("--datapath", '-p', dest="datapath", type="string", default='/home/shared/pygrdata/annotations/HUMAN/hg18', help="""Sets the datafile path. Default=%default""") parser.add_option("--table_name", '-t', dest="table_name", type="string", help="""The resource table's name and data stem, e.g., refGene => datapath/refGene.sqlite """) parser.add_option("--genome", '-g', dest="genome_resource", type="string", default='hg18', help="""The pygr resource for the genome, default=%default""") parser.add_option("--save_resource", '-r', dest="save_resource", type="string", help="""Where to save the created annotationDB and NLMSA. eg, Bio.Annotation.HUMAN.hg18.MotifMap.M0001""") parser.add_option("--bind_attribute", '-b', dest="bind_attribute", type="string", help="""The attribute to access annotationDB from genome region, eg, 'officialGenes' would be accessible via triCas3['ChLG2'][100:200].officialGenes Default is not to bind an attribute to genome""") parser.add_option("--slice_attrs", '-s', dest="slice_attrs", type="string", default='dict(id="chromosome", start="start", stop="stop", orientation="orientation")', help="""dictionary providing aliases in csv file for id, start, stop, etc. default=%default'""") parser.add_option("--bed_format", dest="bed_format", action='store_true', help="""csv file is in BED file format, without headers.""") opts, args = parser.parse_args() if len(args) < 1: parser.print_help() print 'Please specify at least one csv file to read' sys.exit(-1) if None in [opts.save_resource, opts.table_name]: parser.print_help() print 'Required options: save_resource, table_name' sys.exit(-1) fileIn = open(args[0]) if not opts.bed_format: reader = csv.DictReader(fileIn, delimiter='\t') else: fileIn = itertools.ifilter(bedCommentFilter, fileIn) reader = csv.DictReader(fileIn, delimiter='\t', fieldnames=['chromosome', 'start', 'stop'], restkey='junkData') fieldnames = reader.fieldnames print fieldnames print '# Loading genome %s' % opts.genome_resource genome = getGenome(opts.genome_resource) opts.table_name = opts.table_name.replace('.','_') # SQL interprets . as membership tablePath = os.path.join(opts.datapath,opts.table_name + '.sqlite') print '# Creating sqlite table for %s at %s' % (opts.table_name, tablePath) dataTable = convertBedToSQLite(reader, opts.table_name, fieldNames=fieldnames) print '# Making AnnotationDB and NLMSA...' annotDB = annotation.AnnotationDB(dataTable, genome, annotationType=opts.table_name+':', sliceAttrDict=eval(opts.slice_attrs)) annotDB.__doc__ = 'AnnotationDB for %s on %s' % (opts.table_name, opts.genome_resource) msaName = os.path.join(opts.datapath, opts.table_name + '_') annotMap = makeNLMSA([annotDB], dataPath=msaName) print '# Saving results to worldbase as %s and %s...' % (opts.save_resource, opts.save_resource+'_db') worldbase.add_resource(opts.save_resource, annotMap) worldbase.add_resource(opts.save_resource+'_db', annotDB) worldbase.commit()
def main(): """Build an annotation from the given gff file """ usage = """Build and save the annotations defined in the given gff files Saves an annotationDB (representing the file itself) and creates a mapping in the form genome[chromosome][100:200].officialGenes""" parser = optparse.OptionParser("%prog [options] data1.gff [data2.gff ...]\n"+usage) parser.add_option("--genome_resource", '-g', dest="genome_resource", type="string", help="""The pygr resource for the genome, eg, 'Bio.Seq.Genome.TRICA.triCas3'""") parser.add_option("--annotationDB_resource", '-a', dest="annotationDB_resource", type="string", help="""Where to save the created annotationDB. eg, Bio.Annotation.TRICA.triCas3.officialGenes""") parser.add_option("--save_pathstem", '-p', dest="pathstem", type="string", help="""The file to save the exon resource to, eg, '/home/baldig/projects/genomics/pygrdata/annotations/fly/triCas3_official_genes'""") parser.add_option("--map_resource", '-m', dest="map_resource", type="string", help="""the resource to save the annotationDB->Genome map, saved both to worldbase and to worldbase.schema, eg, 'Bio.Annotation.TRICA.triCas3.BeetleBase.officialGenesMap""") parser.add_option("--bind_attribute", '-b', dest="bind_attribute", type="string", help="""The attribute to access annotationDB from genome region, eg, 'officialGenes' would be accessible via triCas3['ChLG2'][100:200].officialGenes Default is not to bind an attribute to genome""") (opts, args) = parser.parse_args() if len(args) < 1: parser.print_help() print 'Please specify at least one gff file to read' sys.exit(-1) if None in [opts.genome_resource, opts.annotationDB_resource, opts.pathstem, opts.map_resource]: parser.print_help() print 'Required options: genome_resource, annotationDB_resource, pathstem, map_resource' sys.exit(-1) print '# Loading original genome db' genome = worldbase(opts.genome_resource) annotDB = annotation.AnnotationDB(None, genome, opts.bind_attribute, filename=opts.pathstem + '_annotDB', mode='c', verbose=False) nlmsa = cnestedlist.NLMSA(opts.pathstem, 'w', pairwiseMode=True, bidirectional=False) index = 0 # unique ID used in annotationD for filename in args: print '# adding to annotationDB from %s' % filename fileIn = open(filename) for row in read_for_pygr(fileIn): curAnnot = annotDB.new_annotation(index, row) nlmsa.addAnnotation(curAnnot) index += 1 annotDB.close() # Flush annotation data to disk print '# building NLMSA from all gff files' nlmsa.build(saveSeqDict=True) print '# saving annotationDB and NLMSA to worldbase as %s and %s' % (opts.annotationDB_resource, opts.map_resource) annotDB.__doc__ = 'Combined gff annotationDB from files %s on genome %s' % (', '.join(args), opts.genome_resource) nlmsa.__doc__ = 'Mapping of %s, from gff files %s onto genome %s' % (opts.annotationDB_resource, ', '.join(args), opts.genome_resource) worldbase.add_resource(opts.annotationDB_resource, annotDB) worldbase.add_resource(opts.map_resource, nlmsa) if opts.bind_attribute: print '# saving worldbase schema with bindAttrs=(%s)' % opts.bind_attribute genome_annotDB_relation = metabase.ManyToManyRelation(genome, annotDB, bindAttrs=(opts.bind_attribute,)) genome_annotDB_relation.__doc__ = 'GFF based mapping from %s to genome %s' % (opts.annotationDB_resource, opts.genome_resource) worldbase.add_schema('%s' % opts.map_resource, genome_annotDB_relation) print '# committing worldbase resources' worldbase.commit()