def run(args): EcsEntry.objects.all().delete() parser = EcsFileParser(args[1]) try: inserter = BulkInserter(EcsEntry) while True: entries = parser.next_ecs_entry() if not entries: break for entry in entries: inserter.save(entry) inserter.close() except Exception: inserter.rollback() raise
def testSave(self): self.dummy_obj.save() generic_inserter = BulkInserter(BulkInserterTestModel) generic_inserter.save(self.obj) generic_inserter.save(self.obj2) generic_inserter.close() tests = BulkInserterTestModel.objects.all() eq_(tests[0].int_field, self.obj.int_field) eq_(tests[0].char_field, self.obj.char_field) eq_(tests[0].fk_field, self.obj.fk_field) eq_(tests[1].int_field, self.obj2.int_field) eq_(tests[1].char_field, self.obj2.char_field) eq_(tests[1].fk_field, self.obj2.fk_field)
def run(args):
try:
parser = geovision.text_to_db.sample_parser.SamplefileParser(args[1])
except IOError:
print "Unable to open file", args[1]
sys.exit(1);
if ImportedData.objects.filter(type='sample', data=parser.sample).exists():
print("Warning: Not importing sample %s because it already exists" % parser.sample)
sys.exit(1)
try:
inserter = BulkInserter(Read)
read_entry = parser.next_read()
while read_entry is not None:
inserter.save(read_entry)
read_entry = parser.next_read()
inserter.close()
ImportedData.objects.create(type='sample', data=parser.sample)
except Exception as e:
inserter.rollback()
raise
def run(args):
try:
parser = geovision.text_to_db.db_parser.DbfileParser(args[1], args[2])
except IOError:
print("Unable to open file", args[1])
sys.exit(1);
if ImportedData.objects.filter(type='dbentry', data=parser.source).exists():
print("Warning: Not importing database %s because it already exists" % args[2])
sys.exit(1)
try:
inserter = BulkInserter(DbEntry)
db_entry = parser.next_db_entry()
while db_entry is not None:
inserter.save(db_entry)
db_entry = parser.next_db_entry()
inserter.close()
ImportedData.objects.create(type='dbentry', data=parser.source)
except Exception as e:
inserter.rollback()
raise
def testObjToCsv(self): inserter = BulkInserter(BulkInserterTestModel) eq_(inserter.obj_to_csv(inserter, self.obj), "1$42$Foo$27\n") inserter.close()
#coding: UTF-8
import os
from geovision.viz.models import Blast, BlastExtra
from geovision.viz.models import Read, DbEntry, ImportedData
from geovision.text_to_db.bulk_inserter import BulkInserter, dict_from_kwargs
inserter = BulkInserter(Blast, use_dict=True)
extra_inserter = BulkInserter(BlastExtra, use_dict=True)
def create_blast(db_name, sample_name, filename):
global inserter, extra_inserter
filehandle = open(filename, 'r')
for line in filehandle:
(read_id, db_seq_id, pident, length, mismatch, gapopen, qstart, qend, sstart, send, error_value, bitscore, read_seq, db_seq) = line.split(None)
if db_name == 'uniprot':
db_seq_id = db_seq_id.split('|')[1]
id = inserter.get_id()
inserter.save(dict_from_kwargs(read_id=read_id, database_name=db_name, sample=sample_name, db_entry_id=db_seq_id, bitscore=bitscore, error_value=error_value))
extra_inserter.save(dict_from_kwargs(blast_id=id, pident=pident, length=length, mismatch=mismatch, gapopen=gapopen, qstart=qstart, qend=qend, sstart=sstart, send=send, read_seq=read_seq, db_seq=db_seq))
inserter.close()
extra_inserter.close()
if __name__ == '__main__':
import sys
filename = sys.argv[1]
(sample_name, db_name, rest) = os.path.basename(filename).split('.')
blast_name = sample_name + '.' + db_name
if ImportedData.objects.filter(type='blast', data=blast_name).exists():
print("Not importing blast %s because it already exists" % blast_name)
sys.exit(1)
