bud_session_maker = prepare_schema_connection(bud, config.BUD_DBTYPE, 'pastry.stanford.edu:1521', config.BUD_DBNAME, config.BUD_SCHEMA, config.BUD_DBUSER, config.BUD_DBPASS) nex_session_maker = prepare_schema_connection(nex, config.NEX_DBTYPE, 'sgd-master-db.stanford.edu:1521', config.NEX_DBNAME, config.NEX_SCHEMA, config.NEX_DBUSER, config.NEX_DBPASS) perf_session_maker = prepare_schema_connection(perf, config.PERF_DBTYPE, 'sgd-db1.stanford.edu:1521', config.PERF_DBNAME, config.PERF_SCHEMA, config.PERF_DBUSER, config.PERF_DBPASS) nex_backend = SGDBackend(config.NEX_DBTYPE, 'sgd-master-db.stanford.edu:1521', config.NEX_DBNAME, config.NEX_SCHEMA, config.NEX_DBUSER, config.NEX_DBPASS, None) # ------------------------------------------ Evidence ------------------------------------------ from src.sgd.model.nex.evidence import Evidence, Geninteractionevidence, Physinteractionevidence from src.sgd.convert.from_bud.evidence import make_interaction_evidence_starter do_conversion(make_interaction_evidence_starter(bud_session_maker, nex_session_maker, 'genetic interactions'), [Json2Obj(Geninteractionevidence), Evidence2NexDB(nex_session_maker, lambda x: x.query(Geninteractionevidence), name='convert.from_bud.evidence.geninteraction', delete_untouched=True, commit_interval=1000, already_deleted=clean_up_orphans(nex_session_maker, Geninteractionevidence, Evidence, 'GENINTERACTION'))]) do_conversion(make_interaction_evidence_starter(bud_session_maker, nex_session_maker, 'physical interactions'), [Json2Obj(Physinteractionevidence), Evidence2NexDB(nex_session_maker, lambda x: x.query(Physinteractionevidence), name='convert.from_bud.evidence.physinteraction', delete_untouched=True, commit_interval=1000, already_deleted=clean_up_orphans(nex_session_maker, Physinteractionevidence, Evidence, 'PHYSINTERACTION'))]) # ------------------------------------------ Perf ------------------------------------------ from src.sgd.model.perf.bioentity_data import BioentityDetails from src.sgd.model.perf.reference_data import ReferenceDetails from src.sgd.model.nex.bioentity import Locus from src.sgd.model.nex.reference import Reference
nex_session_maker = prepare_schema_connection(nex, config.NEX_DBTYPE, 'sgd-master-db.stanford.edu:1521', config.NEX_DBNAME, config.NEX_SCHEMA, config.NEX_DBUSER, config.NEX_DBPASS) perf_session_maker = prepare_schema_connection(perf, config.PERF_DBTYPE, 'sgd-db1.stanford.edu:1521', config.PERF_DBNAME, config.PERF_SCHEMA, config.PERF_DBUSER, config.PERF_DBPASS) nex_backend = SGDBackend(config.NEX_DBTYPE, 'sgd-master-db.stanford.edu:1521', config.NEX_DBNAME, config.NEX_SCHEMA, config.NEX_DBUSER, config.NEX_DBPASS, None) from src.sgd.model.nex.bioitem import Reservedname from src.sgd.model.nex.paragraph import Paragraph, Bioentityparagraph, ParagraphReference from src.sgd.convert.from_bud.paragraph import make_bioentity_paragraph_starter, make_paragraph_reference_starter do_conversion(make_bioentity_paragraph_starter(bud_session_maker, nex_session_maker), [Json2Obj(Bioentityparagraph), Obj2NexDB(nex_session_maker, lambda x: x.query(Bioentityparagraph), name='convert.from_bud.paragraph.bioentity', delete_untouched=True, commit=True, already_deleted=clean_up_orphans(nex_session_maker, Bioentityparagraph, Paragraph, 'BIOENTITY'))]) do_conversion(make_paragraph_reference_starter(nex_session_maker), [Json2Obj(ParagraphReference), Obj2NexDB(nex_session_maker, lambda x: x.query(ParagraphReference), name='convert.from_bud.paragraph_reference', delete_untouched=True, commit=True)]) from src.sgd.model.perf.core import Bioentity as PerfBioentity do_conversion(make_backend_starter(nex_backend, 'all_bioentities', 1000), [Json2CorePerfDB(perf_session_maker, PerfBioentity, name='convert.from_backend.bioentity', commit_interval=1000, delete_untouched=True)]) perf_session_maker = prepare_schema_connection(perf, config.PERF_DBTYPE, 'sgd-db2.stanford.edu:1521', config.PERF_DBNAME, config.PERF_SCHEMA, config.PERF_DBUSER, config.PERF_DBPASS) perf_backend = PerfBackend(config.PERF_DBTYPE, 'sgd-db1.stanford.edu:1521', config.PERF_DBNAME, config.PERF_SCHEMA, config.PERF_DBUSER, config.PERF_DBPASS, None)
bud_session_maker = prepare_schema_connection(bud, config.BUD_DBTYPE, 'pastry.stanford.edu:1521', config.BUD_DBNAME, config.BUD_SCHEMA, config.BUD_DBUSER, config.BUD_DBPASS) nex_session_maker = prepare_schema_connection(nex, config.NEX_DBTYPE, 'sgd-master-db.stanford.edu:1521', config.NEX_DBNAME, config.NEX_SCHEMA, config.NEX_DBUSER, config.NEX_DBPASS) perf_session_maker = prepare_schema_connection(perf, config.PERF_DBTYPE, 'sgd-db1.stanford.edu:1521', config.PERF_DBNAME, config.PERF_SCHEMA, config.PERF_DBUSER, config.PERF_DBPASS) nex_backend = SGDBackend(config.NEX_DBTYPE, 'sgd-master-db.stanford.edu:1521', config.NEX_DBNAME, config.NEX_SCHEMA, config.NEX_DBUSER, config.NEX_DBPASS, None) from src.sgd.model.nex.evidence import Posttranslationalevidence, Evidence from src.sgd.convert.from_bud.evidence import make_posttranslational_evidence_starter do_conversion(make_posttranslational_evidence_starter(nex_session_maker), [Json2Obj(Posttranslationalevidence), Evidence2NexDB(nex_session_maker, lambda x: x.query(Posttranslationalevidence), name='convert.from_bud.evidence.posttranslationsl', delete_untouched=True, commit_interval=1000, already_deleted=clean_up_orphans(nex_session_maker, Posttranslationalevidence, Evidence, 'POSTTRANSLATIONAL')), OutputTransformer(1000)]) from src.sgd.model.nex.bioentity import Locus nex_session = nex_session_maker() locus_ids = [x.id for x in nex_session.query(Locus).all()] nex_session.close() from src.sgd.model.perf.bioentity_data import BioentityDetails do_conversion(make_locus_data_backend_starter(nex_backend, 'posttranslational_details', locus_ids), [Json2DataPerfDB(perf_session_maker, BioentityDetails, 'POSTTRANSLATIONAL', locus_ids, name='convert.from_backend.posttranslational_details', commit_interval=1000, sure=True), OutputTransformer(1000)]) perf_session_maker = prepare_schema_connection(perf, config.PERF_DBTYPE, 'sgd-db2.stanford.edu:1521', config.PERF_DBNAME, config.PERF_SCHEMA, config.PERF_DBUSER, config.PERF_DBPASS) perf_backend = PerfBackend(config.PERF_DBTYPE, 'sgd-db1.stanford.edu:1521', config.PERF_DBNAME, config.PERF_SCHEMA, config.PERF_DBUSER, config.PERF_DBPASS, None)
from src.sgd.model.nex.evidence import Evidence, Goevidence, Literatureevidence, Domainevidence, ECNumberevidence, \ Proteinexperimentevidence, Phenotypeevidence, Historyevidence, Pathwayevidence, Goslimevidence, Posttranslationalevidence from src.sgd.convert.from_bud.evidence import make_go_evidence_starter, make_literature_evidence_starter, \ make_domain_evidence_starter, make_protein_experiment_evidence_starter, \ make_ecnumber_evidence_starter, make_protein_experiment_evidence_starter, make_go_slim_evidence_starter, \ make_phenotype_evidence_starter, make_history_evidence_starter, make_pathway_evidence_starter, make_posttranslational_evidence_starter from src.sgd.convert.from_bud.auxiliary import make_bioconcept_count_starter do_conversion(make_go_evidence_starter(bud_session_maker, nex_session_maker), [Json2Obj(Goevidence), Evidence2NexDB(nex_session_maker, lambda x: x.query(Goevidence), name='convert.from_bud.evidence.go', delete_untouched=True, commit_interval=1000, already_deleted=clean_up_orphans(nex_session_maker, Goevidence, Evidence, 'GO'))]) do_conversion(make_go_slim_evidence_starter(nex_session_maker), [Json2Obj(Goslimevidence), Evidence2NexDB(nex_session_maker, lambda x: x.query(Goslimevidence), name='convert.from_bud.evidence.goslim', delete_untouched=True, commit_interval=1000, already_deleted=clean_up_orphans(nex_session_maker, Goslimevidence, Evidence, 'GOSLIM'))]) do_conversion(make_literature_evidence_starter(bud_session_maker, nex_session_maker), [Json2Obj(Literatureevidence), Evidence2NexDB(nex_session_maker, lambda x: x.query(Literatureevidence), name='convert.from_bud.evidence.literature', delete_untouched=True, commit_interval=1000,
# do_conversion(make_dataset_starter(nex_session_maker, 'src/sgd/convert/data/microarray_05_14'), # [Json2Obj(Dataset), # Obj2NexDB(nex_session_maker, lambda x: x.query(Dataset), # name='convert.from_bud.bioitem.dataset', # delete_untouched=True, # commit_interval=1000, # already_deleted=clean_up_orphans(nex_session_maker, Dataset, Bioitem, 'DATASET'))]) do_conversion(make_datasetcolumn_starter(nex_session_maker, 'src/sgd/convert/data/microarray_05_14'), [Json2Obj(Datasetcolumn), Obj2NexDB(nex_session_maker, lambda x: x.query(Datasetcolumn), name='convert.from_bud.bioitem.datasetcolumn', delete_untouched=False, commit_interval=1000, already_deleted=clean_up_orphans(nex_session_maker, Datasetcolumn, Bioitem, 'DATASETCOLUMN'))]) # do_conversion(make_tag_starter(nex_session_maker), # [Json2Obj(Tag), # Obj2NexDB(nex_session_maker, lambda x: x.query(Tag), # name='convert.from_bud.tag', # delete_untouched=True, # commit=True)]) # # do_conversion(make_disambig_starter(nex_session_maker, Tag, ['id', 'format_name'], 'TAG', None), # [Json2Obj(Disambig), # Obj2NexDB(nex_session_maker, lambda x: x.query(Disambig).filter(Disambig.class_type == 'TAG'), # name='convert.from_bud.bioitem.disambig.tag', # delete_untouched=True, # commit=True)]) #
from src.sgd.convert.from_bud.auxiliary import make_disambig_starter do_conversion(make_strain_starter(bud_session_maker, nex_session_maker), [Json2Obj(Strain), Obj2NexDB(nex_session_maker, lambda x: x.query(Strain), name='convert.from_bud.strain', delete_untouched=True, commit=True)]) do_conversion(make_strain_url_starter(nex_session_maker), [Json2Obj(Strainurl), Obj2NexDB(nex_session_maker, lambda x: x.query(Strainurl), name='convert.from_bud.strain.url', delete_untouched=True, commit_interval=1000, already_deleted=clean_up_orphans(nex_session_maker, Strainurl, Url, 'STRAIN'))]) do_conversion(make_disambig_starter(nex_session_maker, Strain, ['id', 'format_name'], 'STRAIN', None), [Json2Obj(Disambig), Obj2NexDB(nex_session_maker, lambda x: x.query(Disambig).filter(Disambig.class_type == 'STRAIN'), name='convert.from_bud.strain.disambig', delete_untouched=True, commit=True)]) # ------------------------------------------ Paragraph ------------------------------------------ from src.sgd.model.nex.paragraph import Paragraph, Strainparagraph, ParagraphReference from src.sgd.convert.from_bud.paragraph import make_strain_paragraph_starter, make_paragraph_reference_starter do_conversion(make_strain_paragraph_starter(nex_session_maker), [Json2Obj(Strainparagraph), Obj2NexDB(nex_session_maker, lambda x: x.query(Strainparagraph),
# Obj2NexDB(nex_session_maker, lambda x: x.query(DNAsequenceevidence).filter(DNAsequenceevidence.strain_id == 1), name='convert.from_bud.evidence.reference_dnasequence', delete_untouched=True, commit_interval=1000)]) # # do_conversion(make_dna_sequence_tag_starter(bud_session_maker, nex_session_maker), # [Json2Obj(DNAsequencetag), # Obj2NexDB(nex_session_maker, lambda x: x.query(DNAsequencetag), name='convert.from_bud.evidence.dnasequence.tags', delete_untouched=True, commit_interval=1000)]) # nex_session = nex_session_maker() strain_key_to_id = dict([(x.unique_key(), x.id) for x in nex_session.query(Strain).all()]) nex_session.close() for sequence_filename, coding_sequence_filename, strain_key in sequence_files: do_conversion(make_dna_sequence_evidence_starter(nex_session_maker, strain_key, sequence_filename, coding_sequence_filename), [Json2Obj(DNAsequenceevidence), Obj2NexDB(nex_session_maker, lambda x: x.query(DNAsequenceevidence).filter(DNAsequenceevidence.strain_id == strain_key_to_id[strain_key]), name='convert.from_bud.evidence.dnasequence', delete_untouched=True, commit_interval=1000)]) clean_up_orphans(nex_session_maker, DNAsequenceevidence, Evidence, 'DNASEQUENCE') for sequence_filename, coding_sequence_filename, strain_key in new_sequence_files: do_conversion(make_new_dna_sequence_evidence_starter(nex_session_maker, strain_key, sequence_filename, coding_sequence_filename), [Json2Obj(DNAsequenceevidence), Obj2NexDB(nex_session_maker, lambda x: x.query(DNAsequenceevidence).filter(DNAsequenceevidence.strain_id == strain_key_to_id[strain_key]).filter(DNAsequenceevidence.dna_type != '1KB'), name='convert.from_bud.evidence.dnasequence', delete_untouched=True, commit_interval=1000)]) update_contig_centromeres(nex_session_maker) update_contig_reference_alignment(nex_session_maker) protparam_data = dict([(row[0], row) for row in make_file_starter('src/sgd/convert/data/ProtParam.txt')()]) for sequence_filename, strain_key in protein_sequence_files:
commit=True)]) do_conversion(make_strain_starter(bud_session_maker, nex_session_maker), [Json2Obj(Strain), Obj2NexDB(nex_session_maker, lambda x: x.query(Strain), name='convert.from_bud.strain', delete_untouched=True, commit=True)]) do_conversion(make_strain_url_starter(nex_session_maker), [Json2Obj(Strainurl), Obj2NexDB(nex_session_maker, lambda x: x.query(Strainurl), name='convert.from_bud.strain.url', delete_untouched=True, commit_interval=1000, already_deleted=clean_up_orphans(nex_session_maker, Strainurl, Url, 'STRAIN'))]) do_conversion(make_experiment_alias_starter(bud_session_maker, nex_session_maker), [Json2Obj(Experimentalias), Obj2NexDB(nex_session_maker, lambda x: x.query(Experimentalias), name='convert.from_bud.experiment_alias', delete_untouched=True, commit=True, already_deleted=clean_up_orphans(nex_session_maker, Experimentalias, Alias, 'EXPERIMENT'))]) do_conversion(make_experiment_relation_starter(bud_session_maker, nex_session_maker), [Json2Obj(Experimentrelation), Obj2NexDB(nex_session_maker, lambda x: x.query(Experimentrelation), name='convert.from_bud.experiment_relation', delete_untouched=True, commit=True,
nex_session_maker = prepare_schema_connection(nex, config.NEX_DBTYPE, 'sgd-master-db.stanford.edu:1521', config.NEX_DBNAME, config.NEX_SCHEMA, config.NEX_DBUSER, config.NEX_DBPASS) perf_session_maker = prepare_schema_connection(perf, config.PERF_DBTYPE, 'sgd-db1.stanford.edu:1521', config.PERF_DBNAME, config.PERF_SCHEMA, config.PERF_DBUSER, config.PERF_DBPASS) nex_backend = SGDBackend(config.NEX_DBTYPE, 'sgd-master-db.stanford.edu:1521', config.NEX_DBNAME, config.NEX_SCHEMA, config.NEX_DBUSER, config.NEX_DBPASS, None) # ------------------------------------------ Evidence ------------------------------------------ from src.sgd.model.nex.evidence import Evidence, Regulationevidence from src.sgd.convert.from_bud.evidence import make_regulation_evidence_starter do_conversion(make_regulation_evidence_starter(nex_session_maker), [Json2Obj(Regulationevidence), Evidence2NexDB(nex_session_maker, lambda x: x.query(Regulationevidence), name='convert.from_bud.evidence.regulation', delete_untouched=True, commit_interval=1000, already_deleted=clean_up_orphans(nex_session_maker, Regulationevidence, Evidence, 'REGULATION'))]) # ------------------------------------------ Perf ------------------------------------------ from src.sgd.model.perf.bioentity_data import BioentityDetails, BioentityEnrichment from src.sgd.model.perf.reference_data import ReferenceDetails from src.sgd.model.nex.bioentity import Locus from src.sgd.model.nex.reference import Reference nex_session = nex_session_maker() locus_ids = [x.id for x in nex_session.query(Locus).all()] reference_ids = [x.id for x in nex_session.query(Reference).all()] nex_session.close() do_conversion(make_locus_data_backend_starter(nex_backend, 'regulation_details', locus_ids), [Json2DataPerfDB(perf_session_maker, BioentityDetails, 'REGULATION', locus_ids, name='convert.from_backend.regulation_details', commit_interval=1000)])