bud_session_maker = prepare_schema_connection(bud, config.BUD_DBTYPE, 'pastry.stanford.edu:1521', config.BUD_DBNAME, config.BUD_SCHEMA, config.BUD_DBUSER, config.BUD_DBPASS)
nex_session_maker = prepare_schema_connection(nex, config.NEX_DBTYPE, 'sgd-master-db.stanford.edu:1521', config.NEX_DBNAME, config.NEX_SCHEMA, config.NEX_DBUSER, config.NEX_DBPASS)
perf_session_maker = prepare_schema_connection(perf, config.PERF_DBTYPE, 'sgd-db1.stanford.edu:1521', config.PERF_DBNAME, config.PERF_SCHEMA, config.PERF_DBUSER, config.PERF_DBPASS)
nex_backend = SGDBackend(config.NEX_DBTYPE, 'sgd-master-db.stanford.edu:1521', config.NEX_DBNAME, config.NEX_SCHEMA, config.NEX_DBUSER, config.NEX_DBPASS, None)
# ------------------------------------------ Evidence ------------------------------------------
from src.sgd.model.nex.evidence import Evidence, Geninteractionevidence, Physinteractionevidence
from src.sgd.convert.from_bud.evidence import make_interaction_evidence_starter
do_conversion(make_interaction_evidence_starter(bud_session_maker, nex_session_maker, 'genetic interactions'),
[Json2Obj(Geninteractionevidence),
Evidence2NexDB(nex_session_maker, lambda x: x.query(Geninteractionevidence),
name='convert.from_bud.evidence.geninteraction',
delete_untouched=True,
commit_interval=1000,
already_deleted=clean_up_orphans(nex_session_maker, Geninteractionevidence, Evidence, 'GENINTERACTION'))])
do_conversion(make_interaction_evidence_starter(bud_session_maker, nex_session_maker, 'physical interactions'),
[Json2Obj(Physinteractionevidence),
Evidence2NexDB(nex_session_maker, lambda x: x.query(Physinteractionevidence),
name='convert.from_bud.evidence.physinteraction',
delete_untouched=True,
commit_interval=1000,
already_deleted=clean_up_orphans(nex_session_maker, Physinteractionevidence, Evidence, 'PHYSINTERACTION'))])
# ------------------------------------------ Perf ------------------------------------------
from src.sgd.model.perf.bioentity_data import BioentityDetails
from src.sgd.model.perf.reference_data import ReferenceDetails
from src.sgd.model.nex.bioentity import Locus
from src.sgd.model.nex.reference import Reference
nex_session_maker = prepare_schema_connection(nex, config.NEX_DBTYPE, 'sgd-master-db.stanford.edu:1521', config.NEX_DBNAME, config.NEX_SCHEMA, config.NEX_DBUSER, config.NEX_DBPASS)
perf_session_maker = prepare_schema_connection(perf, config.PERF_DBTYPE, 'sgd-db1.stanford.edu:1521', config.PERF_DBNAME, config.PERF_SCHEMA, config.PERF_DBUSER, config.PERF_DBPASS)
nex_backend = SGDBackend(config.NEX_DBTYPE, 'sgd-master-db.stanford.edu:1521', config.NEX_DBNAME, config.NEX_SCHEMA, config.NEX_DBUSER, config.NEX_DBPASS, None)
from src.sgd.model.nex.bioitem import Reservedname
from src.sgd.model.nex.paragraph import Paragraph, Bioentityparagraph, ParagraphReference
from src.sgd.convert.from_bud.paragraph import make_bioentity_paragraph_starter, make_paragraph_reference_starter
do_conversion(make_bioentity_paragraph_starter(bud_session_maker, nex_session_maker),
[Json2Obj(Bioentityparagraph),
Obj2NexDB(nex_session_maker, lambda x: x.query(Bioentityparagraph),
name='convert.from_bud.paragraph.bioentity',
delete_untouched=True,
commit=True,
already_deleted=clean_up_orphans(nex_session_maker, Bioentityparagraph, Paragraph, 'BIOENTITY'))])
do_conversion(make_paragraph_reference_starter(nex_session_maker),
[Json2Obj(ParagraphReference),
Obj2NexDB(nex_session_maker, lambda x: x.query(ParagraphReference),
name='convert.from_bud.paragraph_reference',
delete_untouched=True,
commit=True)])
from src.sgd.model.perf.core import Bioentity as PerfBioentity
do_conversion(make_backend_starter(nex_backend, 'all_bioentities', 1000),
[Json2CorePerfDB(perf_session_maker, PerfBioentity, name='convert.from_backend.bioentity', commit_interval=1000, delete_untouched=True)])
perf_session_maker = prepare_schema_connection(perf, config.PERF_DBTYPE, 'sgd-db2.stanford.edu:1521', config.PERF_DBNAME, config.PERF_SCHEMA, config.PERF_DBUSER, config.PERF_DBPASS)
perf_backend = PerfBackend(config.PERF_DBTYPE, 'sgd-db1.stanford.edu:1521', config.PERF_DBNAME, config.PERF_SCHEMA, config.PERF_DBUSER, config.PERF_DBPASS, None)
bud_session_maker = prepare_schema_connection(bud, config.BUD_DBTYPE, 'pastry.stanford.edu:1521', config.BUD_DBNAME, config.BUD_SCHEMA, config.BUD_DBUSER, config.BUD_DBPASS)
nex_session_maker = prepare_schema_connection(nex, config.NEX_DBTYPE, 'sgd-master-db.stanford.edu:1521', config.NEX_DBNAME, config.NEX_SCHEMA, config.NEX_DBUSER, config.NEX_DBPASS)
perf_session_maker = prepare_schema_connection(perf, config.PERF_DBTYPE, 'sgd-db1.stanford.edu:1521', config.PERF_DBNAME, config.PERF_SCHEMA, config.PERF_DBUSER, config.PERF_DBPASS)
nex_backend = SGDBackend(config.NEX_DBTYPE, 'sgd-master-db.stanford.edu:1521', config.NEX_DBNAME, config.NEX_SCHEMA, config.NEX_DBUSER, config.NEX_DBPASS, None)
from src.sgd.model.nex.evidence import Posttranslationalevidence, Evidence
from src.sgd.convert.from_bud.evidence import make_posttranslational_evidence_starter
do_conversion(make_posttranslational_evidence_starter(nex_session_maker),
[Json2Obj(Posttranslationalevidence),
Evidence2NexDB(nex_session_maker, lambda x: x.query(Posttranslationalevidence),
name='convert.from_bud.evidence.posttranslationsl',
delete_untouched=True,
commit_interval=1000,
already_deleted=clean_up_orphans(nex_session_maker, Posttranslationalevidence, Evidence, 'POSTTRANSLATIONAL')),
OutputTransformer(1000)])
from src.sgd.model.nex.bioentity import Locus
nex_session = nex_session_maker()
locus_ids = [x.id for x in nex_session.query(Locus).all()]
nex_session.close()
from src.sgd.model.perf.bioentity_data import BioentityDetails
do_conversion(make_locus_data_backend_starter(nex_backend, 'posttranslational_details', locus_ids),
[Json2DataPerfDB(perf_session_maker, BioentityDetails, 'POSTTRANSLATIONAL', locus_ids, name='convert.from_backend.posttranslational_details',
commit_interval=1000, sure=True),
OutputTransformer(1000)])
perf_session_maker = prepare_schema_connection(perf, config.PERF_DBTYPE, 'sgd-db2.stanford.edu:1521', config.PERF_DBNAME, config.PERF_SCHEMA, config.PERF_DBUSER, config.PERF_DBPASS)
perf_backend = PerfBackend(config.PERF_DBTYPE, 'sgd-db1.stanford.edu:1521', config.PERF_DBNAME, config.PERF_SCHEMA, config.PERF_DBUSER, config.PERF_DBPASS, None)
from src.sgd.model.nex.evidence import Evidence, Goevidence, Literatureevidence, Domainevidence, ECNumberevidence, \
Proteinexperimentevidence, Phenotypeevidence, Historyevidence, Pathwayevidence, Goslimevidence, Posttranslationalevidence
from src.sgd.convert.from_bud.evidence import make_go_evidence_starter, make_literature_evidence_starter, \
make_domain_evidence_starter, make_protein_experiment_evidence_starter, \
make_ecnumber_evidence_starter, make_protein_experiment_evidence_starter, make_go_slim_evidence_starter, \
make_phenotype_evidence_starter, make_history_evidence_starter, make_pathway_evidence_starter, make_posttranslational_evidence_starter
from src.sgd.convert.from_bud.auxiliary import make_bioconcept_count_starter
do_conversion(make_go_evidence_starter(bud_session_maker, nex_session_maker),
[Json2Obj(Goevidence),
Evidence2NexDB(nex_session_maker, lambda x: x.query(Goevidence),
name='convert.from_bud.evidence.go',
delete_untouched=True,
commit_interval=1000,
already_deleted=clean_up_orphans(nex_session_maker, Goevidence, Evidence, 'GO'))])
do_conversion(make_go_slim_evidence_starter(nex_session_maker),
[Json2Obj(Goslimevidence),
Evidence2NexDB(nex_session_maker, lambda x: x.query(Goslimevidence),
name='convert.from_bud.evidence.goslim',
delete_untouched=True,
commit_interval=1000,
already_deleted=clean_up_orphans(nex_session_maker, Goslimevidence, Evidence, 'GOSLIM'))])
do_conversion(make_literature_evidence_starter(bud_session_maker, nex_session_maker),
[Json2Obj(Literatureevidence),
Evidence2NexDB(nex_session_maker, lambda x: x.query(Literatureevidence),
name='convert.from_bud.evidence.literature',
delete_untouched=True,
commit_interval=1000,
# do_conversion(make_dataset_starter(nex_session_maker, 'src/sgd/convert/data/microarray_05_14'),
# [Json2Obj(Dataset),
# Obj2NexDB(nex_session_maker, lambda x: x.query(Dataset),
# name='convert.from_bud.bioitem.dataset',
# delete_untouched=True,
# commit_interval=1000,
# already_deleted=clean_up_orphans(nex_session_maker, Dataset, Bioitem, 'DATASET'))])
do_conversion(make_datasetcolumn_starter(nex_session_maker, 'src/sgd/convert/data/microarray_05_14'),
[Json2Obj(Datasetcolumn),
Obj2NexDB(nex_session_maker, lambda x: x.query(Datasetcolumn),
name='convert.from_bud.bioitem.datasetcolumn',
delete_untouched=False,
commit_interval=1000,
already_deleted=clean_up_orphans(nex_session_maker, Datasetcolumn, Bioitem, 'DATASETCOLUMN'))])
# do_conversion(make_tag_starter(nex_session_maker),
# [Json2Obj(Tag),
# Obj2NexDB(nex_session_maker, lambda x: x.query(Tag),
# name='convert.from_bud.tag',
# delete_untouched=True,
# commit=True)])
#
# do_conversion(make_disambig_starter(nex_session_maker, Tag, ['id', 'format_name'], 'TAG', None),
# [Json2Obj(Disambig),
# Obj2NexDB(nex_session_maker, lambda x: x.query(Disambig).filter(Disambig.class_type == 'TAG'),
# name='convert.from_bud.bioitem.disambig.tag',
# delete_untouched=True,
# commit=True)])
#
from src.sgd.convert.from_bud.auxiliary import make_disambig_starter
do_conversion(make_strain_starter(bud_session_maker, nex_session_maker),
[Json2Obj(Strain),
Obj2NexDB(nex_session_maker, lambda x: x.query(Strain),
name='convert.from_bud.strain',
delete_untouched=True,
commit=True)])
do_conversion(make_strain_url_starter(nex_session_maker),
[Json2Obj(Strainurl),
Obj2NexDB(nex_session_maker, lambda x: x.query(Strainurl),
name='convert.from_bud.strain.url',
delete_untouched=True,
commit_interval=1000,
already_deleted=clean_up_orphans(nex_session_maker, Strainurl, Url, 'STRAIN'))])
do_conversion(make_disambig_starter(nex_session_maker, Strain, ['id', 'format_name'], 'STRAIN', None),
[Json2Obj(Disambig),
Obj2NexDB(nex_session_maker, lambda x: x.query(Disambig).filter(Disambig.class_type == 'STRAIN'),
name='convert.from_bud.strain.disambig',
delete_untouched=True,
commit=True)])
# ------------------------------------------ Paragraph ------------------------------------------
from src.sgd.model.nex.paragraph import Paragraph, Strainparagraph, ParagraphReference
from src.sgd.convert.from_bud.paragraph import make_strain_paragraph_starter, make_paragraph_reference_starter
do_conversion(make_strain_paragraph_starter(nex_session_maker),
[Json2Obj(Strainparagraph),
Obj2NexDB(nex_session_maker, lambda x: x.query(Strainparagraph),
# Obj2NexDB(nex_session_maker, lambda x: x.query(DNAsequenceevidence).filter(DNAsequenceevidence.strain_id == 1), name='convert.from_bud.evidence.reference_dnasequence', delete_untouched=True, commit_interval=1000)])
#
# do_conversion(make_dna_sequence_tag_starter(bud_session_maker, nex_session_maker),
# [Json2Obj(DNAsequencetag),
# Obj2NexDB(nex_session_maker, lambda x: x.query(DNAsequencetag), name='convert.from_bud.evidence.dnasequence.tags', delete_untouched=True, commit_interval=1000)])
#
nex_session = nex_session_maker()
strain_key_to_id = dict([(x.unique_key(), x.id) for x in nex_session.query(Strain).all()])
nex_session.close()
for sequence_filename, coding_sequence_filename, strain_key in sequence_files:
do_conversion(make_dna_sequence_evidence_starter(nex_session_maker, strain_key, sequence_filename, coding_sequence_filename),
[Json2Obj(DNAsequenceevidence),
Obj2NexDB(nex_session_maker, lambda x: x.query(DNAsequenceevidence).filter(DNAsequenceevidence.strain_id == strain_key_to_id[strain_key]), name='convert.from_bud.evidence.dnasequence', delete_untouched=True, commit_interval=1000)])
clean_up_orphans(nex_session_maker, DNAsequenceevidence, Evidence, 'DNASEQUENCE')
for sequence_filename, coding_sequence_filename, strain_key in new_sequence_files:
do_conversion(make_new_dna_sequence_evidence_starter(nex_session_maker, strain_key, sequence_filename, coding_sequence_filename),
[Json2Obj(DNAsequenceevidence),
Obj2NexDB(nex_session_maker, lambda x: x.query(DNAsequenceevidence).filter(DNAsequenceevidence.strain_id == strain_key_to_id[strain_key]).filter(DNAsequenceevidence.dna_type != '1KB'),
name='convert.from_bud.evidence.dnasequence',
delete_untouched=True,
commit_interval=1000)])
update_contig_centromeres(nex_session_maker)
update_contig_reference_alignment(nex_session_maker)
protparam_data = dict([(row[0], row) for row in make_file_starter('src/sgd/convert/data/ProtParam.txt')()])
for sequence_filename, strain_key in protein_sequence_files:
commit=True)])
do_conversion(make_strain_starter(bud_session_maker, nex_session_maker),
[Json2Obj(Strain),
Obj2NexDB(nex_session_maker, lambda x: x.query(Strain),
name='convert.from_bud.strain',
delete_untouched=True,
commit=True)])
do_conversion(make_strain_url_starter(nex_session_maker),
[Json2Obj(Strainurl),
Obj2NexDB(nex_session_maker, lambda x: x.query(Strainurl),
name='convert.from_bud.strain.url',
delete_untouched=True,
commit_interval=1000,
already_deleted=clean_up_orphans(nex_session_maker, Strainurl, Url, 'STRAIN'))])
do_conversion(make_experiment_alias_starter(bud_session_maker, nex_session_maker),
[Json2Obj(Experimentalias),
Obj2NexDB(nex_session_maker, lambda x: x.query(Experimentalias),
name='convert.from_bud.experiment_alias',
delete_untouched=True,
commit=True,
already_deleted=clean_up_orphans(nex_session_maker, Experimentalias, Alias, 'EXPERIMENT'))])
do_conversion(make_experiment_relation_starter(bud_session_maker, nex_session_maker),
[Json2Obj(Experimentrelation),
Obj2NexDB(nex_session_maker, lambda x: x.query(Experimentrelation),
name='convert.from_bud.experiment_relation',
delete_untouched=True,
commit=True,
nex_session_maker = prepare_schema_connection(nex, config.NEX_DBTYPE, 'sgd-master-db.stanford.edu:1521', config.NEX_DBNAME, config.NEX_SCHEMA, config.NEX_DBUSER, config.NEX_DBPASS)
perf_session_maker = prepare_schema_connection(perf, config.PERF_DBTYPE, 'sgd-db1.stanford.edu:1521', config.PERF_DBNAME, config.PERF_SCHEMA, config.PERF_DBUSER, config.PERF_DBPASS)
nex_backend = SGDBackend(config.NEX_DBTYPE, 'sgd-master-db.stanford.edu:1521', config.NEX_DBNAME, config.NEX_SCHEMA, config.NEX_DBUSER, config.NEX_DBPASS, None)
# ------------------------------------------ Evidence ------------------------------------------
from src.sgd.model.nex.evidence import Evidence, Regulationevidence
from src.sgd.convert.from_bud.evidence import make_regulation_evidence_starter
do_conversion(make_regulation_evidence_starter(nex_session_maker),
[Json2Obj(Regulationevidence),
Evidence2NexDB(nex_session_maker, lambda x: x.query(Regulationevidence),
name='convert.from_bud.evidence.regulation',
delete_untouched=True,
commit_interval=1000,
already_deleted=clean_up_orphans(nex_session_maker, Regulationevidence, Evidence, 'REGULATION'))])
# ------------------------------------------ Perf ------------------------------------------
from src.sgd.model.perf.bioentity_data import BioentityDetails, BioentityEnrichment
from src.sgd.model.perf.reference_data import ReferenceDetails
from src.sgd.model.nex.bioentity import Locus
from src.sgd.model.nex.reference import Reference
nex_session = nex_session_maker()
locus_ids = [x.id for x in nex_session.query(Locus).all()]
reference_ids = [x.id for x in nex_session.query(Reference).all()]
nex_session.close()
do_conversion(make_locus_data_backend_starter(nex_backend, 'regulation_details', locus_ids),
[Json2DataPerfDB(perf_session_maker, BioentityDetails, 'REGULATION', locus_ids, name='convert.from_backend.regulation_details', commit_interval=1000)])
