def main(): conf = config.Victoria() conf_main = config.Config() username = conf.USERNAME password = conf.PASSWORD MongoDB = conf_main.SERVER src = insert_experimental_protein(MongoDB=MongoDB, username=username, password=password, collection="observation", db="datanator-demo") src.process_docs()
def main(): conf = config.Victoria() conf_main = config.Config() username = conf.USERNAME password = conf.PASSWORD MongoDB = conf_main.SERVER src = ParseMetaboliteConcentration(MongoDB=MongoDB, username=username, password=password, collection="metabolite_concentration", db="datanator-demo") src.parse_metabolite()
def main(): conf = config.Justin() conf_main = config.Config() username = conf.USERNAME password = conf.PASSWORD MongoDB = conf_main.SERVER src = ParsePsortExperimental(MongoDB=MongoDB, username=username, password=password, collection="protein_localization", db="datanator-demo") src.parse_psortdb()
def main(): conf = config.Victoria() conf_main = config.Config() username = conf.USERNAME password = conf.PASSWORD MongoDB = conf_main.SERVER src = parse_quantification_budding_yeast_proteome(MongoDB=MongoDB, username=username, password=password, collection="observation", db="datanator-demo") src.process_docs()
def main(): conf = config.Victoria() conf_main = config.Config() username = conf.USERNAME password = conf.PASSWORD MongoDB = conf_main.SERVER src = parse_column_k0_mouse_brain(MongoDB=MongoDB, username=username, password=password, collection="observation", db="datanator-demo") src.process_docs_peptide() src.process_docs_protein()
def setUpClass(cls): cls.cache_dir = tempfile.mkdtemp() cls.src = batch_load.MongoToES( profile_name='es-poweruser', credential_path='~/.wc/third_party/aws_credentials', config_path='~/.wc/third_party/aws_config', elastic_path='~/.wc/third_party/elasticsearch.ini', cache_dir=cls.cache_dir, service_name='es', index='test', max_entries=float('inf'), verbose=True) cls.url = cls.src.es_endpoint + '/' + cls.src.index requests.delete(cls.url, auth=cls.src.awsauth) conf = config.Config() cls.username = conf.USERNAME cls.password = conf.PASSWORD cls.server = conf.SERVER cls.authDB = conf.AUTHDB cls.db = 'datanator'
def setUpClass(cls): cls.src_basic = config.Config() cls.src_prod = config.ProductionConfig() cls.src_test = config.TestConfig() cls.src_ftx = config.FtxConfig()
def main(): conf = config_mongo.Config() username = conf.USERNAME password = conf.PASSWORD server = conf.SERVER authDB = conf.AUTHDB db = 'datanator' manager = MongoToES(verbose=True, profile_name='es-poweruser', credential_path='~/.wc/third_party/aws_credentials', config_path='~/.wc/third_party/aws_config', elastic_path='~/.wc/third_party/elasticsearch.ini') filter_dir = '/root/karr_lab/karr_lab_aws_manager/karr_lab_aws_manager/elasticsearch_kl/filters/autocomplete_filter.json' analyzer_dir = '/root/karr_lab/karr_lab_aws_manager/karr_lab_aws_manager/elasticsearch_kl/analyzers/auto_complete.json' # old_index = 'protein' # new_index = 'protein_something' # _, _, _, = manager.migrate_index(old_index, new_index) # _, _, _, = manager.migrate_index(new_index, old_index) # data from "protein" collection # index_name = 'uniprot' # _ = manager.delete_index(index_name) # count, docs = manager.data_from_mongo_protein(server, db, username, password, authSource=authDB) # mappings_dir = '/root/karr_lab/karr_lab_aws_manager/karr_lab_aws_manager/elasticsearch_kl/mappings/protein.json' # index_manager = index_setting_file.IndexUtil(filter_dir=filter_dir, analyzer_dir=analyzer_dir, mapping_properties_dir=mappings_dir) # setting_file = index_manager.combine_files(_filter=True, analyzer=True, mappings=True) # _ = manager.create_index_with_file(index_name, setting_file) # _ = manager.data_to_es_bulk(docs, count=count, index=index_name, _id='uniprot_id') # data from "ecmdb" and "ymdb" collection # ecmdb_docs, ecmdb_count, ymdb_docs, ymdb_count = manager.data_from_mongo_metabolite(server, # db, username, password, authSource=authDB) # ecmdb = 'ecmdb' # _ = manager.delete_index(ecmdb) # ecmdb_mappings_dir = '/root/karr_lab/karr_lab_aws_manager/karr_lab_aws_manager/elasticsearch_kl/mappings/ecmdb.json' # ecmdb_index_manager = index_setting_file.IndexUtil(filter_dir=filter_dir, analyzer_dir=analyzer_dir, mapping_properties_dir=ecmdb_mappings_dir) # ecmdb_setting_file = ecmdb_index_manager.combine_files(_filter=True, analyzer=True, mappings=True) # _ = manager.create_index_with_file(ecmdb, ecmdb_setting_file) # status = manager.data_to_es_bulk(ecmdb_docs, index=ecmdb, count=ecmdb_count, _id='m2m_id') # ymdb = 'ymdb' # _ = manager.delete_index(ymdb) # ymdb_mappings_dir = '/root/karr_lab/karr_lab_aws_manager/karr_lab_aws_manager/elasticsearch_kl/mappings/ymdb.json' # ymdb_index_manager = index_setting_file.IndexUtil(filter_dir=filter_dir, analyzer_dir=analyzer_dir, mapping_properties_dir=ymdb_mappings_dir) # ymdb_setting_file = ymdb_index_manager.combine_files(_filter=True, analyzer=True, mappings=True) # _ = manager.create_index_with_file(ymdb, ymdb_setting_file) # status = manager.data_to_es_bulk(ymdb_docs, index=ymdb, count=ymdb_count, _id='ymdb_id') # data from "metabolites_meta" collection # index_name = 'metabolites_meta' # _ = manager.delete_index(index_name) # docs = manager.data_from_mongo_metabolites_meta(server, db, username, password, authSource=authDB) # # mappings_dir = '/root/karr_lab/karr_lab_aws_manager/karr_lab_aws_manager/elasticsearch_kl/mappings/metabolites_meta.json' # index_manager = index_setting_file.IndexUtil(filter_dir=filter_dir, analyzer_dir=analyzer_dir) # setting_file = index_manager.combine_files(_filter=True, analyzer=True, mappings=False) # _ = manager.create_index_with_file(index_name, setting_file) # _ = manager.data_to_es_single(5225, docs, index_name, _id='InChI_Key') # data from "sabio_rk_old" collection # count, docs = manager.data_from_mongo_sabiork(server, db, username, password, authSource=authDB) # index_name = 'sabio_rk' # mappings_dir = '/root/karr_lab/karr_lab_aws_manager/karr_lab_aws_manager/elasticsearch_kl/mappings/sabio_rk.json' # index_manager = index_setting_file.IndexUtil(filter_dir=filter_dir, analyzer_dir=analyzer_dir, mapping_properties_dir=mappings_dir) # setting_file = index_manager.combine_files(_filter=True, analyzer=True, mappings=True) # _ = manager.create_index_with_file(index_name, setting_file) # _ = manager.data_to_es_bulk(docs, index=index_name, count=count, _id='kinlaw_id') # # data from "sabio_reaction_entries" collection index_name = 'sabio_reaction_entries' _ = manager.delete_index(index_name) count, docs = manager.data_from_mongo_sabiork_rxn_entries( server, db, username, password, authSource=authDB) r = manager.create_index(index_name) _ = manager.data_to_es_bulk(docs, index=index_name, count=count, _id='rxn_id') # data from "rna_halflife" collection # count, docs = manager.data_from_mongo(server, db, username, password, authSource=authDB) # print(count) # index_schema_path = str(Path('/root/karr_lab/karr_lab_aws_manager/karr_lab_aws_manager/elasticsearch_kl/mappings/rna_halflife.json').expanduser()) # with open(index_schema_path) as json_file: # index_schema = json.load(json_file) # _ = manager.create_index('rna_halflife') # _ = manager.data_to_es_bulk(docs, index='rna_halflife', count=count, _id='_id') # data from "taxon_tree" collection # index_name = 'taxon_tree' # count, docs = manager.data_from_mongo(server, db, username, password, authSource=authDB, collection_str=index_name) # mappings_dir = '/root/karr_lab/karr_lab_aws_manager/karr_lab_aws_manager/elasticsearch_kl/mappings/taxon_tree.json' # index_manager = index_setting_file.IndexUtil(filter_dir=filter_dir, analyzer_dir=analyzer_dir, mapping_properties_dir=mappings_dir) # setting_file = index_manager.combine_files(_filter=True, analyzer=True, mappings=True) # _ = manager.create_index_with_file(index_name, setting_file) # _ = manager.data_to_es_bulk(docs, index=index_name, count=count, _id='tax_id') # data from "kegg_orthology" collection # count, docs = manager.data_from_mongo_kegg_orthology(server, db, username, password, authSource=authDB) # index_name = 'kegg_orthology' # _ = str(Path('/root/karr_lab/karr_lab_aws_manager/karr_lab_aws_manager/elasticsearch_kl/mappings/rna_halflife.json').expanduser()) # with open(index_schema_path) as json_file: # index_schema = json.load(json_file) # _ = manager.delete_index(index_name) # _ = manager.create_index(index_name) # _ = manager.data_to_es_bulk(docs, index=index_name, count=count, _id='kegg_orthology_id') # data from "brenda_reactions" collection # index_name = 'brenda_reactions' # count, docs = manager.data_from_mongo(server, db, username, password, authSource=authDB, collection_str=index_name) # print(count) # index_schema_path = str(Path('/root/karr_lab/karr_lab_aws_manager/karr_lab_aws_manager/elasticsearch_kl/mappings/rna_halflife.json').expanduser()) # with open(index_schema_path) as json_file: # index_schema = json.load(json_file) # index_manager = index_setting_file.IndexUtil(filter_dir=filter_dir, analyzer_dir=analyzer_dir) # setting_file = index_manager.combine_files(_filter=True, analyzer=True, mappings=False) # _ = manager.create_index_with_file(index_name, setting_file) # _ = manager.data_to_es_bulk(docs, index=index_name, count=count, _id='_id') # data from "metabolite_concentrations" collection # index_name = 'metabolite_concentrations' # _ = manager.delete_index(index_name) # count, docs = manager.data_from_mongo(server, db, username, password, authSource=authDB, collection_str=index_name) # print(count) # index_manager = index_setting_file.IndexUtil(filter_dir=filter_dir, analyzer_dir=analyzer_dir) # setting_file = index_manager.combine_files(_filter=True, analyzer=True, mappings=False) # _ = manager.create_index_with_file(index_name, setting_file) # _ = manager.data_to_es_bulk(docs, index=index_name, count=count, _id='inchikey') # data from "rna_modification" collection # index_name = 'rna_modification' # _ = manager.delete_index(index_name) # count, docs = manager.data_from_mongo(server, db, username, password, authSource=authDB, collection_str=index_name) # print(count) # index_manager = index_setting_file.IndexUtil(filter_dir=filter_dir, analyzer_dir=analyzer_dir) # setting_file = index_manager.combine_files(_filter=True, analyzer=True, mappings=False) # _ = manager.create_index_with_file(index_name, setting_file) # _ = manager.data_to_es_bulk(docs, index=index_name, count=count, _id='_id') r = manager.index_health_status() print(r.content.decode('utf-8'))