def main():
conf = config.Victoria()
conf_main = config.Config()
username = conf.USERNAME
password = conf.PASSWORD
MongoDB = conf_main.SERVER
src = insert_experimental_protein(MongoDB=MongoDB,
username=username,
password=password,
collection="observation",
db="datanator-demo")
src.process_docs()
def main():
conf = config.Victoria()
conf_main = config.Config()
username = conf.USERNAME
password = conf.PASSWORD
MongoDB = conf_main.SERVER
src = ParseMetaboliteConcentration(MongoDB=MongoDB,
username=username,
password=password,
collection="metabolite_concentration",
db="datanator-demo")
src.parse_metabolite()
def main():
conf = config.Justin()
conf_main = config.Config()
username = conf.USERNAME
password = conf.PASSWORD
MongoDB = conf_main.SERVER
src = ParsePsortExperimental(MongoDB=MongoDB,
username=username,
password=password,
collection="protein_localization",
db="datanator-demo")
src.parse_psortdb()
def main():
conf = config.Victoria()
conf_main = config.Config()
username = conf.USERNAME
password = conf.PASSWORD
MongoDB = conf_main.SERVER
src = parse_quantification_budding_yeast_proteome(MongoDB=MongoDB,
username=username,
password=password,
collection="observation",
db="datanator-demo")
src.process_docs()
def main():
conf = config.Victoria()
conf_main = config.Config()
username = conf.USERNAME
password = conf.PASSWORD
MongoDB = conf_main.SERVER
src = parse_column_k0_mouse_brain(MongoDB=MongoDB,
username=username,
password=password,
collection="observation",
db="datanator-demo")
src.process_docs_peptide()
src.process_docs_protein()
def setUpClass(cls):
cls.cache_dir = tempfile.mkdtemp()
cls.src = batch_load.MongoToES(
profile_name='es-poweruser',
credential_path='~/.wc/third_party/aws_credentials',
config_path='~/.wc/third_party/aws_config',
elastic_path='~/.wc/third_party/elasticsearch.ini',
cache_dir=cls.cache_dir,
service_name='es',
index='test',
max_entries=float('inf'),
verbose=True)
cls.url = cls.src.es_endpoint + '/' + cls.src.index
requests.delete(cls.url, auth=cls.src.awsauth)
conf = config.Config()
cls.username = conf.USERNAME
cls.password = conf.PASSWORD
cls.server = conf.SERVER
cls.authDB = conf.AUTHDB
cls.db = 'datanator'
def setUpClass(cls):
cls.src_basic = config.Config()
cls.src_prod = config.ProductionConfig()
cls.src_test = config.TestConfig()
cls.src_ftx = config.FtxConfig()
def main():
conf = config_mongo.Config()
username = conf.USERNAME
password = conf.PASSWORD
server = conf.SERVER
authDB = conf.AUTHDB
db = 'datanator'
manager = MongoToES(verbose=True,
profile_name='es-poweruser',
credential_path='~/.wc/third_party/aws_credentials',
config_path='~/.wc/third_party/aws_config',
elastic_path='~/.wc/third_party/elasticsearch.ini')
filter_dir = '/root/karr_lab/karr_lab_aws_manager/karr_lab_aws_manager/elasticsearch_kl/filters/autocomplete_filter.json'
analyzer_dir = '/root/karr_lab/karr_lab_aws_manager/karr_lab_aws_manager/elasticsearch_kl/analyzers/auto_complete.json'
# old_index = 'protein'
# new_index = 'protein_something'
# _, _, _, = manager.migrate_index(old_index, new_index)
# _, _, _, = manager.migrate_index(new_index, old_index)
# data from "protein" collection
# index_name = 'uniprot'
# _ = manager.delete_index(index_name)
# count, docs = manager.data_from_mongo_protein(server, db, username, password, authSource=authDB)
# mappings_dir = '/root/karr_lab/karr_lab_aws_manager/karr_lab_aws_manager/elasticsearch_kl/mappings/protein.json'
# index_manager = index_setting_file.IndexUtil(filter_dir=filter_dir, analyzer_dir=analyzer_dir, mapping_properties_dir=mappings_dir)
# setting_file = index_manager.combine_files(_filter=True, analyzer=True, mappings=True)
# _ = manager.create_index_with_file(index_name, setting_file)
# _ = manager.data_to_es_bulk(docs, count=count, index=index_name, _id='uniprot_id')
# data from "ecmdb" and "ymdb" collection
# ecmdb_docs, ecmdb_count, ymdb_docs, ymdb_count = manager.data_from_mongo_metabolite(server,
# db, username, password, authSource=authDB)
# ecmdb = 'ecmdb'
# _ = manager.delete_index(ecmdb)
# ecmdb_mappings_dir = '/root/karr_lab/karr_lab_aws_manager/karr_lab_aws_manager/elasticsearch_kl/mappings/ecmdb.json'
# ecmdb_index_manager = index_setting_file.IndexUtil(filter_dir=filter_dir, analyzer_dir=analyzer_dir, mapping_properties_dir=ecmdb_mappings_dir)
# ecmdb_setting_file = ecmdb_index_manager.combine_files(_filter=True, analyzer=True, mappings=True)
# _ = manager.create_index_with_file(ecmdb, ecmdb_setting_file)
# status = manager.data_to_es_bulk(ecmdb_docs, index=ecmdb, count=ecmdb_count, _id='m2m_id')
# ymdb = 'ymdb'
# _ = manager.delete_index(ymdb)
# ymdb_mappings_dir = '/root/karr_lab/karr_lab_aws_manager/karr_lab_aws_manager/elasticsearch_kl/mappings/ymdb.json'
# ymdb_index_manager = index_setting_file.IndexUtil(filter_dir=filter_dir, analyzer_dir=analyzer_dir, mapping_properties_dir=ymdb_mappings_dir)
# ymdb_setting_file = ymdb_index_manager.combine_files(_filter=True, analyzer=True, mappings=True)
# _ = manager.create_index_with_file(ymdb, ymdb_setting_file)
# status = manager.data_to_es_bulk(ymdb_docs, index=ymdb, count=ymdb_count, _id='ymdb_id')
# data from "metabolites_meta" collection
# index_name = 'metabolites_meta'
# _ = manager.delete_index(index_name)
# docs = manager.data_from_mongo_metabolites_meta(server, db, username, password, authSource=authDB)
# # mappings_dir = '/root/karr_lab/karr_lab_aws_manager/karr_lab_aws_manager/elasticsearch_kl/mappings/metabolites_meta.json'
# index_manager = index_setting_file.IndexUtil(filter_dir=filter_dir, analyzer_dir=analyzer_dir)
# setting_file = index_manager.combine_files(_filter=True, analyzer=True, mappings=False)
# _ = manager.create_index_with_file(index_name, setting_file)
# _ = manager.data_to_es_single(5225, docs, index_name, _id='InChI_Key')
# data from "sabio_rk_old" collection
# count, docs = manager.data_from_mongo_sabiork(server, db, username, password, authSource=authDB)
# index_name = 'sabio_rk'
# mappings_dir = '/root/karr_lab/karr_lab_aws_manager/karr_lab_aws_manager/elasticsearch_kl/mappings/sabio_rk.json'
# index_manager = index_setting_file.IndexUtil(filter_dir=filter_dir, analyzer_dir=analyzer_dir, mapping_properties_dir=mappings_dir)
# setting_file = index_manager.combine_files(_filter=True, analyzer=True, mappings=True)
# _ = manager.create_index_with_file(index_name, setting_file)
# _ = manager.data_to_es_bulk(docs, index=index_name, count=count, _id='kinlaw_id')
# # data from "sabio_reaction_entries" collection
index_name = 'sabio_reaction_entries'
_ = manager.delete_index(index_name)
count, docs = manager.data_from_mongo_sabiork_rxn_entries(
server, db, username, password, authSource=authDB)
r = manager.create_index(index_name)
_ = manager.data_to_es_bulk(docs,
index=index_name,
count=count,
_id='rxn_id')
# data from "rna_halflife" collection
# count, docs = manager.data_from_mongo(server, db, username, password, authSource=authDB)
# print(count)
# index_schema_path = str(Path('/root/karr_lab/karr_lab_aws_manager/karr_lab_aws_manager/elasticsearch_kl/mappings/rna_halflife.json').expanduser())
# with open(index_schema_path) as json_file:
# index_schema = json.load(json_file)
# _ = manager.create_index('rna_halflife')
# _ = manager.data_to_es_bulk(docs, index='rna_halflife', count=count, _id='_id')
# data from "taxon_tree" collection
# index_name = 'taxon_tree'
# count, docs = manager.data_from_mongo(server, db, username, password, authSource=authDB, collection_str=index_name)
# mappings_dir = '/root/karr_lab/karr_lab_aws_manager/karr_lab_aws_manager/elasticsearch_kl/mappings/taxon_tree.json'
# index_manager = index_setting_file.IndexUtil(filter_dir=filter_dir, analyzer_dir=analyzer_dir, mapping_properties_dir=mappings_dir)
# setting_file = index_manager.combine_files(_filter=True, analyzer=True, mappings=True)
# _ = manager.create_index_with_file(index_name, setting_file)
# _ = manager.data_to_es_bulk(docs, index=index_name, count=count, _id='tax_id')
# data from "kegg_orthology" collection
# count, docs = manager.data_from_mongo_kegg_orthology(server, db, username, password, authSource=authDB)
# index_name = 'kegg_orthology'
# _ = str(Path('/root/karr_lab/karr_lab_aws_manager/karr_lab_aws_manager/elasticsearch_kl/mappings/rna_halflife.json').expanduser())
# with open(index_schema_path) as json_file:
# index_schema = json.load(json_file)
# _ = manager.delete_index(index_name)
# _ = manager.create_index(index_name)
# _ = manager.data_to_es_bulk(docs, index=index_name, count=count, _id='kegg_orthology_id')
# data from "brenda_reactions" collection
# index_name = 'brenda_reactions'
# count, docs = manager.data_from_mongo(server, db, username, password, authSource=authDB, collection_str=index_name)
# print(count)
# index_schema_path = str(Path('/root/karr_lab/karr_lab_aws_manager/karr_lab_aws_manager/elasticsearch_kl/mappings/rna_halflife.json').expanduser())
# with open(index_schema_path) as json_file:
# index_schema = json.load(json_file)
# index_manager = index_setting_file.IndexUtil(filter_dir=filter_dir, analyzer_dir=analyzer_dir)
# setting_file = index_manager.combine_files(_filter=True, analyzer=True, mappings=False)
# _ = manager.create_index_with_file(index_name, setting_file)
# _ = manager.data_to_es_bulk(docs, index=index_name, count=count, _id='_id')
# data from "metabolite_concentrations" collection
# index_name = 'metabolite_concentrations'
# _ = manager.delete_index(index_name)
# count, docs = manager.data_from_mongo(server, db, username, password, authSource=authDB, collection_str=index_name)
# print(count)
# index_manager = index_setting_file.IndexUtil(filter_dir=filter_dir, analyzer_dir=analyzer_dir)
# setting_file = index_manager.combine_files(_filter=True, analyzer=True, mappings=False)
# _ = manager.create_index_with_file(index_name, setting_file)
# _ = manager.data_to_es_bulk(docs, index=index_name, count=count, _id='inchikey')
# data from "rna_modification" collection
# index_name = 'rna_modification'
# _ = manager.delete_index(index_name)
# count, docs = manager.data_from_mongo(server, db, username, password, authSource=authDB, collection_str=index_name)
# print(count)
# index_manager = index_setting_file.IndexUtil(filter_dir=filter_dir, analyzer_dir=analyzer_dir)
# setting_file = index_manager.combine_files(_filter=True, analyzer=True, mappings=False)
# _ = manager.create_index_with_file(index_name, setting_file)
# _ = manager.data_to_es_bulk(docs, index=index_name, count=count, _id='_id')
r = manager.index_health_status()
print(r.content.decode('utf-8'))
