def main():
# FIND DIRECTORIES
load_files_directory = get_load_dir()
load_dir = load_files_directory + '/load_files_disease/'
extract_dir = get_load_dir() + 'extracted_disease/'
descriptions_csv_path = '../config/table_descriptions_03_01.csv'
# CREATE DICTIONARY FROM SCHEMA
db_dict = get_schema(descriptions_csv_path)['OmniSeqKnowledgebase2']
# LOAD FILES
load_files_dict = create_load_files_dict(db_dict, load_dir)
user_id = get_loader_user_id(extract_dir)
data_dict = {'loader_id': user_id}
data_dict['do_disease_dict'] = get_do_disease_dict(extract_dir)
data_dict['do_disease_pmid'] = get_definition_urls_dict(extract_dir)
data_dict['do_parents'] = get_do_disease_parents_dict(extract_dir)
data_dict['ref_by_pmid'] = get_literature_reference_dict(
load_files_directory)
# [graph_id] = [name(of journal)]
data_dict['journal_dict'] = get_journal_dict(load_files_directory)
# [graph_id] = {[first initial], [last initial]}
data_dict['author_dict'] = get_author_dict(load_files_directory)
do_disease_loader(load_files_dict, data_dict)
print()
def main():
print(datetime.datetime.now().strftime("%H:%M:%S"))
my_db = None
my_cursor = None
server_read: str = '165.227.89.140'
descriptions_csv_path = '../config/table_descriptions_02_01.csv'
db_dict = get_schema(descriptions_csv_path)
load_dir = get_load_dir() + 'extracted/'
dict_of_conversion_functions, dict_of_data = create_data_and_conversion_dictionaries(
server_read)
delete_load_files(load_dir)
write_load_files_using_func(db_dict, dict_of_data,
dict_of_conversion_functions, load_dir)
try:
my_db = get_local_db_connection()
my_cursor = my_db.cursor(buffered=True)
for db_name in sorted(db_dict.keys()):
maybe_create_and_select_database(my_cursor, db_name)
for table_name in sorted(db_dict[db_name].keys()):
drop_table_if_exists(my_cursor, table_name)
create_table(my_cursor, table_name, db_name, db_dict)
load_table(my_cursor, table_name,
db_dict[db_name][table_name]['col_order'], load_dir)
my_db.commit()
except mysql.connector.Error as error:
print("Failed in MySQL: {}".format(error))
finally:
if (my_db.is_connected()):
my_cursor.close()
print(datetime.datetime.now().strftime("%H:%M:%S"))
def write_csv_to_sql(descriptions_csv_path, should_drop_database):
load_dir = get_load_dir()
db_dict = get_schema(descriptions_csv_path)
try:
my_db = get_local_db_connection()
my_cursor = my_db.cursor(buffered=True)
if should_drop_database:
db_name = list(db_dict.keys())[0]
drop_database(my_cursor, db_name)
for db_name in sorted(db_dict.keys()):
maybe_create_and_select_database(my_cursor, db_name)
for table_name in sorted(db_dict[db_name].keys()):
drop_table_if_exists(my_cursor, table_name)
create_table(my_cursor, table_name, db_name, db_dict)
load_table(my_cursor, table_name,
db_dict[db_name][table_name]['col_order'], load_dir)
my_db.commit()
except mysql.connector.Error as error:
print("Failed in MySQL: {}".format(error))
finally:
if (my_db.is_connected()):
my_cursor.close()
def write_graphql(should_erase):
print(datetime.datetime.now().strftime("%H:%M:%S"))
schema_graphql = get_most_recent_graphql_schema()
# server_write: str = 'localhost'
# this is the prod server
# server_write: str = '165.227.89.140'
# this is the test server
# server_write:str ='161.35.115.213'
server_write: str = write_to_prod()
my_db = None
my_cursor = None
if should_erase:
erase_neo4j(schema_graphql, server_write)
mutations = get_mutations(server_write)
db_dict = get_schema('../config/table_descriptions_03_01.csv')
try:
my_db = get_local_db_connection()
my_cursor = my_db.cursor(buffered=True)
create_graphql_objects(my_cursor, mutations, 'createUser', db_dict,
'OmniSeqKnowledgebase2', 'User', server_write)
create_graphql_objects(my_cursor, mutations, 'createAuthor', db_dict,
'OmniSeqKnowledgebase2', 'Author', server_write)
create_graphql_objects(my_cursor, mutations, 'createJournal', db_dict,
'OmniSeqKnowledgebase2', 'Journal',
server_write)
create_graphql_objects(my_cursor, mutations,
'createLiteratureReference', db_dict,
'OmniSeqKnowledgebase2', 'LiteratureReference',
server_write)
add_to_graphql_objects(my_cursor, mutations,
'addLiteratureReferenceJournal', db_dict,
'OmniSeqKnowledgebase2', 'LiteratureReference',
server_write)
add_list_to_graphql_objects(my_cursor, mutations,
'addLiteratureReferenceAuthors', db_dict,
'OmniSeqKnowledgebase2',
'LiteratureReference_Author', server_write)
create_graphql_objects(my_cursor, mutations, 'createInternetReference',
db_dict, 'OmniSeqKnowledgebase2',
'InternetReference', server_write)
create_graphql_objects(my_cursor, mutations, 'createEditableStatement',
db_dict, 'OmniSeqKnowledgebase2',
'EditableStatement', server_write)
add_list_to_graphql_objects(my_cursor, mutations,
'addEditableStatementReferences', db_dict,
'OmniSeqKnowledgebase2',
'EditableStatement_LiteratureReference',
server_write)
add_list_to_graphql_objects(my_cursor, mutations,
'addEditableStatementReferences', db_dict,
'OmniSeqKnowledgebase2',
'EditableStatement_InternetReference',
server_write)
add_to_graphql_objects(my_cursor, mutations,
'addEditableStatementEditor', db_dict,
'OmniSeqKnowledgebase2', 'EditableStatement',
server_write)
syn_dict = get_syn_dict('../load_files/Synonym.csv')
create_graphql_objects_with_list(my_cursor, mutations,
'createEditableSynonymList', db_dict,
'OmniSeqKnowledgebase2',
'EditableSynonymList', syn_dict,
server_write)
add_to_graphql_objects(my_cursor, mutations,
'addEditableSynonymListEditor', db_dict,
'OmniSeqKnowledgebase2', 'EditableSynonymList',
server_write)
create_graphql_objects(my_cursor, mutations, 'createJaxGene', db_dict,
'OmniSeqKnowledgebase2', 'JaxGene',
server_write)
add_to_graphql_objects(my_cursor, mutations,
'addJaxGeneCanonicalTranscript', db_dict,
'OmniSeqKnowledgebase2', 'JaxGene',
server_write)
add_to_graphql_objects(my_cursor, mutations, 'addJaxGeneDescription',
db_dict, 'OmniSeqKnowledgebase2', 'JaxGene',
server_write)
add_to_graphql_objects(my_cursor, mutations, 'addJaxGeneSynonyms',
db_dict, 'OmniSeqKnowledgebase2', 'JaxGene',
server_write)
create_graphql_objects(my_cursor, mutations, 'createMyGeneInfoGene',
db_dict, 'OmniSeqKnowledgebase2',
'MyGeneInfoGene', server_write)
add_to_graphql_objects(my_cursor, mutations,
'addMyGeneInfoGeneDescription', db_dict,
'OmniSeqKnowledgebase2', 'MyGeneInfoGene',
server_write)
create_graphql_objects(my_cursor, mutations, 'createUniprotEntry',
db_dict, 'OmniSeqKnowledgebase2',
'UniprotEntry', server_write)
add_to_graphql_objects(my_cursor, mutations, 'addUniprotEntryFunction',
db_dict, 'OmniSeqKnowledgebase2',
'UniprotEntry', server_write)
add_to_graphql_objects(my_cursor, mutations, 'addUniprotEntryGene',
db_dict, 'OmniSeqKnowledgebase2',
'UniprotEntry', server_write)
create_graphql_objects(my_cursor, mutations, 'createOmniGene', db_dict,
'OmniSeqKnowledgebase2', 'OmniGene',
server_write)
add_to_graphql_objects(my_cursor, mutations, 'addOmniGeneTranscript',
db_dict, 'OmniSeqKnowledgebase2', 'OmniGene',
server_write)
add_to_graphql_objects(my_cursor, mutations,
'addOmniGeneGeneDescription', db_dict,
'OmniSeqKnowledgebase2', 'OmniGene',
server_write)
add_to_graphql_objects(my_cursor, mutations,
'addOmniGeneMyGeneInfoGene', db_dict,
'OmniSeqKnowledgebase2', 'OmniGene',
server_write)
add_to_graphql_objects(my_cursor, mutations, 'addOmniGeneJaxGene',
db_dict, 'OmniSeqKnowledgebase2', 'OmniGene',
server_write)
add_to_graphql_objects(my_cursor, mutations, 'addOmniGeneUniprotEntry',
db_dict, 'OmniSeqKnowledgebase2', 'OmniGene',
server_write)
add_to_graphql_objects(my_cursor, mutations,
'addOmniGeneOncogenicCategory', db_dict,
'OmniSeqKnowledgebase2', 'OmniGene',
server_write)
add_to_graphql_objects(my_cursor, mutations, 'addOmniGeneSynonyms',
db_dict, 'OmniSeqKnowledgebase2', 'OmniGene',
server_write)
except mysql.connector.Error as error:
print("Failed in MySQL: {}".format(error))
finally:
if (my_db.is_connected()):
my_cursor.close()
print(datetime.datetime.now().strftime("%H:%M:%S"))
def migrate_and_extend(should_migrate, should_generate_sql):
print(datetime.datetime.now().strftime("%H:%M:%S"))
files_to_copy = [
'Author', 'EditableStatement', 'EditableStatement_InternetReference',
"EditableStatement_LiteratureReference", 'InternetReference',
'Journal', 'LiteratureReference', 'LiteratureReference_Author',
'MyGeneInfoGene', 'UniprotEntry', 'User'
]
# files_to_create = ['EditableInt','EditableInt_LiteratureReference','EditableInt_InternetReference',
# 'EditableBoolean', 'EditableBoolean_LiteratureReference', 'EditableBoolean_InternetReference',
# 'EditableCopyChange', 'EditableCopyChange_LiteratureReference', 'EditableCopyChange_InternetReference',
# 'EditableProteinEffect', 'EditableProteinEffect_LiteratureReference', 'EditableProteinEffect_InternetReference',
# 'EditableOmniGeneReference', 'EditableOmniGeneReference_LiteratureReference', 'EditableOmniGeneReference_InternetReference',
# 'JaxVariant','ClinVarVariant','OncoTreeOccurrence','HotSpotVariant','GOVariant',
# 'VariantSNVIndel','VariantRegion','VariantCNV','VariantFusion','GenomicVariantMarker']
load_dir = get_load_dir()
extract_dir = get_load_dir() + 'extracted/'
descriptions_csv_path = '../config/table_descriptions_03_01.csv'
db_dict = get_schema(descriptions_csv_path)['OmniSeqKnowledgebase2']
if should_migrate:
copy_files(files_to_copy, extract_dir, load_dir)
for file in files_to_copy:
db_dict.pop(file)
if not should_migrate:
db_dict.pop('JaxGene')
db_dict.pop('OmniGene')
db_dict.pop('EditableSynonymList')
load_files_dict = create_load_files_dict(db_dict, load_dir)
open_file_for_append('EditableStatement', load_dir, load_files_dict)
open_file_for_append('EditableStatement_LiteratureReference', load_dir,
load_files_dict)
open_file_for_append('EditableStatement_InternetReference', load_dir,
load_files_dict)
open_file_for_append('LiteratureReference', load_dir, load_files_dict)
open_file_for_append('Author', load_dir, load_files_dict)
open_file_for_append('LiteratureReference_Author', load_dir,
load_files_dict)
open_file_for_append('Journal', load_dir, load_files_dict)
open_file_for_append('InternetReference', load_dir, load_files_dict)
user_id = get_loader_user_id(extract_dir)
data_dict = {'loader_id': user_id}
if should_migrate:
jax_transcript_dict = migrate_jax_gene(load_files_dict, extract_dir,
user_id)
migrate_omnigene(load_files_dict, extract_dir, user_id,
jax_transcript_dict)
data_dict['jax_gene_dict'] = get_jax_gene_dict(extract_dir)
data_dict['ref_by_pmid'] = get_literature_reference_dict(extract_dir)
data_dict['journal_dict'] = get_journal_dict(extract_dir)
data_dict['author_dict'] = get_author_dict(extract_dir)
data_dict['omnigene_dict'] = get_omnigene_dict(extract_dir)
variant_files = [
'JaxVariant', 'ClinVarVariant', 'HotSpotVariant', 'GOVariant',
'VariantSNVIndel', 'VariantRegion', 'VariantCNV', 'VariantFusion',
'GenomicVariantMarker'
]
for file in variant_files:
loader_name = file.lower() + '_loader(load_files_dict,data_dict)'
eval(loader_name)
close_load_files(load_files_dict)
if should_generate_sql:
db_dict = get_schema(descriptions_csv_path)
try:
my_db = get_local_db_connection()
my_cursor = my_db.cursor(buffered=True)
for db_name in sorted(db_dict.keys()):
maybe_create_and_select_database(my_cursor, db_name)
for table_name in sorted(db_dict[db_name].keys()):
drop_table_if_exists(my_cursor, table_name)
create_table(my_cursor, table_name, db_name, db_dict)
load_table(my_cursor, table_name,
db_dict[db_name][table_name]['col_order'],
load_dir)
my_db.commit()
except mysql.connector.Error as error:
print("Failed in MySQL: {}".format(error))
finally:
if (my_db.is_connected()):
my_cursor.close()
print(datetime.datetime.now().strftime("%H:%M:%S"))