import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output, State
from dash.exceptions import PreventUpdate
from app import app, server as application
from apps import initialApp, projectCreationApp, dataUploadApp, dataUpload, projectApp, importsApp, homepageApp, loginApp, projectCreation
from graphdb_builder import builder_utils
from graphdb_builder.builder import loader
from graphdb_builder.experiments import experiments_controller as eh
from report_manager import utils
import config.ckg_config as ckg_config
from worker import create_new_project, create_new_identifiers
from graphdb_connector import connector
log_config = ckg_config.report_manager_log
logger = builder_utils.setup_logging(log_config, key="index page")
try:
config = builder_utils.setup_config('builder')
directories = builder_utils.get_full_path_directories()
except Exception as err:
logger.error("Reading configuration > {}.".format(err))
cwd = os.path.abspath(os.path.dirname(__file__))
experimentDir = os.path.join(directories['dataDirectory'], 'experiments')
experimentsImportDir = directories['experimentsDirectory']
tmpDirectory = directories['tmpDirectory']
driver = connector.getGraphDatabaseConnectionConfiguration()
separator = config["separator"]
app.layout = dcc.Loading(children=[
import sys
import os
import json
import neo4j
import pandas as pd
import ckg_utils
from config import ckg_config
from graphdb_builder import builder_utils
log_config = ckg_config.graphdb_connector_log
logger = builder_utils.setup_logging(log_config, key="connector")
try:
cwd = os.path.abspath(os.path.dirname(__file__))
path = os.path.join(cwd, ckg_config.connector_config_file)
config = ckg_utils.get_configuration(path)
except Exception as err:
logger.error("Reading configuration > {}.".format(err))
def getGraphDatabaseConnectionConfiguration(configuration=None, database=None):
if configuration is None:
configuration = config
host = configuration['db_url']
port = configuration['db_port']
user = configuration['db_user']
password = configuration['db_password']
if database is not None:
host = host + '/' + database
import os
import sys
import config.ckg_config as ckg_config
import ckg_utils
from graphdb_connector import connector
from graphdb_builder import builder_utils
from graphdb_builder.builder import loader
from graphdb_builder.experiments import experiments_controller as eh
log_config = ckg_config.report_manager_log
logger = builder_utils.setup_logging(log_config, key="project_creation")
cwd = os.path.abspath(os.path.dirname(__file__))
experimentDir = os.path.join(cwd, '../../../data/experiments')
importDir = os.path.join(cwd, '../../../data/imports/experiments')
try:
config = builder_utils.get_config(config_name="clinical.yml",
data_type='experiments')
except Exception as err:
logger.error("Reading configuration > {}.".format(err))
def get_project_creation_queries():
"""
Reads the YAML file containing the queries relevant to user creation, parses the given stream and \
returns a Python object (dict[dict]).
:return: Nested dictionary.
"""
try:
import sys
import os.path
from graphdb_builder import builder_utils
from graphdb_builder.experiments.parsers import clinicalParser, proteomicsParser, wesParser
import config.ckg_config as ckg_config
log_config = ckg_config.graphdb_builder_log
logger = builder_utils.setup_logging(log_config, key="experiments_controller")
def generate_dataset_imports(projectId, dataType, dataset_import_dir):
stats = set()
builder_utils.checkDirectory(dataset_import_dir)
try:
if dataType in ['project', 'experimental_design', 'clinical']:
data = clinicalParser.parser(projectId, dataType)
for dtype, ot in data:
generate_graph_files(data[(dtype, ot)], dtype, projectId,
stats, ot, dataset_import_dir)
elif dataType in ["proteomics", "interactomics", "phosphoproteomics"]:
data = proteomicsParser.parser(projectId, dataType)
for dtype, ot in data:
generate_graph_files(data[(dtype, ot)], dtype, projectId,
stats, ot, dataset_import_dir)
elif dataType == "wes":
data = wesParser.parser(projectId)
for dtype, ot in data:
generate_graph_files(data[(dtype, ot)], dtype, projectId,
stats, ot, dataset_import_dir)
else:
raise Exception(
import os.path
import sys
import config.ckg_config as ckg_config
from graphdb_builder import builder_utils
from graphdb_builder.databases.parsers import *
from joblib import Parallel, delayed
from datetime import date
log_config = ckg_config.graphdb_builder_log
logger = builder_utils.setup_logging(log_config, key="database_controller")
try:
dbconfig = builder_utils.setup_config('databases')
except Exception as err:
logger.error("Reading configuration > {}.".format(err))
def parseDatabase(importDirectory, database, download=True):
stats = set()
updated_on = None
if download:
updated_on = str(date.today())
try:
logger.info("Parsing database {}".format(database))
if database.lower() == "jensenlab":
result = jensenlabParser.parser(dbconfig["databasesDir"], download)
for qtype in result:
relationships, header, outputfileName = result[qtype]
outputfile = os.path.join(importDirectory, outputfileName)
builder_utils.write_relationships(relationships, header, outputfile)
logger.info("Database {} - Number of {} relationships: {}".format(database, qtype, len(relationships)))
"""
import os.path
from datetime import datetime
import pandas as pd
from joblib import Parallel, delayed
from uuid import uuid4
import config.ckg_config as ckg_config
from graphdb_builder.ontologies import ontologies_controller as oh
from graphdb_builder.databases import databases_controller as dh
from graphdb_builder.experiments import experiments_controller as eh
from graphdb_builder.users import users_controller as uh
from graphdb_builder import builder_utils
log_config = ckg_config.graphdb_builder_log
logger = builder_utils.setup_logging(log_config, key="importer")
import_id = uuid4()
try:
cwd = os.path.abspath(os.path.dirname(__file__))
config = builder_utils.setup_config('builder')
directories = builder_utils.get_full_path_directories()
oconfig = builder_utils.setup_config('ontologies')
dbconfig = builder_utils.setup_config('databases')
econfig = builder_utils.setup_config('experiments')
uconfig = builder_utils.setup_config('users')
except Exception as err:
logger.error("importer - Reading configuration > {}.".format(err))
START_TIME = datetime.now()
The compressed files for each type of update are named accordingly and saved in the archive/ folder
in data/.
"""
import os
import sys
import re
from datetime import datetime
import config.ckg_config as ckg_config
import ckg_utils
from graphdb_connector import connector
from graphdb_builder import builder_utils
cwd = os.path.abspath(os.path.dirname(__file__))
log_config = ckg_config.graphdb_builder_log
logger = builder_utils.setup_logging(log_config, key="loader")
START_TIME = datetime.now()
try:
config = builder_utils.setup_config('builder')
directories = builder_utils.get_full_path_directories()
except Exception as err:
logger.error("Reading configuration > {}.".format(err))
def load_into_database(driver, queries, requester):
"""
This function runs the queries provided in the graph database using a neo4j driver.
:param driver: neo4j driver, which provides the connection to the neo4j graph database.
:type driver: neo4j driver
import os
import sys
import argparse
import pandas as pd
from datetime import datetime, timedelta
from passlib.hash import bcrypt
import config.ckg_config as ckg_config
from graphdb_connector import connector
from graphdb_builder import builder_utils
from graphdb_builder.users import users_controller as uh
log_config = ckg_config.graphdb_builder_log
logger = builder_utils.setup_logging(log_config, key='user_creation')
try:
config = builder_utils.setup_config('builder')
directories = builder_utils.get_full_path_directories()
uconfig = builder_utils.setup_config('users')
except Exception as err:
logger.error("Reading configuration > {}.".format(err))
cwd = os.path.abspath(os.path.dirname(__file__))
def create_user_from_dict(driver, data):
"""
Creates graph database node for new user and adds properties to the node.
:param driver: neo4j driver, which provides the connection to the neo4j graph database.
:param dict data: dictionary with the user information).
"""
from graphdb_builder import mapping as mp, builder_utils
import config.ckg_config as ckg_config
from graphdb_builder.ontologies.parsers import *
import os.path
import pandas as pd
import csv
from datetime import date
import sys
log_config = ckg_config.graphdb_builder_log
logger = builder_utils.setup_logging(log_config, key="ontologies_controller")
try:
config = builder_utils.setup_config('ontologies')
except Exception as err:
logger.error("Reading configuration > {}.".format(err))
def entries_to_remove(entries, the_dict):
"""
This function removes pairs from a given dictionary, based on a list of provided keys.
:param list entries: list of keys to be deleted from dictionary.
:param dict the_dict: dictionary.
:return: The original dictionary minus the key,value pairs from the provided entries list.
"""
for key in entries:
if key in the_dict:
del the_dict[key]
import os
import sys
import re
import pandas as pd
import numpy as np
import config.ckg_config as ckg_config
import ckg_utils
from graphdb_connector import connector
from graphdb_builder import builder_utils
from graphdb_connector import query_utils
from analytics_core.viz import viz
log_config = ckg_config.graphdb_builder_log
logger = builder_utils.setup_logging(log_config, key="data_upload")
cwd = os.path.abspath(os.path.dirname(__file__))
def get_data_upload_queries():
"""
Reads the YAML file containing the queries relevant to parsing of clinical data and \
returns a Python object (dict[dict]).
:return: Nested dictionary.
"""
try:
queries_path = "../queries/data_upload_cypher.yml"
data_upload_cypher = ckg_utils.get_queries(
os.path.join(cwd, queries_path))
except Exception as err:
exc_type, exc_obj, exc_tb = sys.exc_info()
import os
import sys
import re
import argparse
import pandas as pd
import numpy as np
from datetime import datetime, timedelta
from passlib.hash import bcrypt
import ckg_utils
import config.ckg_config as ckg_config
from graphdb_connector import connector
from graphdb_builder import builder_utils
log_config = ckg_config.graphdb_builder_log
logger = builder_utils.setup_logging(log_config, key='users_controller')
try:
config = builder_utils.setup_config('users')
except Exception as err:
logger.error("Reading configuration > {}.".format(err))
cwd = os.path.abspath(os.path.dirname(__file__))
def parseUsersFile(importDirectory, expiration=365):
"""
Creates new user in the graph database and corresponding node, through the following steps:
1. Generates new user identifier
2. Checks if a user with given properties already exists in the database. If not:
3. Creates new local user (access to graph database)
4. Saves data to tab-delimited file.
