import dash_core_components as dcc import dash_html_components as html from dash.dependencies import Input, Output, State from dash.exceptions import PreventUpdate from app import app, server as application from apps import initialApp, projectCreationApp, dataUploadApp, dataUpload, projectApp, importsApp, homepageApp, loginApp, projectCreation from graphdb_builder import builder_utils from graphdb_builder.builder import loader from graphdb_builder.experiments import experiments_controller as eh from report_manager import utils import config.ckg_config as ckg_config from worker import create_new_project, create_new_identifiers from graphdb_connector import connector log_config = ckg_config.report_manager_log logger = builder_utils.setup_logging(log_config, key="index page") try: config = builder_utils.setup_config('builder') directories = builder_utils.get_full_path_directories() except Exception as err: logger.error("Reading configuration > {}.".format(err)) cwd = os.path.abspath(os.path.dirname(__file__)) experimentDir = os.path.join(directories['dataDirectory'], 'experiments') experimentsImportDir = directories['experimentsDirectory'] tmpDirectory = directories['tmpDirectory'] driver = connector.getGraphDatabaseConnectionConfiguration() separator = config["separator"] app.layout = dcc.Loading(children=[
import sys import os import json import neo4j import pandas as pd import ckg_utils from config import ckg_config from graphdb_builder import builder_utils log_config = ckg_config.graphdb_connector_log logger = builder_utils.setup_logging(log_config, key="connector") try: cwd = os.path.abspath(os.path.dirname(__file__)) path = os.path.join(cwd, ckg_config.connector_config_file) config = ckg_utils.get_configuration(path) except Exception as err: logger.error("Reading configuration > {}.".format(err)) def getGraphDatabaseConnectionConfiguration(configuration=None, database=None): if configuration is None: configuration = config host = configuration['db_url'] port = configuration['db_port'] user = configuration['db_user'] password = configuration['db_password'] if database is not None: host = host + '/' + database
import os import sys import config.ckg_config as ckg_config import ckg_utils from graphdb_connector import connector from graphdb_builder import builder_utils from graphdb_builder.builder import loader from graphdb_builder.experiments import experiments_controller as eh log_config = ckg_config.report_manager_log logger = builder_utils.setup_logging(log_config, key="project_creation") cwd = os.path.abspath(os.path.dirname(__file__)) experimentDir = os.path.join(cwd, '../../../data/experiments') importDir = os.path.join(cwd, '../../../data/imports/experiments') try: config = builder_utils.get_config(config_name="clinical.yml", data_type='experiments') except Exception as err: logger.error("Reading configuration > {}.".format(err)) def get_project_creation_queries(): """ Reads the YAML file containing the queries relevant to user creation, parses the given stream and \ returns a Python object (dict[dict]). :return: Nested dictionary. """ try:
import sys import os.path from graphdb_builder import builder_utils from graphdb_builder.experiments.parsers import clinicalParser, proteomicsParser, wesParser import config.ckg_config as ckg_config log_config = ckg_config.graphdb_builder_log logger = builder_utils.setup_logging(log_config, key="experiments_controller") def generate_dataset_imports(projectId, dataType, dataset_import_dir): stats = set() builder_utils.checkDirectory(dataset_import_dir) try: if dataType in ['project', 'experimental_design', 'clinical']: data = clinicalParser.parser(projectId, dataType) for dtype, ot in data: generate_graph_files(data[(dtype, ot)], dtype, projectId, stats, ot, dataset_import_dir) elif dataType in ["proteomics", "interactomics", "phosphoproteomics"]: data = proteomicsParser.parser(projectId, dataType) for dtype, ot in data: generate_graph_files(data[(dtype, ot)], dtype, projectId, stats, ot, dataset_import_dir) elif dataType == "wes": data = wesParser.parser(projectId) for dtype, ot in data: generate_graph_files(data[(dtype, ot)], dtype, projectId, stats, ot, dataset_import_dir) else: raise Exception(
import os.path import sys import config.ckg_config as ckg_config from graphdb_builder import builder_utils from graphdb_builder.databases.parsers import * from joblib import Parallel, delayed from datetime import date log_config = ckg_config.graphdb_builder_log logger = builder_utils.setup_logging(log_config, key="database_controller") try: dbconfig = builder_utils.setup_config('databases') except Exception as err: logger.error("Reading configuration > {}.".format(err)) def parseDatabase(importDirectory, database, download=True): stats = set() updated_on = None if download: updated_on = str(date.today()) try: logger.info("Parsing database {}".format(database)) if database.lower() == "jensenlab": result = jensenlabParser.parser(dbconfig["databasesDir"], download) for qtype in result: relationships, header, outputfileName = result[qtype] outputfile = os.path.join(importDirectory, outputfileName) builder_utils.write_relationships(relationships, header, outputfile) logger.info("Database {} - Number of {} relationships: {}".format(database, qtype, len(relationships)))
""" import os.path from datetime import datetime import pandas as pd from joblib import Parallel, delayed from uuid import uuid4 import config.ckg_config as ckg_config from graphdb_builder.ontologies import ontologies_controller as oh from graphdb_builder.databases import databases_controller as dh from graphdb_builder.experiments import experiments_controller as eh from graphdb_builder.users import users_controller as uh from graphdb_builder import builder_utils log_config = ckg_config.graphdb_builder_log logger = builder_utils.setup_logging(log_config, key="importer") import_id = uuid4() try: cwd = os.path.abspath(os.path.dirname(__file__)) config = builder_utils.setup_config('builder') directories = builder_utils.get_full_path_directories() oconfig = builder_utils.setup_config('ontologies') dbconfig = builder_utils.setup_config('databases') econfig = builder_utils.setup_config('experiments') uconfig = builder_utils.setup_config('users') except Exception as err: logger.error("importer - Reading configuration > {}.".format(err)) START_TIME = datetime.now()
The compressed files for each type of update are named accordingly and saved in the archive/ folder in data/. """ import os import sys import re from datetime import datetime import config.ckg_config as ckg_config import ckg_utils from graphdb_connector import connector from graphdb_builder import builder_utils cwd = os.path.abspath(os.path.dirname(__file__)) log_config = ckg_config.graphdb_builder_log logger = builder_utils.setup_logging(log_config, key="loader") START_TIME = datetime.now() try: config = builder_utils.setup_config('builder') directories = builder_utils.get_full_path_directories() except Exception as err: logger.error("Reading configuration > {}.".format(err)) def load_into_database(driver, queries, requester): """ This function runs the queries provided in the graph database using a neo4j driver. :param driver: neo4j driver, which provides the connection to the neo4j graph database. :type driver: neo4j driver
import os import sys import argparse import pandas as pd from datetime import datetime, timedelta from passlib.hash import bcrypt import config.ckg_config as ckg_config from graphdb_connector import connector from graphdb_builder import builder_utils from graphdb_builder.users import users_controller as uh log_config = ckg_config.graphdb_builder_log logger = builder_utils.setup_logging(log_config, key='user_creation') try: config = builder_utils.setup_config('builder') directories = builder_utils.get_full_path_directories() uconfig = builder_utils.setup_config('users') except Exception as err: logger.error("Reading configuration > {}.".format(err)) cwd = os.path.abspath(os.path.dirname(__file__)) def create_user_from_dict(driver, data): """ Creates graph database node for new user and adds properties to the node. :param driver: neo4j driver, which provides the connection to the neo4j graph database. :param dict data: dictionary with the user information). """
from graphdb_builder import mapping as mp, builder_utils import config.ckg_config as ckg_config from graphdb_builder.ontologies.parsers import * import os.path import pandas as pd import csv from datetime import date import sys log_config = ckg_config.graphdb_builder_log logger = builder_utils.setup_logging(log_config, key="ontologies_controller") try: config = builder_utils.setup_config('ontologies') except Exception as err: logger.error("Reading configuration > {}.".format(err)) def entries_to_remove(entries, the_dict): """ This function removes pairs from a given dictionary, based on a list of provided keys. :param list entries: list of keys to be deleted from dictionary. :param dict the_dict: dictionary. :return: The original dictionary minus the key,value pairs from the provided entries list. """ for key in entries: if key in the_dict: del the_dict[key]
import os import sys import re import pandas as pd import numpy as np import config.ckg_config as ckg_config import ckg_utils from graphdb_connector import connector from graphdb_builder import builder_utils from graphdb_connector import query_utils from analytics_core.viz import viz log_config = ckg_config.graphdb_builder_log logger = builder_utils.setup_logging(log_config, key="data_upload") cwd = os.path.abspath(os.path.dirname(__file__)) def get_data_upload_queries(): """ Reads the YAML file containing the queries relevant to parsing of clinical data and \ returns a Python object (dict[dict]). :return: Nested dictionary. """ try: queries_path = "../queries/data_upload_cypher.yml" data_upload_cypher = ckg_utils.get_queries( os.path.join(cwd, queries_path)) except Exception as err: exc_type, exc_obj, exc_tb = sys.exc_info()
import os import sys import re import argparse import pandas as pd import numpy as np from datetime import datetime, timedelta from passlib.hash import bcrypt import ckg_utils import config.ckg_config as ckg_config from graphdb_connector import connector from graphdb_builder import builder_utils log_config = ckg_config.graphdb_builder_log logger = builder_utils.setup_logging(log_config, key='users_controller') try: config = builder_utils.setup_config('users') except Exception as err: logger.error("Reading configuration > {}.".format(err)) cwd = os.path.abspath(os.path.dirname(__file__)) def parseUsersFile(importDirectory, expiration=365): """ Creates new user in the graph database and corresponding node, through the following steps: 1. Generates new user identifier 2. Checks if a user with given properties already exists in the database. If not: 3. Creates new local user (access to graph database) 4. Saves data to tab-delimited file.