def task_wrapper(python_callable, **kwargs):
"""
Overrides configuration with config from airflow.
:param python_callable:
:param kwargs:
:return:
"""
# get dag config provided
dag_run = kwargs.get('dag_run')
dag_conf = {}
logger = get_logger()
config = get_config()
# config.update({'data_root': ''})
if dag_run:
dag_conf = dag_run.conf
# remove this since to send every other argument to the python callable.
del kwargs['dag_run']
# overrides values
config.update(dag_conf)
logger.info("Config")
logger.info(config)
return python_callable(to_string=True, config=config)
import shutil
import sys
import time
import yaml
from bmt import Toolkit
from collections import defaultdict
from enum import Enum
from io import StringIO
from roger.Config import get_default_config as get_config
from roger.roger_util import get_logger
from roger.components.data_conversion_utils import TypeConversionUtil
from redisgraph_bulk_loader.bulk_insert import bulk_insert
from roger.roger_db import RedisGraph
from string import Template
log = get_logger()
config = get_config()
class SchemaType(Enum):
""" High level semantic metatdata concepts.
Categories are classes in an ontological model like Biolink.
Predicates are links between nodes. """
CATEGORY = "category"
PREDICATE = "predicate"
class FileFormat(Enum):
""" File formats this module knows about. """
JSON = "json"
YAML = "yaml"
import logging
import redis
from redisgraph import Node, Edge, Graph
from redis.exceptions import ResponseError
from roger.roger_util import get_logger
logger = get_logger()
class RedisGraph:
""" Graph abstraction over RedisGraph. A thin wrapper but provides us some options. """
def __init__(self,
host='localhost',
port=6379,
graph='default',
password=''):
""" Construct a connection to Redis Graph. """
self.r = redis.Redis(host=host, port=port, password=password)
self.redis_graph = Graph(graph, self.r)
def add_node(self, identifier=None, label=None, properties=None):
""" Add a node with the given label and properties. """
logger.debug(
f"--adding node id:{identifier} label:{label} prop:{properties}")
if identifier and properties:
properties['id'] = identifier
node = Node(node_id=identifier,
alias=identifier,
label=label,
properties=properties)
self.redis_graph.add_node(node)