import inspect
import pickle
import sys
from abc import abstractmethod
import six
from decorator import decorator
from omniduct.duct import Duct
from omniduct.utils.config import config
from omniduct.utils.docs import quirk_docs
from ..utils.debug import logger
config.register('cache_fail_hard',
description='Raise exception if cache fails to save.',
default=False)
def cached_method(
id_str,
cache=lambda self: self.cache,
id_duct=lambda self, kwargs: "{}.{}".format(self.__class__.__name__, self.
name),
use_cache=lambda self, kwargs: kwargs.pop('use_cache', True),
renew=lambda self, kwargs: kwargs.pop('renew', False),
format=lambda self, kwargs: None,
serializer=lambda format: pickle.
dump, # Serializers accept obj, file handle and format.
deserializer=lambda format: pickle.load
): # Deserializers accept file handle and format.
import pandas
import six
import yaml
from decorator import decorator
from interface_meta import quirk_docs
from omniduct.duct import Duct
from omniduct.utils.config import config
from omniduct.utils.debug import logger
from omniduct.utils.decorators import function_args_as_kwargs, require_connection
from ._serializers import PickleSerializer
config.register(
'cache_fail_hard',
description=
'Raise an exception if a cache fails to save (otherwise errors are logged and suppressed).',
default=False)
def cached_method(key,
namespace=lambda self, kwargs:
(self.cache_namespace or "{}.{}".format(
self.__class__.__name__, self.name)),
cache=lambda self, kwargs: self.cache,
use_cache=lambda self, kwargs: kwargs.pop('use_cache', True),
renew=lambda self, kwargs: kwargs.pop('renew', False),
serializer=lambda self, kwargs: PickleSerializer(),
metadata=lambda self, kwargs: None):
"""
Wrap a method of a `Duct` class and add caching capabilities.
import pandas as pd
import pandas.io.sql
import sqlparse
from decorator import decorator
from jinja2 import Template
from omniduct.caches.base import cached_method
from omniduct.duct import Duct
from omniduct.utils.config import config
from omniduct.utils.debug import logger, logging_scope
from omniduct.utils.magics import MagicsProvider, process_line_arguments
logging.getLogger('requests').setLevel(logging.WARNING)
config.register(
'date_fields',
description='Default date fields to attempt to parse when databases',
type=list)
@decorator
def sanitize_sqlalchemy_statement(f, self, statement, *args, **kwargs):
try:
from sqlalchemy.sql.base import Executable
if isinstance(statement, Executable):
statement = str(
statement.compile(compile_kwargs={"literal_binds": True}))
except ImportError:
pass
return f(self, statement, *args, **kwargs)
from abc import abstractmethod
import six
import yaml
from decorator import decorator
from omniduct.duct import Duct
from omniduct.utils.config import config
from omniduct.utils.debug import logger
from omniduct.utils.decorators import function_args_as_kwargs
from omniduct.utils.docs import quirk_docs
from ._serializers import PickleSerializer
config.register('cache_fail_hard',
description='Raise exception if cache fails to save.',
default=False)
def cached_method(
key,
namespace=lambda self, kwargs: (
self.cache_namespace or "{}.{}".format(self.__class__.__name__, self.name)
),
cache=lambda self, kwargs: self.cache,
use_cache=lambda self, kwargs: kwargs.pop('use_cache', True),
renew=lambda self, kwargs: kwargs.pop('renew', False),
serializer=lambda self, kwargs: PickleSerializer(),
metadata=lambda self, kwargs: None
):
@decorator