def __init__(self, node_name: str, lru_size: int):
"""
Initialize the first two layers of a multi-layered cache
:param node_name:
:param lru_size:
"""
super(LayeredCache, self).__init__()
# Track the name of the node Type. This should be whatever
# unique identifier can be used in dgraph queries.
self.node_name = node_name
# Initialize a Least Recently Used in-memory cache
# with a maximum number of elements.
# layer 1
self.lru_local_cache = LRUCache(maxsize=lru_size)
# Initialize a redis client.
# layer 2 cache
self.redis = Redis("localhost")
# This should be set to True if we should add a timeout to the
# redis key value store values.
self.set_timeout = False
class InMemoryAgentCache(AgentCache):
cache = LRUCache(maxsize=100)
def set(self, bot: Text, agent: Agent):
"""
loads bot agent in LRU cache
:param bot: bot id
:param agent: bot agent
:return: None
"""
if bot in InMemoryAgentCache.cache.keys():
InMemoryAgentCache.cache.pop(bot)
InMemoryAgentCache.cache.__setitem__(bot, agent)
def get(self, bot: Text) -> Agent:
"""
fetches bot agent from LRU cache
:param bot: bot id
:return: Agent object
"""
return InMemoryAgentCache.cache.get(bot)
def is_exists(self, bot: Text) -> bool:
"""
checks if bot agent exist in LRU cache
:param bot: bot id
:return: True/False
"""
return bot in InMemoryAgentCache.cache.keys()
class InMemoryAgentCache(AgentCache):
cache = LRUCache(maxsize=100)
@staticmethod
def set(bot: Text, agent: Agent):
InMemoryAgentCache.cache.__setitem__(bot, agent)
@staticmethod
def get(bot: Text) -> Agent:
InMemoryAgentCache.get(bot)
class InMemoryAgentCache(AgentCache):
cache = LRUCache(maxsize=100)
@staticmethod
def set(bot: Text, agent: Agent):
InMemoryAgentCache.cache.__setitem__(bot, agent)
@staticmethod
def get(bot: Text) -> Agent:
return InMemoryAgentCache.cache.get(bot)
@staticmethod
def is_exists(bot: Text) -> Agent:
return bot in InMemoryAgentCache.cache.keys()
class Scene:
@classmethod
@cached(cache=LRUCache(maxsize=32))
def fromfile(cls, filepath):
filepath = os.fspath(filepath)
c = cls(**yaml.safe_load(open(filepath)))
c.origin = filepath
return c
def __init__(self, title, content, author='', choices=[]):
self.title = title
self.content = content
self.author = author
self.choices = {recook(c['capture']): c['to'] for c in choices}
def pick(self, key):
next_ = self.choices[key]
if type(next_) is str:
next_ = self.fromfile(next_)
return next_
def examine(self, ans):
for k in self.choices:
if re.match(k, ans):
return self.pick(k)
return None
def reload(self):
if 'origin' in self.__dict__:
print('\0337\033[1;200H[Reloaded]\0338')
return self.fromfile(self.origin)
return self
from typing import Dict, List, Union
from typing import get_type_hints as typing_get_type_hints
from django.conf import settings
from django.utils.html import _json_script_escapes
from django.utils.safestring import mark_safe
import shortuuid
from cachetools.lru import LRUCache
import django_unicorn
from django_unicorn.errors import UnicornCacheError
logger = logging.getLogger(__name__)
type_hints_cache = LRUCache(maxsize=100)
function_signature_cache = LRUCache(maxsize=100)
def generate_checksum(data: Union[str, bytes]) -> str:
"""
Generates a checksum for the passed-in data.
"""
if isinstance(data, str):
data_bytes = str.encode(data)
else:
data_bytes = data
checksum = hmac.new(
str.encode(settings.SECRET_KEY),
from bs4 import BeautifulSoup
from cachetools.lru import LRUCache
from .call_method_parser import InvalidKwarg, parse_call_method_name, parse_kwarg
from .components import UnicornField, UnicornView
from .decorators import timed
from .errors import UnicornViewError
from .message import ComponentRequest, Return
from .serializer import dumps, loads
from .settings import get_cache_alias, get_serial_enabled, get_serial_timeout
from .utils import generate_checksum, get_cacheable_component
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
type_hints_cache = LRUCache(maxsize=100)
def handle_error(view_func):
"""
Returns a JSON response with an error if necessary.
"""
def wrapped_view(*args, **kwargs):
try:
return view_func(*args, **kwargs)
except UnicornViewError as e:
return JsonResponse({"error": str(e)})
except AssertionError as e:
return JsonResponse({"error": str(e)})
return wraps(view_func)(wrapped_view)
class SpacyNLP(Component):
cache = LRUCache(maxsize=10)
defaults = {
# name of the language model to load - if it is not set
# we will be looking for a language model that is named
# after the language of the model, e.g. `en`
"model": None,
# when retrieving word vectors, this will decide if the casing
# of the word is relevant. E.g. `hello` and `Hello` will
# retrieve the same vector, if set to `False`. For some
# applications and models it makes sense to differentiate
# between these two words, therefore setting this to `True`.
"case_sensitive": False,
}
def __init__(self,
component_config: Dict[Text, Any] = None,
nlp: "Language" = None) -> None:
self.nlp = nlp
super().__init__(component_config)
@staticmethod
def load_model(spacy_model_name: Text) -> "Language":
"""Try loading the model, catching the OSError if missing."""
import spacy
try:
if spacy_model_name not in SpacyNLP.cache.keys():
SpacyNLP.cache.__setitem__(
spacy_model_name,
spacy.load(spacy_model_name, disable=["parser"]))
return SpacyNLP.cache.get(spacy_model_name)
except OSError:
raise InvalidModelError(
"Model '{}' is not a linked spaCy model. "
"Please download and/or link a spaCy model, "
"e.g. by running:\npython -m spacy download "
"en_core_web_md\npython -m spacy link "
"en_core_web_md en".format(spacy_model_name))
@classmethod
def required_packages(cls) -> List[Text]:
return ["spacy"]
@classmethod
def create(cls, component_config: Dict[Text, Any],
config: RasaNLUModelConfig) -> "SpacyNLP":
component_config = override_defaults(cls.defaults, component_config)
spacy_model_name = component_config.get("model")
# if no model is specified, we fall back to the language string
if not spacy_model_name:
spacy_model_name = config.language
component_config["model"] = config.language
logger.info(
f"Trying to load spacy model with name '{spacy_model_name}'")
nlp = cls.load_model(spacy_model_name)
cls.ensure_proper_language_model(nlp)
return cls(component_config, nlp)
@classmethod
def cache_key(cls, component_meta: Dict[Text, Any],
model_metadata: "Metadata") -> Optional[Text]:
# Fallback, use the language name, e.g. "en",
# as the model name if no explicit name is defined
spacy_model_name = component_meta.get("model", model_metadata.language)
return cls.name + "-" + spacy_model_name
def provide_context(self) -> Dict[Text, Any]:
return {"spacy_nlp": self.nlp}
def doc_for_text(self, text: Text) -> "Doc":
return self.nlp(self.preprocess_text(text))
def preprocess_text(self, text: Optional[Text]) -> Text:
if text is None:
# converted to empty string so that it can still be passed to spacy.
# Another option could be to neglect tokenization of the attribute of this example, but since we are
# processing in batch mode, it would get complex to collect all processed and neglected examples.
text = ""
if self.component_config.get("case_sensitive"):
return text
else:
return text.lower()
def get_text(self, example: Dict[Text, Any], attribute: Text) -> Text:
return self.preprocess_text(example.get(attribute))
@staticmethod
def merge_content_lists(
indexed_training_samples: List[Tuple[int, Text]],
doc_lists: List[Tuple[int, "Doc"]],
) -> List[Tuple[int, "Doc"]]:
"""Merge lists with processed Docs back into their original order."""
dct = dict(indexed_training_samples)
dct.update(dict(doc_lists))
return sorted(dct.items())
@staticmethod
def filter_training_samples_by_content(
indexed_training_samples: List[Tuple[int, Text]]
) -> Tuple[List[Tuple[int, Text]], List[Tuple[int, Text]]]:
"""Separates empty training samples from content bearing ones."""
docs_to_pipe = list(
filter(
lambda training_sample: training_sample[1] != "",
indexed_training_samples,
))
empty_docs = list(
filter(
lambda training_sample: training_sample[1] == "",
indexed_training_samples,
))
return docs_to_pipe, empty_docs
def process_content_bearing_samples(
self,
samples_to_pipe: List[Tuple[int,
Text]]) -> List[Tuple[int, "Doc"]]:
"""Sends content bearing training samples to spaCy's pipe."""
docs = [(to_pipe_sample[0], doc) for to_pipe_sample, doc in zip(
samples_to_pipe,
[
doc
for doc in self.nlp.pipe([txt for _, txt in samples_to_pipe],
batch_size=50)
],
)]
return docs
def process_non_content_bearing_samples(
self, empty_samples: List[Tuple[int,
Text]]) -> List[Tuple[int, "Doc"]]:
"""Creates empty Doc-objects from zero-lengthed training samples strings."""
from spacy.tokens import Doc
n_docs = [(empty_sample[0], doc) for empty_sample, doc in zip(
empty_samples, [Doc(self.nlp.vocab) for doc in empty_samples])]
return n_docs
def docs_for_training_data(
self, training_data: TrainingData) -> Dict[Text, List[Any]]:
attribute_docs = {}
for attribute in DENSE_FEATURIZABLE_ATTRIBUTES:
texts = [
self.get_text(e, attribute)
for e in training_data.intent_examples
]
# Index and freeze indices of the training samples for preserving the order
# after processing the data.
indexed_training_samples = [(idx, text)
for idx, text in enumerate(texts)]
samples_to_pipe, empty_samples = self.filter_training_samples_by_content(
indexed_training_samples)
content_bearing_docs = self.process_content_bearing_samples(
samples_to_pipe)
non_content_bearing_docs = self.process_non_content_bearing_samples(
empty_samples)
attribute_document_list = self.merge_content_lists(
indexed_training_samples,
content_bearing_docs + non_content_bearing_docs,
)
# Since we only need the training samples strings, we create a list to get them out
# of the tuple.
attribute_docs[attribute] = [
doc for _, doc in attribute_document_list
]
return attribute_docs
def train(self, training_data: TrainingData, config: RasaNLUModelConfig,
**kwargs: Any) -> None:
attribute_docs = self.docs_for_training_data(training_data)
for attribute in DENSE_FEATURIZABLE_ATTRIBUTES:
for idx, example in enumerate(training_data.training_examples):
example_attribute_doc = attribute_docs[attribute][idx]
if len(example_attribute_doc):
# If length is 0, that means the initial text feature was None and was replaced by ''
# in preprocess method
example.set(SPACY_DOCS[attribute], example_attribute_doc)
def process(self, message: Message, **kwargs: Any) -> None:
message.set(SPACY_DOCS[TEXT], self.doc_for_text(message.text))
@classmethod
def load(
cls,
meta: Dict[Text, Any],
model_dir: Text = None,
model_metadata: "Metadata" = None,
cached_component: Optional["SpacyNLP"] = None,
**kwargs: Any,
) -> "SpacyNLP":
if cached_component:
return cached_component
model_name = meta.get("model")
nlp = cls.load_model(model_name)
cls.ensure_proper_language_model(nlp)
return cls(meta, nlp)
@staticmethod
def ensure_proper_language_model(nlp: Optional["Language"]) -> None:
"""Checks if the spacy language model is properly loaded.
Raises an exception if the model is invalid."""
if nlp is None:
raise Exception("Failed to load spacy language model. "
"Loading the model returned 'None'.")
if nlp.path is None:
# Spacy sets the path to `None` if
# it did not load the model from disk.
# In this case `nlp` is an unusable stub.
raise Exception("Failed to load spacy language model for "
"lang '{}'. Make sure you have downloaded the "
"correct model (https://spacy.io/docs/usage/)."
"".format(nlp.lang))
def __del__(self):
del self.nlp
def __init__(self):
self.cache = LRUCache(maxsize=100)
from django.views.generic.base import TemplateView
from cachetools.lru import LRUCache
from .. import serializer
from ..decorators import timed
from ..errors import ComponentLoadError
from ..settings import get_setting
from .fields import UnicornField
from .unicorn_template_response import UnicornTemplateResponse
logger = logging.getLogger(__name__)
# TODO: Make maxsize configurable
# Module cache to store the found component classes
views_cache = LRUCache(maxsize=100)
# Module cache for constructed component classes
# This can create a subtle race condition so a more long-term solution needs to be found
constructed_views_cache = LRUCache(maxsize=100)
def convert_to_snake_case(s: str) -> str:
# TODO: Better handling of dash->snake
return s.replace("-", "_")
def convert_to_pascal_case(s: str) -> str:
# TODO: Better handling of dash/snake->pascal-case
s = convert_to_snake_case(s)
return "".join(word.title() for word in s.split("_"))
from uuid import UUID
from cachetools import cached
from cachetools.lru import LRUCache
from typing_inspect import is_union_type # type: ignore
from dataclasses_json import cfg
from dataclasses_json.utils import (_get_type_cons,
_handle_undefined_parameters_safe,
_is_collection, _is_mapping, _is_new_type,
_is_optional, _isinstance_safe,
_issubclass_safe)
Json = Union[dict, list, str, int, float, bool, None]
_MAX_CACHE_SIZE = 128
_get_type_hints_cache = LRUCache(maxsize=_MAX_CACHE_SIZE)
_is_supported_generic_cache = LRUCache(maxsize=_MAX_CACHE_SIZE)
_user_overrides_cache = LRUCache(maxsize=_MAX_CACHE_SIZE)
class _ExtendedEncoder(json.JSONEncoder):
def default(self, o) -> Json:
result: Json
if _isinstance_safe(o, Collection):
if _isinstance_safe(o, Mapping):
result = dict(o)
else:
result = list(o)
elif _isinstance_safe(o, datetime):
result = o.timestamp()
elif _isinstance_safe(o, UUID):
class LayeredCache(object):
"""
Multi-Layered key value store.
Layer 1: In Memory LRU Key Value Map
Layer 2: Redis Key Value Store
This cache type is great for things that can exist in memory, either
locally in the LRU layer or in the redis layer.
"""
def __init__(self, node_name: str, lru_size: int):
"""
Initialize the first two layers of a multi-layered cache
:param node_name:
:param lru_size:
"""
super(LayeredCache, self).__init__()
# Track the name of the node Type. This should be whatever
# unique identifier can be used in dgraph queries.
self.node_name = node_name
# Initialize a Least Recently Used in-memory cache
# with a maximum number of elements.
# layer 1
self.lru_local_cache = LRUCache(maxsize=lru_size)
# Initialize a redis client.
# layer 2 cache
self.redis = Redis("localhost")
# This should be set to True if we should add a timeout to the
# redis key value store values.
self.set_timeout = False
def _get_key(self, key: str) -> str:
"""
Get the unique key that is used at each cached layer.
:param key:
:return:
"""
return f"{self.node_name}-{key}"
def __setitem__(self, key: str, value: str):
"""
Store a key value pair in each cache layer.
:param key:
:param value:
:return:
"""
# Store in layer 1 local LRU cache
self.lru_local_cache[self._get_key(key)] = value
# Store in layer 2 redis cache
# If we want to have key value pairs timeout in redis, use setex with 5
# minutes before until timeout
if self.set_timeout:
timeout = 300
self.redis.setex(self._get_key(key), timeout, value)
else:
self.redis.set(self._get_key(key), value)
def __contains__(self, key: str) -> bool:
"""
Check to see if key is in a layer of the cache. We will start at
layer 1 and go walk through each layer until we find a result.
We will update previous layers if we cache miss.
We'll return True if the key was found at a layer, False if we
cache miss.
:param key:
:return:
"""
# Check the layer 1 local LRU cache
local_result = self.lru_local_cache.get(self._get_key(key), None)
if local_result is not None:
return True
# Check the layer 2 redis cache
redis_result = self.redis.get(self._get_key(key))
if redis_result is not None:
# Update the each layer with the value
self[key] = redis_result.decode()
return True
# Cache miss, return False
return False
def __getitem__(self, key: str) -> Union[str, None]:
"""
Check each layer iteratively for the key specified. If we find the result
at a given layer, we update previous layers with the result.
If the result was not found, return None.
:param key:
:return:
"""
# Check the layer 1 local LRU cache
local_result = self.lru_local_cache.get(self._get_key(key), None)
if local_result is not None:
return local_result
# Check the layer 2 redis cache
redis_result = self.redis.get(self._get_key(key))
if redis_result is not None:
# Update the each layer with the value
self[key] = redis_result.decode()
return redis_result.decode()
# Cache miss, return None
return None
def close(self):
"""
Close any outstanding connections.
:return:
"""
self.redis.close()