class VLANProfile(Document):
meta = {
"collection": "vlanprofiles",
"strict": False,
"auto_create_index": False
}
name = StringField(unique=True)
description = StringField()
# VLAN is management VLAN
enable_management = BooleanField(default=False)
# VLAN is multicast VLAN
enable_multicast = BooleanField(default=False)
# VLAN should be automatically provisioned
enable_provisioning = BooleanField(default=False)
# VLAN workflow
workflow = PlainReferenceField(Workflow)
style = ForeignKeyField(Style)
tags = ListField(StringField())
# Integration with external NRI and TT systems
# Reference to remote system object has been imported from
remote_system = PlainReferenceField(RemoteSystem)
# Object id in remote system
remote_id = StringField()
# Object id in BI
bi_id = LongField(unique=True)
_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
_bi_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
def __str__(self):
return self.name
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"),
lock=lambda _: id_lock)
def get_by_id(cls, id):
return VLANProfile.objects.filter(id=id).first()
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"),
lock=lambda _: id_lock)
def get_by_bi_id(cls, id):
return VLANProfile.objects.filter(bi_id=id).first()
def __init__(self,
api,
name,
mode='default',
sortby='filename',
credentials=None):
# set params
super().__init__(name, sortby)
self.api = api
self.api_islocal = api.__class__.__name__ == 'APILocal'
if not mode in _cfg_mode_valid:
raise ValueError(
'Invalid mode, needs to be {}'.format(_cfg_mode_valid))
self.mode = mode
# get remote details
self.cnxnapi = api.cnxn
self.cnxnpipe = self.cnxnapi.pipes._(name)
self.settings = self.cnxnpipe.get()[1]
if not self.settings:
raise ValueError('pipe not found, make sure it was created')
if self.settings['protocol'] not in ['s3', 'ftp', 'sftp']:
raise NotImplementedError(
'Unsupported protocol, only s3 and (s)ftp supported')
self.settings['options'] = self.settings.get('options', {})
self.remote_prefix = self.settings['options']['remotepath']
self.encrypted_pipe = self.settings['options'].get('encrypted', False)
if self.encrypted_pipe:
self.settings = self.api.decode(self.settings)
self.role = self.settings.get('role')
self.cfg_profile = api.cfg_profile
self._set_dir(self.name)
self.credentials_override = credentials
# DDL
self.schema = self.settings.get('schema', {})
# create db connection
self._db = TinyDB(self.cfg_profile['filedb'],
storage=_tdbserialization)
self.dbfiles = self._db.table(name + '-files')
self.dbconfig = self._db.table(name + '-cfg')
self._cache_scan = cachetools.TTLCache(maxsize=1, ttl=5 * 60)
# connect msg
msg = 'Successfully connected to pipe {}. '.format(self.name)
if self.role == 'read':
msg += ' Read only access'
print(msg)
self.dbconfig.upsert({
'name': self.name,
'pipe': self.settings
},
Query().name == self.name)
class InterfaceValidationPolicy(Document):
meta = {
"collection": "interfacevalidationpolicies",
"strict": False,
"auto_create_index": False,
}
name = StringField(unique=True)
description = StringField()
filter_query = PlainReferenceField(ConfDBQuery)
rules = ListField(EmbeddedDocumentField(InterfaceValidationRule))
_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
def __str__(self):
return self.name
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock)
def get_by_id(cls, id):
return InterfaceValidationPolicy.objects.filter(id=id).first()
def iter_problems(self, engine, ifname):
"""
Check rules against ConfDB engine
:param engine: ConfDB Engine instance
:param ifname: Interface name
:return: List of problems
"""
# Check filter query, if any
if self.filter_query:
if not self.filter_query.any(engine, ifname=ifname):
return
# Process rules
for rule in self.rules:
if not rule.is_active:
continue
if rule.filter_query:
if not rule.filter_query.any(engine, ifname=ifname):
continue
for ctx in rule.query.query(engine, ifname=ifname, **rule.query_params):
if "error" in ctx:
tpl = Template(rule.error_text_template)
path = [ifname]
if rule.error_code:
path += [rule.error_code]
problem = {
"alarm_class": rule.alarm_class.name if rule.alarm_class else None,
"path": path,
"message": tpl.render(ctx),
"code": rule.error_code or None,
}
yield problem
if rule.is_fatal:
return
def __init__(self, storage, key, statement_id, group_id):
super().__init__()
self.cookie_cache = cachetools.TTLCache(
1,
config.informatics_cookie_ttl,
)
self._storage = storage
self._key = key
self._statement_id = statement_id
self._group_id = group_id
class AllocationGroup(Document):
meta = {
"collection": "allocationgroups",
"strict": False,
"auto_create_index": False
}
name = StringField(unique=True)
description = StringField()
# Labels
labels = ListField(StringField())
effective_labels = ListField(StringField())
# Integration with external NRI and TT systems
# Reference to remote system object has been imported from
remote_system = PlainReferenceField(RemoteSystem)
# Object id in remote system
remote_id = StringField()
# Object id in BI
bi_id = LongField(unique=True)
_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
_bi_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
def __str__(self):
return self.name
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"),
lock=lambda _: id_lock)
def get_by_id(cls, id):
return AllocationGroup.objects.filter(id=id).first()
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"),
lock=lambda _: id_lock)
def get_by_bi_id(cls, id):
return AllocationGroup.objects.filter(bi_id=id).first()
@classmethod
def can_set_label(cls, label):
if label.enable_allocationgroup:
return True
return False
class TermTypesResource(object):
"""Get Namespaces, Entity Types, and Context Types in TermStore
Get facet counts for each (top 100 for each)
"""
@cachetools.cached(cachetools.TTLCache(maxsize=512, ttl=600))
def on_get(self, req, resp):
""" Get stats """
resp.media = terms.term_types()
class Car(models.Model):
class Meta:
unique_together = (("model", "fuel", "capacity", "start_year",
"end_year", "etype", "now"), )
ordering = ["model__name"]
indexes = [
models.Index(fields=["id", "model"]),
]
model = models.ForeignKey(BrandModel, on_delete=models.CASCADE)
fuel = models.CharField(
max_length=16,
choices=[("Petrol", "Petrol"), ("Diesel", "Diesel")],
default="Diesel",
)
etype = models.CharField(max_length=15, blank=True)
capacity = models.FloatField(default=0.0)
start_year = models.IntegerField(
default=2000,
validators=[
MinValueValidator(2000),
MaxValueValidator(datetime.datetime.today().year),
],
)
end_year = models.IntegerField(
default=0,
validators=[
MinValueValidator(2000),
MaxValueValidator(datetime.datetime.today().year),
],
blank=True,
)
now = models.BooleanField(default=True)
active = models.BooleanField(default=True)
_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
def __str__(self):
if self.end_year == 0 and self.now:
end_year = "Now"
else:
end_year = self.end_year
return "%s %s %s %s-%s" % (
self.model.name,
self.capacity,
self.fuel,
self.start_year,
end_year,
)
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"),
lock=lambda _: id_lock)
def get_by_id(cls, id):
return Car.objects.filter(id=id).first()
def to_vtt(self, output_path):
last_modified_at = self.started_at
timer = Timer(self.started_at.timestamp())
ttl_cache = cachetools.TTLCache(maxsize=CONTENT_MAXSIZE,
ttl=CONTENT_TTL,
timer=timer.timer)
last_context = {'ts': None, 'contents': []}
with open(output_path, 'w', encoding='utf8') as fp:
fp.write(VTT_HEADERS)
for each in self.iterator:
content = each['data']['content']
if is_spam(content):
continue
current_time = arrow.get(each['_id'].generation_time)
ts_seconds = current_time.timestamp(
) - self.started_at.timestamp()
ts = arrow.get(ts_seconds)
timer.ts = current_time.timestamp()
cache_key = f'{ts.format(VTT_TIME_FORMAT)}\n{content}'
ttl_cache[cache_key] = content
content_list = list(ttl_cache.values()) # 获取当前字幕需要展示的所有弹幕
if not last_context['contents']:
# 第一次循环
last_context['ts'] = ts
last_context['contents'] = content_list
continue
caption_interval = current_time - last_modified_at
if caption_interval.seconds < CAPTION_MINIMAL_INTERVAL:
# 控制两条字幕之间间隔时间
last_context['contents'] = content_list
continue
last_start = last_context['ts']
last_end = ts if ts.shift(
seconds=-CONTENT_TTL) < last_start else last_start.shift(
seconds=CONTENT_TTL)
line = f'{last_start.format(VTT_TIME_FORMAT)} --> {last_end.format(VTT_TIME_FORMAT)}\n'
for each_content in last_context['contents']:
line += f'{each_content}\n'
line += '\n'
fp.write(line)
last_context['ts'] = ts
last_context['contents'] = content_list
last_modified_at = current_time
class AlarmGnocchiThresholdRule(base.AlarmRule):
comparison_operator = base.AdvEnum('comparison_operator',
str,
'lt',
'le',
'eq',
'ne',
'ge',
'gt',
default='eq')
"The comparison against the alarm threshold"
threshold = wsme.wsattr(float, mandatory=True)
"The threshold of the alarm"
aggregation_method = wsme.wsattr(wtypes.text, mandatory=True)
"The aggregation_method to compare to the threshold"
evaluation_periods = wsme.wsattr(wtypes.IntegerType(minimum=1), default=1)
"The number of historical periods to evaluate the threshold"
granularity = wsme.wsattr(wtypes.IntegerType(minimum=1), default=60)
"The time range in seconds over which query"
cache = cachetools.TTLCache(maxsize=1, ttl=3600)
lock = threading.RLock()
@classmethod
def validate_alarm(cls, alarm):
alarm_rule = getattr(alarm, "%s_rule" % alarm.type)
aggregation_method = alarm_rule.aggregation_method
if aggregation_method not in cls._get_aggregation_methods():
raise base.ClientSideError(
'aggregation_method should be in %s not %s' %
(cls._get_aggregation_methods(), aggregation_method))
@staticmethod
@cachetools.cached(cache, lock=lock)
def _get_aggregation_methods():
conf = pecan.request.cfg
gnocchi_client = client.Client('1',
keystone_client.get_session(conf),
adapter_options={
'interface':
conf.service_credentials.interface,
'region_name':
conf.service_credentials.region_name
})
try:
return gnocchi_client.capabilities.list().get(
'aggregation_methods', [])
except exceptions.ClientException as e:
raise base.ClientSideError(e.message, status_code=e.code)
except Exception as e:
raise GnocchiUnavailable(e)
class Template(models.Model):
class Meta:
app_label = "main"
db_table = "main_template"
verbose_name = "Template"
verbose_name_plural = "Templates"
ordering = ["name"]
name = models.CharField("Name", unique=True, max_length=128)
subject = models.TextField("Subject", validators=[template_validator])
body = models.TextField("Body", validators=[template_validator])
_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
_name_cache = cachetools.TTLCache(maxsize=100, ttl=60)
def __unicode__(self):
return self.name
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"),
lock=lambda _: id_lock)
def get_by_id(cls, id):
t = Template.objects.filter(id=id)[:1]
if t:
return t[0]
return None
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_name_cache"),
lock=lambda _: id_lock)
def get_by_name(cls, name):
t = Template.objects.filter(name=name)[:1]
if t:
return t[0]
return None
def render_subject(self, LANG=None, **kwargs):
return jinja2.Template(self.subject).render(**kwargs)
def render_body(self, LANG=None, **kwargs):
return jinja2.Template(self.body).render(**kwargs)
def __init__(self, parentLogger=None):
"""Constructor"""
self.__permValues = ["USER", "GROUP", "VO", "ALL"]
self.__permAttrs = ["ReadAccess", "PublishAccess"]
self.__cache = cachetools.TTLCache(1024, 15)
DB.__init__(self,
"UserProfileDB",
"Framework/UserProfileDB",
parentLogger=parentLogger)
retVal = self.__initializeDB()
if not retVal["OK"]:
raise Exception("Can't create tables: %s" % retVal["Message"])
def __init__(self,
db_endpoint: str,
fs_root: str,
max_cache_mem: int,
ttl: int,
engine_kwargs: dict = None):
"""
Initialize a new instance of SQLAlchemyModelRepository.
:param db_endpoint: SQLAlchemy connection string.
:param fs_root: Root directory where to store the models.
:param max_cache_mem: Maximum memory size to use for model cache (in bytes).
:param ttl: Time-to-live for each model in the cache (in seconds).
:param engine_kwargs: Passed directly to SQLAlchemy's `create_engine()`.
"""
self.fs_root = fs_root
# A version of db_endpoint that never contain password is needed for logging
db_endpoint_components = urlparse(db_endpoint)
if db_endpoint_components.password is not None:
password, netloc = db_endpoint_components.password, db_endpoint_components.netloc
password_index = netloc.rindex(password)
safe_netloc = "%s%s%s" % (
db_endpoint_components.netloc[:password_index], "<PASSWORD>",
db_endpoint_components.netloc[password_index + len(password):])
safe_db_endpoint_components = list(db_endpoint_components)
safe_db_endpoint_components[1] = safe_netloc
self._safe_db_endpoint = urlunparse(safe_db_endpoint_components)
else:
self._safe_db_endpoint = db_endpoint
must_initialize = not database_exists(db_endpoint)
if must_initialize:
self._log.debug("%s does not exist, creating",
self._safe_db_endpoint)
create_database(db_endpoint)
self._log.warning("created a new database at %s",
self._safe_db_endpoint)
self._engine = create_engine(
db_endpoint,
**(engine_kwargs if engine_kwargs is not None else {}))
must_initialize |= not self._engine.has_table(Model.__tablename__)
if must_initialize:
Model.metadata.create_all(self._engine)
self._sessionmaker = ContextSessionMaker(
sessionmaker(bind=self._engine))
bakery = baked.bakery()
self._get_query = bakery(lambda session: session.query(Model))
self._get_query += lambda query: query.filter(
and_(Model.analyzer == bindparam("analyzer"), Model.repository ==
bindparam("repository")))
self._cache = cachetools.TTLCache(maxsize=max_cache_mem,
ttl=ttl,
getsizeof=asizeof)
self._cache_lock = threading.Lock()
def open(self):
"""Allocate ressources used by the cache."""
super(MemoryCache, self).open()
def _timer():
# Use a custom timer to try to spread expirations. Within one instance it
# won't change anything but it will be better if you run multiple instances.
return time.time() + self.__ttl * random.uniform(-0.25, 0.25)
self.__cache = cachetools.TTLCache(maxsize=self.__size,
ttl=self.__ttl,
timer=_timer)
class IfDescPatterns(Document):
meta = {
"collection": "ifdescpatterns",
"strict": False,
"auto_create_index": False,
}
name = StringField(unique=True)
description = StringField()
resolve_remote_port_by_object = BooleanField(default=False)
patterns = ListField(EmbeddedDocumentField(IfDescPatternRule))
_id_cache = cachetools.TTLCache(100, ttl=60)
_re_cache = {}
def __str__(self):
return self.name
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock)
def get_by_id(cls, id: Union[bson.ObjectId, str]) -> Optional["IfDescPatterns"]:
return IfDescPatterns.objects.filter(id=id).first()
def clean(self):
super().clean()
for p in self.patterns:
rx = self._get_re(p.pattern)
if not rx:
raise ValidationError("Invalid regular expression: %s" % p.pattern)
def iter_match(self, s: str) -> Iterable[Dict[str, str]]:
"""
Match patterns against string yield all matching groups
:param s: Input string
:return:
"""
for rule in self.patterns:
if not rule.is_active:
continue
rx = self._get_re(rule.pattern)
if not rx:
continue
match = rx.search(s)
if match:
yield match.groupdict()
@cachetools.cachedmethod(operator.attrgetter("_re_cache"), lock=lambda _: re_lock)
def _get_re(self, pattern: str) -> Optional[re.Pattern]:
try:
return re.compile(pattern)
except re.error:
return None
class ServiceProfile(Document):
meta = {
"collection": "noc.serviceprofiles",
"strict": False,
"auto_create_index": False
}
name = StringField(unique=True)
description = StringField()
# Jinja2 service label template
card_title_template = StringField()
# Short service code for reporting
code = StringField()
# FontAwesome glyph
glyph = StringField()
# Glyph order in summary
display_order = IntField(default=100)
# Show in total summary
show_in_summary = BooleanField(default=True)
# Auto-assign interface profile when service binds to interface
interface_profile = ReferenceField(InterfaceProfile)
# Alarm weight
weight = IntField(default=0)
# Integration with external NRI and TT systems
# Reference to remote system object has been imported from
remote_system = ReferenceField(RemoteSystem)
# Object id in remote system
remote_id = StringField()
# Object id in BI
bi_id = LongField(unique=True)
# Tags
tags = ListField(StringField())
_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
def __str__(self):
return self.name
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"),
lock=lambda _: id_lock)
def get_by_id(cls, id):
return ServiceProfile.objects.filter(id=id).first()
def on_save(self):
if not hasattr(self, "_changed_fields"
) or "interface_profile" in self._changed_fields:
call_later(
"noc.sa.models.serviceprofile.refresh_interface_profiles",
sp_id=self.id,
ip_id=self.interface_profile.id
if self.interface_profile else None,
)
def __init__(self,
base_tnc,
base_name,
base_port=None,
echo_packets=True,
buffer_size=10000,
buffer_time=30):
self.packet_cache = cachetools.TTLCache(buffer_size, buffer_time)
self.lock = threading.Lock()
self.base_tnc = base_tnc
self.base_port = base_port
self.base_name = base_name
self.echo_packets = echo_packets
def activate(self):
super(GerritBotPlugin, self).activate()
self.work_queue = compat_queue.Queue()
self.dying = False
self.seen_reviews = cachetools.TTLCache(
self.config['max_cache_size'], self.config['max_cache_seen_ttl'])
self.statistics = copy.deepcopy(self.DEF_STATS)
self.client = threading.Thread(target=self.loop_client_recv)
self.client.daemon = True
self.client.start()
self.processor = threading.Thread(target=self.loop_process_events)
self.processor.daemon = True
self.processor.start()
def __init__(self, throttle: Optional[DurationLiteral] = None, **kwargs: Any):
"""
Initializer.
Args:
throttle: If set to a valid duration literal (e.g. 5m) the return value of the
called functions will be cached for the given amount of time.
"""
super().__init__(**kwargs)
self.throttle = throttle and parse_duration_literal(throttle)
self._cache = None
if self.throttle:
self._cache = cachetools.TTLCache(self.MAX_CACHE_SIZE, ttl=self.throttle)
class TimePattern(models.Model):
"""
Time Patterns
"""
class Meta(object):
verbose_name = "Time Pattern"
verbose_name_plural = "Time Patterns"
db_table = "main_timepattern"
app_label = "main"
name = models.CharField("Name", max_length=64, unique=True)
description = models.TextField("Description", null=True, blank=True)
_code_cache = cachetools.TTLCache(1000, ttl=60)
def __unicode__(self):
return self.name
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_code_cache"),
lock=lambda _: id_lock)
def get_by_id(cls, id):
p = TimePattern.objects.filter(id=id)[:1]
if p:
return p[0]
else:
return None
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_code_cache"),
lock=lambda _: id_lock)
def get_code(cls, id):
p = TimePattern.get_by_id(id)
if p:
return TP.compile_to_python(
[t.term for t in p.timepatternterm_set.all()])
else:
return None
@property
def time_pattern(self):
"""
Returns associated Time Pattern object
"""
return TP([t.term for t in self.timepatternterm_set.all()])
def match(self, d):
"""
Matches DateTime objects against time pattern
"""
return self.time_pattern.match(d)
class ThresholdProfile(Document):
meta = {"collection": "thresholdprofiles", "strict": False, "auto_create_index": False}
name = StringField(unique=True)
description = StringField()
# Handler to filter and modify umbrella alarms
umbrella_filter_handler = StringField()
# Window function settings
# Window depth
window_type = StringField(max_length=1, choices=[("m", "Measurements"), ("t", "Time")])
# Window size. Depends on window type
# * m - amount of measurements
# * t - time in seconds
window = IntField(default=1)
# Window function
# Accepts window as a list of [(timestamp, value)]
# and window_config
# and returns float value
window_function = StringField(choices=wf_choices, default="last")
# Window function configuration
window_config = StringField()
# thresholds config
thresholds = ListField(EmbeddedDocumentField(ThresholdConfig))
_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
def __str__(self):
return self.name
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock)
def get_by_id(cls, id):
return ThresholdProfile.objects.filter(id=id).first()
def get_window_function(self):
"""
Returns window funciton or None if invalid name given
:returns: Callable or None
"""
return get_window_function(self.window_function)
def find_threshold(self, name):
"""
Find Threshold Config by name
:param name: Threshold name
:return: ThresholdConfig or None
"""
for cfg in self.thresholds:
if cfg.name == name:
return cfg
return None
class DataStreamConfig(Document):
meta = {
"collection": "datastreamconfigs",
"strict": False,
"auto_create_index": False
}
name = StringField(unique=True)
formats = ListField(EmbeddedDocumentField(DSFormat))
_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
_name_cache = cachetools.TTLCache(maxsize=100, ttl=60)
def __str__(self) -> str:
return self.name
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"),
lock=lambda _: id_lock)
def get_by_id(cls, id: Union[str,
ObjectId]) -> Optional["DataStreamConfig"]:
return DataStreamConfig.objects.filter(id=id).first()
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_name_cache"),
lock=lambda _: id_lock)
def get_by_name(cls, name: str) -> Optional["DataStreamConfig"]:
return DataStreamConfig.objects.filter(name=name).first()
def iter_formats(
self,
) -> Iterable[Tuple[str, Callable[[Dict[str, Any]], Iterable[Dict[str,
Any]]]]]:
for fmt in self.formats:
if fmt.is_active:
handler = fmt.handler.get_handler()
if handler:
yield fmt.name, handler
class AuthAPIRequestHandler(APIRequestHandler):
_user_cache = cachetools.TTLCache(maxsize=1000, ttl=60)
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_user_cache"),
lock=lambda _: user_lock)
def get_user_by_name(cls, name):
try:
return User.objects.get(username=name)
except User.DoesNotExist:
return None
def get_current_user(self):
return self.get_user_by_name(self.request.headers.get("Remote-User"))
def send_funds(destination, asset, quantity):
from picopayments import api
extra_btc = _get_fee_multaple(factor=3)
key = find_key_with_funds(asset, quantity, extra_btc)
if key is None:
raise err.InsufficientFunds(asset, quantity)
unsigned_rawtx = api.locked_create_send(source=key["address"],
destination=destination,
asset=asset,
regular_dust_size=extra_btc,
quantity=quantity)
_LOCKS[key["address"]] = cachetools.TTLCache(_LOCKS_MAX, _LOCKS_TTL)
signed_rawtx = scripts.sign_deposit(get_tx, key["wif"], unsigned_rawtx)
return publish(signed_rawtx)
class NumberCategory(Document):
meta = {
"collection": "noc.numbercategories",
"strict": False,
"auto_create_index": False
}
name = StringField(unique=True)
is_active = BooleanField()
description = StringField()
order = IntField(default=1000)
rules = ListField(EmbeddedDocumentField(NumberCategoryRule))
_id_cache = cachetools.TTLCache(100, ttl=60)
_rule_cache = cachetools.TTLCache(100, ttl=60)
def __str__(self):
return self.name
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"),
lock=lambda _: id_lock)
def get_by_id(cls, id):
return NumberCategory.objects.filter(id=id).first()
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_rule_cache"),
lock=lambda _: id_lock)
def get_rules(cls):
r = []
for nc in NumberCategory.objects.filter(
is_active=True).order_by("order"):
for rule in nc.rules:
if not rule.is_active:
continue
r += [(rule.dialplan, re.compile(rule.mask), nc)]
return r
class Project(NOCModel):
"""
Projects are used to track investment projects expenses and profits
"""
class Meta(object):
verbose_name = "Project"
verbose_name_plural = "Projects"
app_label = "project"
db_table = "project_project"
code = models.CharField("Code", max_length=256, unique=True)
name = models.CharField("Name", max_length=256)
description = models.TextField("Description", null=True, blank=True)
shape_overlay_glyph = DocumentReferenceField(Glyph, null=True, blank=True)
shape_overlay_position = models.CharField(
"S.O. Position",
max_length=2,
choices=[(x.value, x.value) for x in ShapeOverlayPosition],
null=True,
blank=True,
)
shape_overlay_form = models.CharField(
"S.O. Form",
max_length=1,
choices=[(x.value, x.value) for x in ShapeOverlayForm],
null=True,
blank=True,
)
# Integration with external NRI systems
# Reference to remote system object has been imported from
remote_system = DocumentReferenceField(RemoteSystem, null=True, blank=True)
# Object id in remote system
remote_id = models.CharField(max_length=64, null=True, blank=True)
# Object id in BI
bi_id = models.BigIntegerField(unique=True)
_id_cache = cachetools.TTLCache(100, ttl=60)
def __str__(self):
return self.code
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"),
lock=lambda x: id_lock)
def get_by_id(cls, id):
p = Project.objects.filter(id=id)[:1]
if p:
return p[0]
return None
class CoursePageDownloader:
_url = attr.ib(default=settings.COURSE_URL)
_cache = attr.ib(
default=cachetools.TTLCache(maxsize=1, ttl=settings.COURSE_CACHE_TTL))
@cachetools.cachedmethod(operator.attrgetter("_cache"))
def download_course_page(self):
try:
response = requests.get(self._url)
response.raise_for_status()
return response.text
except requests.RequestException as error:
raise DownloadError() from error
except urllib3.exceptions.ProtocolError as error:
raise DownloadError() from error
def async_ttl_cache(ttl: int = 3600, maxsize: int = 1):
cache = cachetools.TTLCache(ttl=ttl, maxsize=maxsize)
def decorator(fn):
@functools.wraps(fn)
async def memoize(*args, **kwargs):
key = str((args, kwargs))
try:
return cache[key]
except KeyError:
cache[key] = await fn(*args, **kwargs)
return cache[key]
return memoize
return decorator
def __init__(self, model: t.Type[Model], schema: Schema, index_name: t.Optional[str] = None):
"""
:param model: django.db.models.Model subclass whose instances are to be searched
:param schema: field schema for the search index
:param index_name: name of the search index
:param pk_name: field name of the model's primary key
"""
self.model = model
self.schema = schema
self.pk_name = model._meta.pk.name
self.schema.add(self.pk_name, PK_FIELDTYPE)
self.index = self._init_index(index_name)
query_fields = set(schema.names()) - {self.pk_name}
self.query_parser = LectorQueryParser(query_fields, self.schema)
self._search_cache: t.MutableMapping[str, Results] = cachetools.TTLCache(64, 60.0)
class Font(Document):
meta = {
"collection": "fonts",
"strict": False,
"auto_create_index": False,
"json_collection": "main.fonts",
}
name = StringField(unique=True)
uuid = UUIDField(unique=True, binary=True)
font_family = StringField()
description = StringField(required=False)
stylesheet_href = StringField(required=False)
_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
def __str__(self):
return self.name
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"),
lock=lambda _: id_lock)
def get_by_id(cls, id: Union[str, bson.ObjectId]) -> "Font":
return Font.objects.filter(id=id).first()
@property
def json_data(self):
r = {
"name": self.name,
"$collection": self._meta["json_collection"],
"uuid": str(self.uuid),
"font_family": self.font_family,
}
if self.description:
r["description"] = self.description
if self.stylesheet_href:
r["stylesheet_href"] = self.stylesheet_href
return r
def to_json(self):
return to_json(self.json_data,
order=[
"name", "$collection", "uuid", "font_family",
"description"
])
def get_json_path(self):
return "%s.json" % quote_safe_path(self.name)
def test_lru_caching_func(self):
from pyrolysis.common.cache import caching
import cachetools
c = cachetools.TTLCache(5, 1)
mod = 0
@caching(c)
def f(x):
return x * x + mod
self.assertEqual(f(2), 4)
# cache effect
mod = 1
self.assertEqual(f(2), 4)
# cache expired
time.sleep(2)
self.assertEqual(f(2), 5)
