class RoboCache(object):
def __init__(self, max_age=None, max_count=None):
self.data = OrderedDict()
self.max_age = max_age
self.max_count = max_count
def _reduce_age(self, now):
"""Reduce size of cache by date.
:param datetime.datetime now: Current time
"""
if self.max_age:
for key, value in iteritems(self.data):
if now - value['date'] > self.max_age:
del self.data[key]
def _reduce_count(self):
"""Reduce size of cache by count.
"""
if self.max_count:
while len(self.data) > self.max_count:
self.data.popitem(last=False)
def store(self, response):
"""Store response in cache, skipping if code is forbidden.
:param requests.Response response: HTTP response
"""
if response.status_code not in CACHE_CODES:
return
now = datetime.datetime.now()
self.data[response.url] = {
'date': now,
'response': response,
}
logger.info('Stored response in cache')
self._reduce_age(now)
self._reduce_count()
def retrieve(self, request):
"""Look up request in cache, skipping if verb is forbidden.
:param requests.Request request: HTTP request
"""
if request.method not in CACHE_VERBS:
return
try:
response = self.data[request.url]['response']
logger.info('Retrieved response from cache')
return response
except KeyError:
return None
def clear(self):
"Clear cache."
self.data = OrderedDict()
def _parse_fields(parsed):
"""Parse form fields from HTML.
:param BeautifulSoup parsed: Parsed HTML
:return OrderedDict: Collection of field objects
"""
# Note: Call this `rv` to avoid name conflict with `fields` module
rv = OrderedDict()
# Prepare field tags
tags = parsed.find_all(_tag_ptn)
for tag in tags:
helpers.lowercase_attr_names(tag)
while tags:
tag = tags.pop(0)
tag_type = tag.name.lower()
# Get name attribute, skipping if undefined
name = tag.get('name')
if name is None:
continue
name = name.lower()
field = None
# Create form field
if tag_type == 'input':
tag_type = tag.get('type', '').lower()
if tag_type == 'file':
field = fields.FileInput(tag)
elif tag_type == 'radio':
radios = _group_flat_tags(tag, tags)
field = fields.Radio(radios)
elif tag_type == 'checkbox':
checkboxes = _group_flat_tags(tag, tags)
field = fields.Checkbox(checkboxes)
else:
field = fields.Input(tag)
elif tag_type == 'textarea':
field = fields.Textarea(tag)
elif tag_type == 'select':
if tag.get('multiple') is not None:
field = fields.MultiSelect(tag)
else:
field = fields.Select(tag)
# Add field
if field is not None:
rv[name] = field
return rv
def clear(self):
"Clear cache."
self.data = OrderedDict()
def __init__(self, max_age=None, max_count=None):
self.data = OrderedDict()
self.max_age = max_age
self.max_count = max_count
def clear(self):
"Clear cache."
self.data = OrderedDict()
def __init__(self, max_age=None, max_count=None):
self.data = OrderedDict()
self.max_age = max_age
self.max_count = max_count
class RoboCache(object):
def __init__(self, max_age=None, max_count=None):
self.data = OrderedDict()
self.max_age = max_age
self.max_count = max_count
def _reduce_age(self, now):
"""Reduce size of cache by date.
:param datetime.datetime now: Current time
"""
if self.max_age:
for key, value in iteritems(self.data):
if now - value['date'] > self.max_age:
del self.data[key]
def _reduce_count(self):
"""Reduce size of cache by count.
"""
if self.max_count:
while len(self.data) > self.max_count:
self.data.popitem(last=False)
def store(self, response):
"""Store response in cache, skipping if code is forbidden.
:param requests.Response response: HTTP response
"""
if response.status_code not in CACHE_CODES:
return
now = datetime.datetime.now()
self.data[response.url] = {
'date': now,
'response': response,
}
logger.info('Stored response in cache')
self._reduce_age(now)
self._reduce_count()
def retrieve(self, request):
"""Look up request in cache, skipping if verb is forbidden.
:param requests.Request request: HTTP request
"""
if request.method not in CACHE_VERBS:
return
try:
response = self.data[request.url]['response']
logger.info('Retrieved response from cache')
return response
except KeyError:
return None
def clear(self):
"Clear cache."
self.data = OrderedDict()
