class RoboCache(object): def __init__(self, max_age=None, max_count=None): self.data = OrderedDict() self.max_age = max_age self.max_count = max_count def _reduce_age(self, now): """Reduce size of cache by date. :param datetime.datetime now: Current time """ if self.max_age: for key, value in iteritems(self.data): if now - value['date'] > self.max_age: del self.data[key] def _reduce_count(self): """Reduce size of cache by count. """ if self.max_count: while len(self.data) > self.max_count: self.data.popitem(last=False) def store(self, response): """Store response in cache, skipping if code is forbidden. :param requests.Response response: HTTP response """ if response.status_code not in CACHE_CODES: return now = datetime.datetime.now() self.data[response.url] = { 'date': now, 'response': response, } logger.info('Stored response in cache') self._reduce_age(now) self._reduce_count() def retrieve(self, request): """Look up request in cache, skipping if verb is forbidden. :param requests.Request request: HTTP request """ if request.method not in CACHE_VERBS: return try: response = self.data[request.url]['response'] logger.info('Retrieved response from cache') return response except KeyError: return None def clear(self): "Clear cache." self.data = OrderedDict()
def _parse_fields(parsed): """Parse form fields from HTML. :param BeautifulSoup parsed: Parsed HTML :return OrderedDict: Collection of field objects """ # Note: Call this `rv` to avoid name conflict with `fields` module rv = OrderedDict() # Prepare field tags tags = parsed.find_all(_tag_ptn) for tag in tags: helpers.lowercase_attr_names(tag) while tags: tag = tags.pop(0) tag_type = tag.name.lower() # Get name attribute, skipping if undefined name = tag.get('name') if name is None: continue name = name.lower() field = None # Create form field if tag_type == 'input': tag_type = tag.get('type', '').lower() if tag_type == 'file': field = fields.FileInput(tag) elif tag_type == 'radio': radios = _group_flat_tags(tag, tags) field = fields.Radio(radios) elif tag_type == 'checkbox': checkboxes = _group_flat_tags(tag, tags) field = fields.Checkbox(checkboxes) else: field = fields.Input(tag) elif tag_type == 'textarea': field = fields.Textarea(tag) elif tag_type == 'select': if tag.get('multiple') is not None: field = fields.MultiSelect(tag) else: field = fields.Select(tag) # Add field if field is not None: rv[name] = field return rv
def clear(self): "Clear cache." self.data = OrderedDict()
def __init__(self, max_age=None, max_count=None): self.data = OrderedDict() self.max_age = max_age self.max_count = max_count