def _parse_robots(self, response):
#A lot of work to provide the expire time which we don't actually use
ttl = max(self.min_ttl, Utility.get_ttl(response.headers, self.default_ttl))
rp = Rules(response.url, response.status, response.body, time.time() + ttl)
rp.parse(response.body)
self._parsers[urlparse_cached(response).netloc] = rp
