def spi_response(self, response, *args, **kwargs): '''Response of the spi_request are handled here ''' if 'text/html' in response.headers['Content-Type']: hash_val = Hasher.HashMD5(response.content) if hash_val not in self.URLhash: self.URLhash.add(hash_val) self.URLset.union(Links.parse_link(response))
def spi_response(self, response): '''Response of the spi_request are handled here ''' if 'text/html' in response.headers[ 'Content-Type'] and response.status_code == 200: hash_val = Hasher.HashMD5(response.content) if self.redis.getVariable(hash_val) is None: if self.database.isConn(): self.database.saveData(hash=hash_val, url=response.url, content=response) self.redis.setVariable(hash_val, response.url) [self.URLset.put(link) for link in Links.parse_link(response)]