def spi_response(self, response, *args, **kwargs):
'''Response of the spi_request are handled here
'''
if 'text/html' in response.headers['Content-Type']:
hash_val = Hasher.HashMD5(response.content)
if hash_val not in self.URLhash:
self.URLhash.add(hash_val)
self.URLset.union(Links.parse_link(response))
def spi_response(self, response):
'''Response of the spi_request are handled here
'''
if 'text/html' in response.headers[
'Content-Type'] and response.status_code == 200:
hash_val = Hasher.HashMD5(response.content)
if self.redis.getVariable(hash_val) is None:
if self.database.isConn():
self.database.saveData(hash=hash_val,
url=response.url,
content=response)
self.redis.setVariable(hash_val, response.url)
[self.URLset.put(link) for link in Links.parse_link(response)]
