def _search(d, dbl_err_key, j):
customer_id, domain = j['customer_id'], j['domain']
log.info("start customer_id: {}, domain: {}".format(customer_id, domain))
url = "http://www.spamhaus.org/query/domain/{}".format(domain)
https_url = "http://www.spamhaus.org/query/domain/{}".format(domain)
ref_url = "http://www.spamhaus.org/query/domain/{}".format(domain)
s = requests.session()
proxies = next(sf_settings.CYCLE_PROXIES)
log.info("domain: {}, proxies: {}".format(domain, proxies))
s.proxies = proxies
s.headers.update(sf_settings.HEADERS)
s.headers.update({'User-Agent': random.choice(sf_settings.UA)})
s.headers.update({'Referer': ref_url})
r = s.get(https_url)
jschl_vc, passwd, jschl_answer = get_js_return(r.content)
log.info("jschl_vc: {}, passwd: {}, jschl_answer: {}".format(jschl_vc, passwd, jschl_answer))
if jschl_answer is not None:
payload = {'jschl_vc': jschl_vc, 'pass': passwd, 'jschl_answer': jschl_answer}
time.sleep(4)
s.get("https://www.spamhaus.org/cdn-cgi/l/chk_jschl?", params=payload)
r = s.get(https_url)
T = False
content = r.content
if content.find("is not listed in the DBL") > 0:
T = True
log.info("{} is not listed in the DBL".format(domain))
if content.find("is listed in the DBL") > 0:
T = True
log.error("{} is listed in the DBL".format(domain))
redis.lpush(dbl_err_key, d)
if not T:
raise Exception("query dbl with no response")
return
def init_routine():
while True:
if signal_stop: break
try:
log.info('init routine...')
init()
log.info('init routine finish...')
except BaseException as e:
log.error('init routine error...')
gevent.sleep(7200)
def save():
dbl_err_key = "domain:dbl:error"
while True:
if signal_stop: break
_, d = redis.brpop(dbl_err_key)
try:
_save(json.loads(d) )
except BaseException as e:
redis.lpush(dbl_err_key, d)
log.error(traceback.format_exc())
def search():
key = "rbl:search:domain"
while True:
if signal_stop: break
_, d = redis_db.brpop(key)
try:
_search(d)
except BaseException as e:
redis_db.lpush(key, d)
log.error(traceback.format_exc())
def search():
dbl_key = "domain:dbl:check"
dbl_err_key = "domain:dbl:error"
while True:
if signal_stop: break
_, d = redis.brpop(dbl_key)
try:
_search(d, dbl_err_key, json.loads(d) )
except BaseException as e:
redis.lpush(dbl_key, d)
log.error(traceback.format_exc())
gevent.sleep(10)
def distribute():
key = "rbl:search:ip"
while True:
if signal_stop: break
_, ip = redis_db.brpop(key)
ret_key = "rbl:search:ret:{}".format(ip)
if redis_db.exists(ret_key): continue
try:
_distribute(ip)
except BaseException as e:
redis_db.lpush(key, ip)
log.error(traceback.format_exc())
def hostname():
key = "rbl:search:hostname"
while True:
if signal_stop: break
_, ip = redis_db.brpop(key)
try:
ret = rblsearch.get_hostname(ip)
ret_key = "rbl:search:ret:{}".format(ip)
if not ret:
redis_db.hincrby(ret_key, "score", -5)
ret = "No Reverse DNS"
redis_db.hset(ret_key, "domain", ret)
log.info("search hostname: {}".format(ret))
except BaseException as e:
redis_db.lpush(key, ip)
log.error(traceback.format_exc())