def load_url_data():
load_index_data = UseMongoData(load_mongodb_host, load_mongodb_port,
load_mongodb_user, load_mongodb_pass)
domain_data = [
i['subdomains']
for i in load_index_data.client_mongodb_load_domain_index()
]
return domain_data
class checkDomainHttpResponse(object):
def __init__(self):
self.timeout = 2
self.verify = True
self.load_index_data = UseMongoData(load_mongodb_host,load_mongodb_port,load_mongodb_user,load_mongodb_pass)
def _GET(self):
for load_domain in self.load_index_data.client_mongodb_load_domain_index():
domain = load_domain['subdomains']
if 'http' not in domain:
url = 'http://'+ str(domain)
try:
load_response_data = requests.get(url=url,headers={},verify=self.verify,timeout=self.timeout)
domain_status = load_response_data.status_code
load_response_data.encoding = 'utf-8'
soup = BeautifulSoup(load_response_data.text, 'lxml')
domain_title = soup.title.text
self.load_index_data.client_mongodb_update_domainStatus(domain,"online",domain_title,domain_status)
except Exception as e:
self.load_index_data.client_mongodb_update_domainStatus(domain,"offline","",0)