class Kuai(object):
def __init__(self):
self.log = Log(__class__.__name__)
pass
@staticmethod
@check
def get_response(url):
resp = requests.get(url=url, headers=HEADER)
resp.encoding = resp.apparent_encoding
resp.raise_for_status
return resp
@property
def ips(self):
for url in URL:
r = self.get_response(url)
try:
html = etree.HTML(r.text)
except AttributeError as e:
self.log.error("Response is None,ErrorType:%s" % e)
else:
iplist = html.xpath(XPATH)
for ip in iplist:
proxy = ':'.join(ip.xpath("./td/text()")[:2])
self.log.info("Find:%s" % proxy)
yield proxy
class Mongo(object):
def __init__(self, dbname="test", collection="test", host=None, port=None):
self.dbn = dbname
self.coln = collection
self.client = MongoClient(host, port)
self.db = self.client[dbname]
self.collection = self.db[collection]
self.log = Log(__class__.__name__)
def insert(self, dict_data):
if isinstance(dict_data, dict):
# check existance before insert.
if not self.is_exist(dict_data):
self.collection.insert(dict_data)
self.log.info("Insert dict is :%s" % (str(dict_data)))
else:
self.log.info("Error: %s,dict is :%s" %
(TypeError, eval(dict_data)))
raise TypeError
def delete(self, key, value):
if value or key:
if self.collection.delete({key: value}):
self.log.info(
"MongoDB delete Error: %s,key is :%s,value is :%s" %
("Not exist", key, value))
else:
self.log.error("MongoDB delete Error: %s,key is :%s,value is :%s" %
(AttributeError, key, value))
raise AttributeError
# find all remarks with key
def find_all(self):
for results in self.collection.find():
yield results
# find one remark with key
def find_one(self):
return random.choice([proxy for proxy in self.find_all()])
def is_exist(self, dict_data):
if self.collection.find_one(dict_data):
self.log.warning("Dict is exist :%s" % (str(dict_data)))
return True
else:
return False
def delete_all(self):
self.collection.remove()
self.log.warning("MongoDB delete all remarks.")
# function drop_database is belong to MongoClient.A difference between Mongo shell.
def drop_db(self):
self.client.drop_database(self.dbn)
self.log.warning("MongoDB delete db %s." % self.dbn)
def drop_collection(self):
self.collection.drop()
self.log.warning("MongoDB delete collection %s." % self.coln)
class You(object):
def __init__(self):
self.log = Log(__class__.__name__)
pass
@staticmethod
def get_response(url):
resp = requests.get(url=url, headers = HEADER)
resp.encoding = resp.apparent_encoding
resp.raise_for_status
return resp
@property
def ips(self):
lis = []
start_resp = self.get_response(start_url)
try:
html = etree.HTML(start_resp.text)
except AttributeError as e:
self.log.error("Response is None,ErrorType:%s" % e)
else:
parse_url = html.xpath(XPATH)[0]
num = re.search("\d+",parse_url)
link = re.sub("\d+",num.group(0)+"_{index}",parse_url)
urls = (link.format(index=ind) for ind in range(2, PAGE + 1))
r = self.get_response(parse_url)
try:
lis = lis + re.findall("(?:\d{1,3}\.){3}\d{1,3}\:\d{1,4}", r.text)
except AttributeError as e:
self.log.error("Response is None,ErrorType:%s" % e)
else:
for url in urls:
r = self.get_response(url)
lis = lis + re.findall("(?:\d{1,3}\.){3}\d{1,3}\:\d{1,4}",r.text)
for x in lis:
self.log.info("Find:%s" % x)
yield x