def __regular_parser(self, response, parser):
"""
Parsing for regular expressions
:param response:
:param parser:
:return:
"""
proxy_list = []
pattern = re.compile(parser['pattern'])
matches = pattern.findall(response)
if matches is not None:
for match in matches:
try:
ip = match[parser['position']['ip']]
port = match[parser['position']['port']]
# The type of the site has not been reliable so it is still the default, will be detected later
types = 0
protocol = 0
addr = self.ips.getIpAddr(self.ips.str_to_ip(ip))
# print(ip,port)
if text_('省') in addr or self.home_country(addr):
country = text_('国内')
area = addr
else:
country = text_('国外')
area = addr
except Exception as e:
log.warning(e.args)
continue
proxy = {'ip': ip, 'port': port, 'types': types, 'protocol': protocol, 'country': country, 'area': area,
'speed': 100}
proxy_list.append(proxy)
return proxy_list
def __xpath_parser(self, response, parser):
"""
Parsing against xpath methods
:param response:
:param parser:
:return:
"""
proxy_list = []
root = html.etree.HTML(response)
proxies = root.xpath(parser['pattern'])
for proxy in proxies:
try:
ip = proxy.xpath(parser['position']['ip'])[0].text
port = proxy.xpath(parser['position']['port'])[0].text
types = 0
protocol = 0
addr = self.ips.getIpAddr(self.ips.str_to_ip(ip))
if text_('省') in addr or self.home_country(addr):
country = text_('国内')
area = addr
else:
country = text_('国外')
area = addr
except Exception as e:
print(e.__str__())
continue
# ip，port，types(0高匿名，1透明)，protocol(0 http,1 https http),country,area,speed
proxy = {'ip': ip, 'port': int(port), 'types': int(types), 'protocol': int(protocol),
'country': country, 'area': area, 'speed': 100}
proxy_list.append(proxy)
return proxy_list
def proxy_list_parser(self, response, parser):
proxy_list = []
pattern = re.compile(parser['pattern'])
matches = pattern.findall(response)
if matches:
for match in matches:
try:
ip_port = base64.b64decode(match.replace("Proxy('", "").replace("')", ""))
ip = ip_port.split(':')[0]
port = ip_port.split(':')[1]
types = 0
protocol = 0
addr = self.ips.getIpAddr(self.ips.str_to_ip(ip))
# print(ip,port)
if text_('省') in addr or self.home_country(addr):
country = text_('国内')
area = addr
else:
country = text_('国外')
area = addr
except Exception as e:
log.warning(e.args)
continue
proxy = {'ip': ip, 'port': int(port), 'types': types, 'protocol': protocol, 'country': country,
'area': area, 'speed': 100}
proxy_list.append(proxy)
return proxy_list
def save_message(event):
user_id = event['user_id']
group_id = event['group_id']
message = event['message']
obj = ChatMessage.objects.create(create_user_id=user_id, group_id=group_id, message=message)
serializer = ChatMessageReadSerializer(obj)
j = JSONRenderer().render(serializer.data)
data = json.loads(text_(j))
return data
def home_country(cls, addr):
"""
To determine which country the address is from
:param addr:
:return:
"""
for area in CHINA_AREA:
if text_(area) in addr:
return True
return False
def save_cmd(event):
cmd = event['cmd']
try:
obj = History.objects.get(cmd=cmd)
obj.num += 1
obj.save()
except:
obj = History.objects.create(cmd=cmd, num=1)
serializer = HistorySerializer(obj)
j = JSONRenderer().render(serializer.data)
data = json.loads(text_(j))
return data