def save_proxies(url):
try:
r = fetch(url)
except requests.exceptions.RequestException:
return False
addresses = re.findall(PROXY_REGEX, r.text)
for address in addresses:
proxy = Proxy(address=address)
try:
proxy.save()
except NotUniqueError:
pass
def save_proxies(url):
try:
r = fetch(url)
except requests.exceptions.RequestException:
return False
addresses = re.findall(PROXY_REGEX, r.text)
for address in addresses:
proxy = Proxy(address=address)
try:
proxy.save()
except NotUniqueError:
pass
def add_proxy(request):
urls = request.body
print urls
urlList = urls.split(',')
rst=[]
for url in urlList:
try:
p = Proxy(url = url, rate=0)
p.save()
except Exception as e:
print rst.append(e.message)
continue
return HttpResponse(reduce(lambda x,y:x+','+y, rst))
def add_proxy(request):
urls = request.body
print urls
urlList = urls.split(',')
rst = []
for url in urlList:
try:
p = Proxy(url=url, rate=0)
p.save()
except Exception as e:
print rst.append(e.message)
continue
return HttpResponse(reduce(lambda x, y: x + ',' + y, rst))
def save(self):
## create diff
Proxy.objects.filter(delegating=self.user).delete()
newproxy = Proxy(delegating=self.user,isdefault=True)
newproxy.save()
if 'main_proxy' in self.data.keys():
for user in self.data.getlist('main_proxy'):
user_object = CustomUser.objects.get(pk=user)
newproxy.delegates.add(user_object)
newproxy.save()
for count in xrange(int(self.data["tagfieldcount"])):
if "side_proxy%d"%count in self.data.keys() and "side_proxy_tags%d"%count in self.data.keys():
newproxy = Proxy(delegating=self.user)
newproxy.save()
for user in self.data.getlist("side_proxy%d"%count):
user_object = CustomUser.objects.get(pk=user)
newproxy.delegates.add(user_object)
for tag in self.data.getlist("side_proxy_tags%d"%count):
tag_object = Tag.objects.get(pk=tag)
newproxy.tags.add(tag_object)
newproxy.save()
return
def save(self):
### first disable all existing proxies (suboptimal, but easiest) ###
for proxy in Proxy.objects.filter(delegating=self.user):
proxy.disable()
### add new proxy for each form element ###
newproxy = Proxy(delegating=self.user, isdefault=True)
newproxy.save()
if 'main_proxy' in self.data.keys():
for user in self.data.getlist('main_proxy'):
user_object = CustomUser.objects.get(pk=user)
newproxy.delegates.add(user_object)
newproxy.save()
for count in xrange(int(self.data["tagfieldcount"])):
if "side_proxy%d" % count in self.data.keys() and "side_proxy_tags%d" % count in self.data.keys():
newproxy = Proxy(delegating=self.user)
newproxy.save()
for user in self.data.getlist("side_proxy%d"%count):
user_object = CustomUser.objects.get(pk=user)
newproxy.delegates.add(user_object)
for tag in self.data.getlist("side_proxy_tags%d"%count):
tag_object = Tag.objects.get(pk=tag)
newproxy.tags.add(tag_object)
newproxy.save()
return
def save(self):
## create diff
Proxy.objects.filter(delegating=self.user).delete()
newproxy = Proxy(delegating=self.user, isdefault=True)
newproxy.save()
if 'main_proxy' in self.data.keys():
for user in self.data.getlist('main_proxy'):
user_object = CustomUser.objects.get(pk=user)
newproxy.delegates.add(user_object)
newproxy.save()
for count in xrange(int(self.data["tagfieldcount"])):
if "side_proxy%d" % count in self.data.keys(
) and "side_proxy_tags%d" % count in self.data.keys():
newproxy = Proxy(delegating=self.user)
newproxy.save()
for user in self.data.getlist("side_proxy%d" % count):
user_object = CustomUser.objects.get(pk=user)
newproxy.delegates.add(user_object)
for tag in self.data.getlist("side_proxy_tags%d" % count):
tag_object = Tag.objects.get(pk=tag)
newproxy.tags.add(tag_object)
newproxy.save()
return
def save(self):
### first disable all existing proxies (suboptimal, but easiest) ###
for proxy in Proxy.objects.filter(delegating=self.user):
proxy.disable()
### add new proxy for each form element ###
newproxy = Proxy(delegating=self.user, isdefault=True)
newproxy.save()
if 'main_proxy' in self.data.keys():
for user in self.data.getlist('main_proxy'):
user_object = CustomUser.objects.get(pk=user)
newproxy.delegates.add(user_object)
newproxy.save()
for count in xrange(int(self.data["tagfieldcount"])):
if "side_proxy%d" % count in self.data.keys(
) and "side_proxy_tags%d" % count in self.data.keys():
newproxy = Proxy(delegating=self.user)
newproxy.save()
for user in self.data.getlist("side_proxy%d" % count):
user_object = CustomUser.objects.get(pk=user)
newproxy.delegates.add(user_object)
for tag in self.data.getlist("side_proxy_tags%d" % count):
tag_object = Tag.objects.get(pk=tag)
newproxy.tags.add(tag_object)
newproxy.save()
return
def add_proxy(request):
if request.method == 'POST':
port = request.POST.get('port', '0')
destination = request.POST.get('destination', 'https://empty')
delay = request.POST.get('delay', '0')
p = Proxy(delay=delay,
port=port,
destination=destination,
started=False,
pid=0)
p.save()
new_id = p.id
proxy = Proxy.objects.filter(id=new_id).values()
response = [{"message": "proxy was started", "proxy": list(proxy)[0]}]
return JsonResponse(response, safe=False)
def use_thread_with_queue2():
cleanup()
# in_queue = Queue.Queue()
# out_queue = Queue.Queue()
#
# for i in range(5):
# t = threading.Thread(target=save_proxies_with_queue2,
# args=(in_queue, out_queue))
# t.setDaemon(True)
# t.start()
#
# for url in PROXY_SITES:
# in_queue.put(url)
#
# result = []
#
# for i in range(5):
# t = threading.Thread(target=append_result,
# args=(out_queue, result))
# t.setDaemon(True)
# t.start()
#
# in_queue.join()
# out_queue.join()
addresses = []
# mogu_key = ""
# res = requests.get(mogu_key)
# addresses = res.json()['msg']
for address in addresses:
proxy = Proxy(address=address['ip'] + ':' + address['port'])
try:
proxy.save()
except NotUniqueError:
pass
pool = Pool(10)
pool.map(check_proxy, Proxy.objects.all())
print(len(addresses))
print(Proxy.objects.count())
def save_proxies(url):
proxies = []
try:
# if url == 'http://www.kuaidaili.com/free':
# import pdb;
# pdb.set_trace()
res = requests.get(url)
except requests.exceptions.RequestException:
return False
addresses = re.findall(PROXY_REGEX, res.text)
for address in addresses:
proxy = Proxy(address=address)
try:
proxy.save()
except NotUniqueError:
pass
else:
proxies.append(address)
return proxies
from models import Proxy
from main_proxy import text_proxy_generator
for proxy in text_proxy_generator():
proxy = Proxy(host_port=proxy)
proxy.save()
