def save_proxies(url): try: r = fetch(url) except requests.exceptions.RequestException: return False addresses = re.findall(PROXY_REGEX, r.text) for address in addresses: proxy = Proxy(address=address) try: proxy.save() except NotUniqueError: pass
def add_proxy(request): urls = request.body print urls urlList = urls.split(',') rst=[] for url in urlList: try: p = Proxy(url = url, rate=0) p.save() except Exception as e: print rst.append(e.message) continue return HttpResponse(reduce(lambda x,y:x+','+y, rst))
def add_proxy(request): urls = request.body print urls urlList = urls.split(',') rst = [] for url in urlList: try: p = Proxy(url=url, rate=0) p.save() except Exception as e: print rst.append(e.message) continue return HttpResponse(reduce(lambda x, y: x + ',' + y, rst))
def save(self): ## create diff Proxy.objects.filter(delegating=self.user).delete() newproxy = Proxy(delegating=self.user,isdefault=True) newproxy.save() if 'main_proxy' in self.data.keys(): for user in self.data.getlist('main_proxy'): user_object = CustomUser.objects.get(pk=user) newproxy.delegates.add(user_object) newproxy.save() for count in xrange(int(self.data["tagfieldcount"])): if "side_proxy%d"%count in self.data.keys() and "side_proxy_tags%d"%count in self.data.keys(): newproxy = Proxy(delegating=self.user) newproxy.save() for user in self.data.getlist("side_proxy%d"%count): user_object = CustomUser.objects.get(pk=user) newproxy.delegates.add(user_object) for tag in self.data.getlist("side_proxy_tags%d"%count): tag_object = Tag.objects.get(pk=tag) newproxy.tags.add(tag_object) newproxy.save() return
def save(self): ### first disable all existing proxies (suboptimal, but easiest) ### for proxy in Proxy.objects.filter(delegating=self.user): proxy.disable() ### add new proxy for each form element ### newproxy = Proxy(delegating=self.user, isdefault=True) newproxy.save() if 'main_proxy' in self.data.keys(): for user in self.data.getlist('main_proxy'): user_object = CustomUser.objects.get(pk=user) newproxy.delegates.add(user_object) newproxy.save() for count in xrange(int(self.data["tagfieldcount"])): if "side_proxy%d" % count in self.data.keys() and "side_proxy_tags%d" % count in self.data.keys(): newproxy = Proxy(delegating=self.user) newproxy.save() for user in self.data.getlist("side_proxy%d"%count): user_object = CustomUser.objects.get(pk=user) newproxy.delegates.add(user_object) for tag in self.data.getlist("side_proxy_tags%d"%count): tag_object = Tag.objects.get(pk=tag) newproxy.tags.add(tag_object) newproxy.save() return
def save(self): ## create diff Proxy.objects.filter(delegating=self.user).delete() newproxy = Proxy(delegating=self.user, isdefault=True) newproxy.save() if 'main_proxy' in self.data.keys(): for user in self.data.getlist('main_proxy'): user_object = CustomUser.objects.get(pk=user) newproxy.delegates.add(user_object) newproxy.save() for count in xrange(int(self.data["tagfieldcount"])): if "side_proxy%d" % count in self.data.keys( ) and "side_proxy_tags%d" % count in self.data.keys(): newproxy = Proxy(delegating=self.user) newproxy.save() for user in self.data.getlist("side_proxy%d" % count): user_object = CustomUser.objects.get(pk=user) newproxy.delegates.add(user_object) for tag in self.data.getlist("side_proxy_tags%d" % count): tag_object = Tag.objects.get(pk=tag) newproxy.tags.add(tag_object) newproxy.save() return
def save(self): ### first disable all existing proxies (suboptimal, but easiest) ### for proxy in Proxy.objects.filter(delegating=self.user): proxy.disable() ### add new proxy for each form element ### newproxy = Proxy(delegating=self.user, isdefault=True) newproxy.save() if 'main_proxy' in self.data.keys(): for user in self.data.getlist('main_proxy'): user_object = CustomUser.objects.get(pk=user) newproxy.delegates.add(user_object) newproxy.save() for count in xrange(int(self.data["tagfieldcount"])): if "side_proxy%d" % count in self.data.keys( ) and "side_proxy_tags%d" % count in self.data.keys(): newproxy = Proxy(delegating=self.user) newproxy.save() for user in self.data.getlist("side_proxy%d" % count): user_object = CustomUser.objects.get(pk=user) newproxy.delegates.add(user_object) for tag in self.data.getlist("side_proxy_tags%d" % count): tag_object = Tag.objects.get(pk=tag) newproxy.tags.add(tag_object) newproxy.save() return
def add_proxy(request): if request.method == 'POST': port = request.POST.get('port', '0') destination = request.POST.get('destination', 'https://empty') delay = request.POST.get('delay', '0') p = Proxy(delay=delay, port=port, destination=destination, started=False, pid=0) p.save() new_id = p.id proxy = Proxy.objects.filter(id=new_id).values() response = [{"message": "proxy was started", "proxy": list(proxy)[0]}] return JsonResponse(response, safe=False)
def use_thread_with_queue2(): cleanup() # in_queue = Queue.Queue() # out_queue = Queue.Queue() # # for i in range(5): # t = threading.Thread(target=save_proxies_with_queue2, # args=(in_queue, out_queue)) # t.setDaemon(True) # t.start() # # for url in PROXY_SITES: # in_queue.put(url) # # result = [] # # for i in range(5): # t = threading.Thread(target=append_result, # args=(out_queue, result)) # t.setDaemon(True) # t.start() # # in_queue.join() # out_queue.join() addresses = [] # mogu_key = "" # res = requests.get(mogu_key) # addresses = res.json()['msg'] for address in addresses: proxy = Proxy(address=address['ip'] + ':' + address['port']) try: proxy.save() except NotUniqueError: pass pool = Pool(10) pool.map(check_proxy, Proxy.objects.all()) print(len(addresses)) print(Proxy.objects.count())
def save_proxies(url): proxies = [] try: # if url == 'http://www.kuaidaili.com/free': # import pdb; # pdb.set_trace() res = requests.get(url) except requests.exceptions.RequestException: return False addresses = re.findall(PROXY_REGEX, res.text) for address in addresses: proxy = Proxy(address=address) try: proxy.save() except NotUniqueError: pass else: proxies.append(address) return proxies
from models import Proxy from main_proxy import text_proxy_generator for proxy in text_proxy_generator(): proxy = Proxy(host_port=proxy) proxy.save()