def __test_manager(self): notice_timer = default_timer() while True: now = default_timer() # Print statistics regularly. if now >= notice_timer + self.notice_interval: log.info('Tested a total of %d good and %d bad proxies.', self.stats['total_valid'], self.stats['total_fail']) log.info('Tested %d good and %d bad proxies in last %ds.', self.stats['valid'], self.stats['fail'], self.notice_interval) notice_timer = now self.stats['valid'] = 0 self.stats['fail'] = 0 try: with self.proxy_updates_lock: queue_size = self.test_queue.qsize() log.debug('%d proxy tests running...', len(self.test_hashes) - queue_size) # Upsert updated proxies into database. updates_count = len(self.proxy_updates) if updates_count > 10: proxies = self.proxy_updates.values() result = False with Proxy.database().atomic(): query = Proxy.insert_many(proxies).upsert() result = query.execute() if result: log.debug('Upserted %d proxies to database.', updates_count) else: log.warning('Failed to upsert %d proxies.', updates_count) self.proxy_updates = {} # Request more proxies to test. refill = self.max_concurrency - queue_size if refill > 0: refill = min(refill, self.max_concurrency) proxylist = Proxy.get_scan(refill, self.test_hashes, self.scan_interval) count = 0 for proxy in proxylist: self.test_queue.put(proxy) self.test_hashes.append(proxy['hash']) count += 1 log.debug('Enqueued %d proxies for testing.', count) except Exception as e: log.exception('Exception in proxy manager: %s.', e) if self.running.is_set(): log.debug('Proxy manager shutting down...') break time.sleep(5)
def new(): form = Campaign_form() if form.validate_on_submit(): params = { 'utm_source': form.e_cs.data, 'utm_medium': form.e_cm.data, 'utm_term': form.e_ct.data, 'utm_content': form.e_cc.data, 'utm_campaign': form.e_cn.data } order = [ 'utm_source', 'utm_medium', 'utm_term', 'utm_content', 'utm_campaign' ] queryString = "&".join( [item + '=' + urllib.quote_plus(params[item]) for item in order]) url = form.url.data if url[-1] != '/': url = url + '/' email_url = url + '?' + queryString campaign = Campaign.create( name=form.name.data, active=form.active.data, url=form.url.data, time_on_site_min=form.time_on_site_min.data, time_on_site_max=form.time_on_site_max.data, pages_min=form.pages_min.data, pages_max=form.pages_max.data, visits_per_day_min=form.visits_per_day_min.data, visits_per_day_max=form.visits_per_day_max.data, length_from=form.length_from.data, length_to=form.length_to.data, bounce_rate=form.bounce_rate.data, organic_source=form.organic_source.data, direct_source=form.direct_source.data, social_source=form.social_source.data, email_source=form.email_source.data, referral_source=form.referral_source.data, organic_google=form.organic_google.data, organic_yahoo=form.organic_yahoo.data, organic_bing=form.organic_bing.data, organic_aol=form.organic_aol.data, organic_keywords=form.organic_keywords.data, referral_settings=form.referral_settings.data, social_settings=form.social_settings.data, e_cs=form.e_cs.data, e_cm=form.e_cm.data, e_ct=form.e_ct.data, e_cc=form.e_cc.data, e_cn=form.e_cn.data, email_url=email_url, use_proxy_list=form.use_proxy_list.data, use_proxy_api=form.use_proxy_api.data, proxy_api_url=form.proxy_api_url.data, reused_proxy=form.reused_proxy.data, ua_safari=form.ua_safari.data, ua_firefox=form.ua_firefox.data, ua_ie=form.ua_ie.data, ua_opera=form.ua_opera.data, ua_chrome=form.ua_chrome.data, ua_iphone=form.ua_iphone.data, ua_ipad=form.ua_ipad.data, ua_android=form.ua_android.data, ua_win=form.ua_win.data, ua_mac=form.ua_mac.data, ua_linux=form.ua_linux.data) if form.proxy.data: campaign.proxy_filename = form.proxy.data.filename campaign.save() proxies = form.proxy.data.read().splitlines() data_source = [] for proxy in proxies: p = proxy.split(':') data_source.append({ 'ip': p[0], 'port': p[1], 'campaign': campaign, 'from_list': True }) with db.atomic(): for idx in range(0, len(data_source), 100): Proxy.insert_many(data_source[idx:idx + 100]).execute() return redirect(url_for('main')) # print form.errors return render_template('new.jade', form=form)