def __test_manager(self):
notice_timer = default_timer()
while True:
now = default_timer()
# Print statistics regularly.
if now >= notice_timer + self.notice_interval:
log.info('Tested a total of %d good and %d bad proxies.',
self.stats['total_valid'], self.stats['total_fail'])
log.info('Tested %d good and %d bad proxies in last %ds.',
self.stats['valid'], self.stats['fail'],
self.notice_interval)
notice_timer = now
self.stats['valid'] = 0
self.stats['fail'] = 0
try:
with self.proxy_updates_lock:
queue_size = self.test_queue.qsize()
log.debug('%d proxy tests running...',
len(self.test_hashes) - queue_size)
# Upsert updated proxies into database.
updates_count = len(self.proxy_updates)
if updates_count > 10:
proxies = self.proxy_updates.values()
result = False
with Proxy.database().atomic():
query = Proxy.insert_many(proxies).upsert()
result = query.execute()
if result:
log.debug('Upserted %d proxies to database.',
updates_count)
else:
log.warning('Failed to upsert %d proxies.',
updates_count)
self.proxy_updates = {}
# Request more proxies to test.
refill = self.max_concurrency - queue_size
if refill > 0:
refill = min(refill, self.max_concurrency)
proxylist = Proxy.get_scan(refill, self.test_hashes,
self.scan_interval)
count = 0
for proxy in proxylist:
self.test_queue.put(proxy)
self.test_hashes.append(proxy['hash'])
count += 1
log.debug('Enqueued %d proxies for testing.', count)
except Exception as e:
log.exception('Exception in proxy manager: %s.', e)
if self.running.is_set():
log.debug('Proxy manager shutting down...')
break
time.sleep(5)
def new():
form = Campaign_form()
if form.validate_on_submit():
params = {
'utm_source': form.e_cs.data,
'utm_medium': form.e_cm.data,
'utm_term': form.e_ct.data,
'utm_content': form.e_cc.data,
'utm_campaign': form.e_cn.data
}
order = [
'utm_source', 'utm_medium', 'utm_term', 'utm_content',
'utm_campaign'
]
queryString = "&".join(
[item + '=' + urllib.quote_plus(params[item]) for item in order])
url = form.url.data
if url[-1] != '/':
url = url + '/'
email_url = url + '?' + queryString
campaign = Campaign.create(
name=form.name.data,
active=form.active.data,
url=form.url.data,
time_on_site_min=form.time_on_site_min.data,
time_on_site_max=form.time_on_site_max.data,
pages_min=form.pages_min.data,
pages_max=form.pages_max.data,
visits_per_day_min=form.visits_per_day_min.data,
visits_per_day_max=form.visits_per_day_max.data,
length_from=form.length_from.data,
length_to=form.length_to.data,
bounce_rate=form.bounce_rate.data,
organic_source=form.organic_source.data,
direct_source=form.direct_source.data,
social_source=form.social_source.data,
email_source=form.email_source.data,
referral_source=form.referral_source.data,
organic_google=form.organic_google.data,
organic_yahoo=form.organic_yahoo.data,
organic_bing=form.organic_bing.data,
organic_aol=form.organic_aol.data,
organic_keywords=form.organic_keywords.data,
referral_settings=form.referral_settings.data,
social_settings=form.social_settings.data,
e_cs=form.e_cs.data,
e_cm=form.e_cm.data,
e_ct=form.e_ct.data,
e_cc=form.e_cc.data,
e_cn=form.e_cn.data,
email_url=email_url,
use_proxy_list=form.use_proxy_list.data,
use_proxy_api=form.use_proxy_api.data,
proxy_api_url=form.proxy_api_url.data,
reused_proxy=form.reused_proxy.data,
ua_safari=form.ua_safari.data,
ua_firefox=form.ua_firefox.data,
ua_ie=form.ua_ie.data,
ua_opera=form.ua_opera.data,
ua_chrome=form.ua_chrome.data,
ua_iphone=form.ua_iphone.data,
ua_ipad=form.ua_ipad.data,
ua_android=form.ua_android.data,
ua_win=form.ua_win.data,
ua_mac=form.ua_mac.data,
ua_linux=form.ua_linux.data)
if form.proxy.data:
campaign.proxy_filename = form.proxy.data.filename
campaign.save()
proxies = form.proxy.data.read().splitlines()
data_source = []
for proxy in proxies:
p = proxy.split(':')
data_source.append({
'ip': p[0],
'port': p[1],
'campaign': campaign,
'from_list': True
})
with db.atomic():
for idx in range(0, len(data_source), 100):
Proxy.insert_many(data_source[idx:idx + 100]).execute()
return redirect(url_for('main'))
# print form.errors
return render_template('new.jade', form=form)
