def alternate_process():
ap = proxy_processing.Proxy_processing(type='alternate')
s_time = time.time()
while time.time() - s_time < 3600:
if ap.should_add_to_db():
loginfo.info('test alternate')
for i in range(etc.alternate_llen):
try:
tf, IP_info = ap.get_a_proxy()
if tf:
loginfo.info(
('in alternate mode get a proxy', IP_info))
ap.push_or_discare(IP_info)
loginfo.info('push, or discare')
else:
if not crawl_proxies.crawlProxies():
time.sleep(10)
logerr.error('crawlproxies con`t had been used')
else:
loginfo.info('run mimvp once')
except Exception as e:
logerr.error(e)
logerr.error('*******************')
time.sleep(5)
continue
else:
pass
# time.sleep(10)
else:
time.sleep(etc.alternate_sleep_time)
def check_proxy(type):
from Code import proxy_processing
ap = proxy_processing.Proxy_processing(type=type)
yn, proxy_info = ap.get_a_proxy()
if yn:
try:
ap.push_or_discare(proxy_info)
except Exception as e:
logerr.error(e)
def crawl_process():
ap = proxy_processing.Proxy_processing(type='crawl')
s_time = time.time()
while time.time() - s_time < 3600:
if ap.should_add_to_db():
loginfo.info('add to crawl')
try:
if not crawl_proxies.crawlProxies():
time.sleep(etc.crawl_sleep_time)
except Exception as e:
print(e)
time.sleep(10)
logerr.error(e)
else:
time.sleep(etc.crawl_sleep_time)
def immediate_db_rool():
ap = proxy_processing.Proxy_processing(type='immediate',from_db=etc.immediate_db)
s_time = time.time()
multiple_hits = 0
while time.time() - s_time < 3600:
tf,proxy_info = ap.get_a_proxy()
if tf:
if ap.push_or_discare(proxy_info):
multiple_hits = 0
else:
multiple_hits += 1
else:
time.sleep(etc.alternate_effective_time)
if multiple_hits > ap.from_redis.check_len_db():
time.sleep(etc.alternate_effective_time/2)
time.sleep(2)
def immediate_process():
ap = proxy_processing.Proxy_processing(type='immediate',from_db=etc.alternate_db,to_db=etc.immediate_db)
s_time = time.time()
while time.time() - s_time < 3600:
if ap.should_add_to_db():
loginfo.info('in the immediate_process')
for i in range(etc.immediate_llen):
try:
tf,IP_info=ap.get_a_proxy()
loginfo.info((tf,IP_info))
if tf:
ap.push_or_discare(IP_info)
loginfo.info('ab')
else:
time.sleep(10)
except Exception as e:
logerr.error(e)
time.sleep(5)
continue
else:
time.sleep(etc.immediate_sleep_time)
def get_a_proxy(anonymity=False):
from Code import proxy_processing
ap = proxy_processing.Proxy_processing(type='to_use',
from_db=etc.immediate_db,
to_db=etc.immediate_db)
yn, proxy_info = ap.get_a_proxy()
if yn:
try:
ap.push_or_discare(proxy_info)
if proxy_info['anonymity'] == u'高匿名':
loginfo.info(proxy_info)
return {
'http':
'http://' + proxy_info['IP'] + ':' + proxy_info['port']
}
else:
print(proxy_info['anonymity'])
get_a_proxy()
except Exception as e:
logerr.error(e)
return get_a_proxy()
loginfo.info(None)
return None