def change_proxy(self):
proxy = self.proxies.pop(0)
self.proxies.append(proxy)
msg = 'changing proxy on ' + proxy
logger.write('ProxyDM',msg=msg)
print msg
schemes = ['http','https']
for scheme in schemes:
self.opener.proxies[scheme]="%s://%s" % (scheme, proxy)
def tasks_executing_loop(self):
task = self.storage.get_task()
if task.need_html:
print ' downloading: %s' % task.link
if not helpers.check_local_link('http://lib.ru', task.link):
logger.write('TM_LibRu: not local link!', notlocallink=task.link)
self.storage.mark_executed()
return
page = (self.dm.download(task.link) , )
url = page[0][1].url
if not helpers.check_local_link('http://lib.ru', url):
logger.write('TM_LibRu: not local link!', notlocallink=url,link=task.link)
self.storage.mark_executed()
return
else:
page = ()
self.print_executing(task)
self.execute_task(task, page)
def mark_executed(self):
if self.curtask:
self.state.tasks.remove(self.curtask)
self.curtask = None
else:
logger.write("Task Stated Storage,strange behavior", task=self.curtask)