for img in spider.imgs:
images.add(DBImage(url = img, from_website = web.id, save_path = spider.title))
web.title = spider.title
break
if web.request_state != REQUEST_STATE.SUCC:
Log.e("{procMain} fetch url(%s) id(%d) failed!", web.url, web.id)
updateWebsiteStates(websites)
uploadWesites(wbs)
uploadImages(images)
else:
sleep(3) # sleep for a while to wait for the database update
except KeyboardInterrupt:
Log.i("{procMain} spider process exit for a KeyboardInterrupt")
if __name__ == '__main__':
Log.setup('spider')
Log.d('setting up spider......')
#procMain(1, {})
num = 1
if len(os.sys.argv) > 1:
num = int(os.sys.argv[1])
if num < 1:
num = 1
pm = ProcessManager(procMain, maxWorker = num)
pm.run()
else:
print("Restart all processes")
self.terminateAll()
self.setupAll()
self.startAll()
elif cmd == "raw":
print self.processes
print self.procStates
elif cmd == "h":
self.help()
self.terminateAll()
if __name__ == '__main__':
Log.setup(__name__)
def worker(pid, states):
from time import sleep
try:
states[pid] = STATE_IDLE
sleep(3)
states[pid] = STATE_BUSY
sleep(30)
states[pid] = STATE_TERMINATE
except KeyboardInterrupt:
Log.d("worker end!")
spm = ProcessManager(worker)
spm.run()
img.request_state = REQUEST_STATE.SUCC
img.save_path = save_dir
break
if img.request_state != REQUEST_STATE.SUCC:
bSite.addFaileSite(site)
bSite.save()
#Log.e("{procMain} download image(%s) failed!", img.url)
updateImages(images)
else:
time.sleep(
3) # sleep for a while to wait for the database update
except KeyboardInterrupt:
Log.i("{procMain} downloader process exit for a KeyboardInterrupt")
if __name__ == '__main__':
Log.setup("download")
Log.d('setting up downloader......')
"""
procMain(1, {})
"""
num = 1
if len(os.sys.argv) > 1:
num = int(os.sys.argv[1])
if num < 1:
num = 1
pm = ProcessManager(procMain, maxWorker=num)
pm.run()
