def run(self, session_id): """ Run the main function ... """ self.thread_count = self.config_result["content_thread"] # debug #print "Thread count to run mount %s"% self.thread_count pool = ThreadPool(int(self.thread_count)) if session_id == 0: try: read_file = open(self.share_session, "r").read().splitlines() except Exception, err_mess: print err_mess sys.exit(1) for line in read_file: pool.add_task(self.mount_sharing, line) pool.wait_completion() self.write_session_id(str(session_id)) session_id = session_id + 1
def cateGoodsControl(self, n):
# 恢复爬取状态
tp = ThreadPool(n, max_task_num=1)
while True:
if not tp.q.empty():
continue
state = self._loadState()
leave_list = list(range(183))
leave_list.reverse()
if state:
goods_down = state.get('goods_down', [])
leave_list = list(set(leave_list) - set(goods_down))
if not leave_list:
print("All Things is Done！！！")
break
j = 0
for i in leave_list:
print(i)
tp.put(self.cateGoods, (i, ))
if j == 0:
time.sleep(60)
j = 1
time.sleep(3)
def run(self):
pool = ThreadPool(10)
while self.running:
requests = makeRequests(some_callable, list_of_args)
[pool.putRequest(req) for req in requests]
pool.wait()
sleep(1)
print "ActionHandlerThread"
def cateGoodsControl(self, n):
# 恢复爬取状态
tp = ThreadPool(n)
while True:
state = self._loadState()
leave_list = list(range(183))
if state:
goods_down = state.get('goods_down', [])
leave_list = list(set(leave_list) - set(goods_down))
j = 0
for i in leave_list:
print(i)
tp.put(self.cateGoods, (i, ))
if j == 0:
time.sleep(60)
j = 1
def process_user(uid):
auth_profile = AuthProfile.objects.filter(dropbox_userid=uid)[0]
token = auth_profile.dropbox_access_token
user = auth_profile.user
logger.info("START process user '%s'" % user)
logger.debug("Process change notfication for user: %s" % user.username)
cursor = cache.get("cursor-%s" % uid)
client = Connection(token)
has_more = True
while has_more:
result = client.delta(cursor)
pool = ThreadPool(1)
for path, metadata in result['entries']:
logger.info("Add task for %s to pool" % path)
pool.add_task(process_file, path, metadata, client, user)
logger.debug("Waiting for completion ... %s" % path)
pool.wait_completion()
logger.debug("Tasks completed.")
# Update cursor
cursor = result['cursor']
cursor = cache.set("cursor-%s" % uid, cursor)
# Repeat only if there's more to do
has_more = result['has_more']
logger.info("END process user '%s'" % user)
def run(self, session_id):
"""
Run the main function ...
"""
self.thread_count = self.config_result["content_thread"]
# debug
if self.verbose > 0:
print " [+] Thread count to run mount %s" % self.thread_count
pool = ThreadPool(int(self.thread_count))
if session_id == 0:
try:
read_file = open(self.share_session, "r").read().splitlines()
except Exception, err_mess:
print err_mess
sys.exit(1)
for line in read_file:
pool.add_task(self.mount_sharing, line)
pool.wait_completion()
self.write_session_id(str(session_id))
session_id = session_id + 1
print(ctime(), 'IDSET = ', len(IDSETs[0]), len(IDSETs[1]), len(IDSETs[2]))
print(ctime(), 'dbQueue = ', db_tp.q.qsize())
print(ctime(), 'parseQueue = ', parse_tp.q.qsize())
ins = input()
except:
traceback.print_exc()
def loadIdSet():
IDSET = dbo.getallidlist()
return IDSET
if __name__ == "__main__":
dbo = DbOperate()
IDSET = loadIdSet()
IDSETs = [set(IDSET.get('goods', ())), set(IDSET.get('opts', ())), set(IDSET.get('top_opts', ()))]
Dupli = 0
parse = Parse()
# 解析线程池
parse_tp = ThreadPool(2)
# 数据库操作线程池,
db_tp = ThreadPool(1)
threading.Thread(target=Monitor).start()
baseproxy = AsyncMitmProxy(server_addr=('', 8888), https=True)
baseproxy.register(RspInterceptor)
baseproxy.serve_forever()
#print "Thread count to run mount %s"% self.thread_count pool = ThreadPool(int(self.thread_count)) if session_id == 0: try: read_file = open(self.share_session, "r").read().splitlines() except Exception, err_mess: print err_mess sys.exit(1) for line in read_file: pool.add_task(self.mount_sharing, line) pool.wait_completion() self.write_session_id(str(session_id)) session_id = session_id + 1 else: # debug #print "Session files will be used ..." pool = ThreadPool(int(self.thread_count)) while self.get_session(session_id): line = self.get_session(session_id) pool.add_task(self.mount_sharing, line) self.write_session_id(str(session_id)) session_id = int(session_id) + 1 pool.wait_completion()
pool = ThreadPool(int(self.thread_count))
if session_id == 0:
try:
read_file = open(self.share_session, "r").read().splitlines()
except Exception, err_mess:
print err_mess
sys.exit(1)
for line in read_file:
pool.add_task(self.mount_sharing, line)
pool.wait_completion()
self.write_session_id(str(session_id))
session_id = session_id + 1
else:
# debug
if self.verbose > 0:
print " [+] Session files will be used ..."
pool = ThreadPool(int(self.thread_count))
while self.get_session(session_id):
line = self.get_session(session_id)
pool.add_task(self.mount_sharing, line)
self.write_session_id(str(session_id))
session_id = int(session_id) + 1
pool.wait_completion()
def __create_pool(self, poolnode):
"""\brief Creates a thread pool
\param poolnode (\c xml.dom.Node) The thread pool in xml form
"""
self.__thread_pools[poolnode.attributes['id'].value] = ThreadPool(poolnode,\
self.__blockmon)