def testProcessPage():
listCollectionName = "daily_dice_list_2014-07-11"
infoCollectionName = "daily_dice_info_2014-07-11"
dbClient = DbClient('localhost', 27017, "jobaly_daily")
listCollection = dbClient.getCollection(listCollectionName)
infoCollection = dbClient.getCollection(infoCollectionName)
getter = DicePageGetter(infoCollection)
pageSize = 100
pageNo = 1
has_more = True
pageNum = 10000
find_sort = None
find_spec = None
while has_more and pageNo <= pageNum:
page = dbClient.getPage(listCollection, find_spec, find_sort, pageSize,
pageNo)
getter.processPage(page, pageNo)
pageNo += 1
count = page.count(with_limit_and_skip=True)
# print "count=",count
if (count < pageSize):
has_more = False
class JobGetter(threading.Thread):
"""Threaded Url Grab"""
def __init__(self, queue, infoCollection):
threading.Thread.__init__(self)
self.queue = queue
self.getter = DicePageGetter(infoCollection)
def run(self):
while True:
#grabs host from queue
page, pageNo = self.queue.get()
self.getter.processPage(page, pageNo)
self.queue.task_done()
class JobGetter(threading.Thread):
"""Threaded Url Grab"""
def __init__(self, queue, infoCollection):
threading.Thread.__init__(self)
self.queue = queue
self.getter = DicePageGetter(infoCollection)
def run(self):
while True:
#grabs host from queue
page,pageNo = self.queue.get()
self.getter.processPage(page,pageNo)
self.queue.task_done()
def testProcessPage():
listCollectionName = "daily_dice_list_2014-07-11"
infoCollectionName = "daily_dice_info_2014-07-11"
dbClient = DbClient('localhost', 27017, "jobaly_daily")
listCollection = dbClient.getCollection(listCollectionName)
infoCollection = dbClient.getCollection(infoCollectionName)
getter = DicePageGetter(infoCollection)
pageSize = 100
pageNo = 1
has_more = True
pageNum = 10000
find_sort = None
find_spec=None
while has_more and pageNo <= pageNum :
page = dbClient.getPage(listCollection, find_spec,find_sort, pageSize, pageNo)
getter.processPage(page,pageNo)
pageNo+=1
count = page.count(with_limit_and_skip = True)
# print "count=",count
if ( count < pageSize ) :
has_more = False
def __init__(self, queue, infoCollection):
threading.Thread.__init__(self)
self.queue = queue
self.getter = DicePageGetter(infoCollection)
def __init__(self, queue, infoCollection):
threading.Thread.__init__(self)
self.queue = queue
self.getter = DicePageGetter(infoCollection)
