def _put(self, item): if self.unique: if item not in self.all_items: WSJob._put(self, item) self.all_items.add(item) else: _str = "WARNING: try to add not unique item `{0}`".format(item) if Registry().isset('logger'): #Registry().get('logger').log(_str) pass else: #print _str pass else: WSJob._put(self, item)
def get(self, block=False, timeout=None): """ Get next item from queue """ if self.empty() or self.qsize() < 50: self.load_data() if self.empty(): raise Queue.Empty return WSJob.get(self, block, timeout)
def get(self, block=True, timeout=None): """ Get next element from queue """ if self.empty(): while self.get_blocked: time.sleep(1) self.get_blocked = True self._fill_queue() self.get_blocked = False if self.empty(): raise Queue.Empty return WSJob.get(self, block, timeout)
def task_done(self, name): """ Mark current row as done """ self.counter += 1 self.collection.update({'name': str(unicode(name)), "getted": 1}, {"$set": {"checked": 1}}) WSJob.task_done(self)
def __init__(self, maxsize=0): WSJob.__init__(self, maxsize) self.collection = Registry().get('mongo')[self.collection_name]
def _init(self, maxsize): WSJob._init(self, maxsize) if self.unique: self.all_items = set()
def __init__(self, maxsize=0): WSJob.__init__(self, maxsize) self._db = Registry().get('mongo')