class SqliteSpiderQueue(object): def __init__(self, database=None, table='spider_queue'): self.q = JsonSqlitePriorityQueue(database, table) def add(self, name, **spider_args): d = spider_args.copy() d['name'] = name priority = float(d.pop('priority', 0)) self.q.put(d, priority) def pop(self): return self.q.pop() def count(self): return len(self.q) def list(self): return [x[0] for x in self.q] def remove(self, func): return self.q.remove(func) def clear(self): self.q.clear()
class SqliteSpiderQueue(object): implements(ISpiderQueue) def __init__(self, database=None, table='spider_queue'): self.q = JsonSqlitePriorityQueue(database, table) def add(self, name, **spider_args): d = spider_args.copy() d['name'] = name priority = float(d.pop('priority', 0)) self.q.put(d, priority) def pop(self): return self.q.pop() def count(self): return len(self.q) def list(self): return [x[0] for x in self.q] def remove(self, func): return self.q.remove(func) def clear(self): self.q.clear()
class SqliteSpiderQueue(object): def __init__(self, database=None, table='spider_queue'): self.q = JsonSqlitePriorityQueue(database, table) self.q2 = [] def add(self, name, priority=0.0, **spider_args): # self.q2.append() logging.debug("#####################") logging.debug("#####################") logging.debug("#####################") logging.debug("#####################") logging.debug("#####################") logging.debug("#####################") logging.debug("#####################") logging.debug("#####################") d = spider_args.copy() d['name'] = name # self.q.put(d, priority=priority) self.q2.append(d) logging.debug(len(self.q)) logging.debug("//#################") logging.debug("//#################") logging.debug("//#################") def pop(self): return self.q2.pop() ret = self.q.pop() logging.debug("//#################") logging.debug(ret) logging.debug("//#################") return ret def count(self): return len(self.q2) def list(self): return [x for x in self.q2] def remove(self, func): return 1 return self.q.remove(func) def clear(self): self.q2 = [] self.q.clear()