def __init__(self): ProcesserBase.__init__(self) self.db = pymongo.Connection("192.168.2.229", 2291).dcrawler_final self.solr = pysolr.Solr('http://192.168.2.233:1984/solr/', timeout=10) self.helper = PushIndexHelper() self.ipusher = pusher()
def __init__(self): ProcesserBase.__init__(self) self.db = pymongo.Connection("192.168.2.229", 2291).dcrawler_final self.solr = pysolr.Solr('http://192.168.2.233:1984/solr/', timeout=10) self.helper = PushIndexHelper() self.ipusher = pusher()
class Processer(ProcesserBase): seq = "p1000" def __init__(self): ProcesserBase.__init__(self) self.db = pymongo.Connection("192.168.2.229", 2291).dcrawler_final self.solr = pysolr.Solr('http://192.168.2.233:1984/solr/', timeout=10) self.helper = PushIndexHelper() self.ipusher = pusher() def process(self, item): try: self.solr.add([self.helper.constructData(item), ]) except: traceback.print_exc() assert self.ipusher.push(item)[0] == "{success:'T'}" self.logger.info("item %s push ok." % item['url']) #self.rd.zadd("avurls:%s" % data['domain'], data['url'], time.time()) is_exists = self.db.car_info.find_one({'url': item['url']}) if is_exists: self.logger.debug('(%s) old item append %s (%s) to queues.' % (item['domain'], item['url'], item['id'])) else: self.logger.debug('(%s) new item append %s (%s) to queues.' % (item['domain'], item['url'], item['id']))
class Processer(ProcesserBase): seq = "p1000" def __init__(self): ProcesserBase.__init__(self) self.db = pymongo.Connection("192.168.2.229", 2291).dcrawler_final self.solr = pysolr.Solr('http://192.168.2.233:1984/solr/', timeout=10) self.helper = PushIndexHelper() self.ipusher = pusher() def process(self, item): try: self.solr.add([ self.helper.constructData(item), ]) except: traceback.print_exc() assert self.ipusher.push(item)[0] == "{success:'T'}" self.logger.info("item %s push ok." % item['url']) #self.rd.zadd("avurls:%s" % data['domain'], data['url'], time.time()) is_exists = self.db.car_info.find_one({'url': item['url']}) if is_exists: self.logger.debug('(%s) old item append %s (%s) to queues.' % (item['domain'], item['url'], item['id'])) else: self.logger.debug('(%s) new item append %s (%s) to queues.' % (item['domain'], item['url'], item['id']))
def push(self, obj): oitem = PushIndexHelper().constructData(obj) return [ ins.saveDoc(json.dumps(oitem)) for ins in self.indexServiceClients ]