def initialize(self,storm_conf, context): try: settings = all_settings.get_settings(storm_conf['topology.deployment']) self.conf = settings self.connector = factory.getEntityDataConnector(self.conf) except: self.log(traceback.format_exc(),level='error') raise
def initialize(self, storm_conf, context): try: self.log("WriterBolt INIT") settings = all_settings.get_settings( storm_conf['topology.deployment']) self.conf = settings self.connector = factory.getEntityDataConnector(self.conf) except: self.log("WriterBolt initialize error", level='error') self.log(traceback.format_exc(), level='error') raise
def initialize(self, stormconf, context): try: settings = all_settings.get_settings(stormconf['topology.deployment']) self.topic = settings['crawler-out-topic'].encode() self.conn_pool = settings['conn_pool'].encode() self.log('KafkaDatawakeLookaheadSpout initialized with topic =' + self.topic + ' conn_pool=' + self.conn_pool) self.kafka = KafkaClient(self.conn_pool) self.consumer = SimpleConsumer(self.kafka, self.group, self.topic, max_buffer_size=None) self.consumer.seek(0, 2) # move to the tail of the queue except: self.log("KafkaDatawakeLookaheadSpout initialize error", level='error') self.log(traceback.format_exc(), level='error') raise
def initialize(self,storm_conf, context): try: self.log("CrawlerBolt INIT") settings = all_settings.get_settings(storm_conf['topology.deployment']) self.topic = settings['crawler-out-topic'].encode() self.conn_pool = settings['conn_pool'].encode() self.log('CrawlerQueueWriter initialized with topic ='+self.topic+' conn_pool='+self.conn_pool) self.kafka = KafkaClient(self.conn_pool) self.producer = SimpleProducer(self.kafka, async=False) except: self.log("CrawlerBolt initialize error",level='error') self.log(traceback.format_exc(),level='error') raise
def initialize(self, storm_conf, context): try: self.log("CrawlerBolt INIT") settings = all_settings.get_settings(storm_conf['topology.deployment']) self.topic = settings['crawler-out-topic'].encode() self.conn_pool = settings['conn_pool'].encode() self.log('CrawlerQueueWriter initialized with topic =' + self.topic + ' conn_pool=' + self.conn_pool) self.kafka = KafkaClient(self.conn_pool) self.producer = SimpleProducer(self.kafka, async=False) except: self.log("CrawlerBolt initialize error", level='error') self.log(traceback.format_exc(), level='error') raise
def initialize(self, stormconf, context): try: settings = all_settings.get_settings(stormconf['topology.deployment']) self.topic = settings['crawler-in-topic'].encode() self.conn_pool = settings['conn_pool'].encode() self.log('CrawlerSpout initialized with topic ='+self.topic+' conn_pool='+self.conn_pool) self.kafka = KafkaClient(self.conn_pool) self.kafka.ensure_topic_exists(self.topic) self.consumer = SimpleConsumer(self.kafka,self.group,self.topic,max_buffer_size=None, fetch_size_bytes=2000000) self.consumer.seek(0,2) # move to the tail of the queue except: self.log("CrawlerSpout initialize error",level='error') self.log(traceback.format_exc(),level='error') raise