def initialize(self,storm_conf, context):
try:
settings = all_settings.get_settings(storm_conf['topology.deployment'])
self.conf = settings
self.connector = factory.getEntityDataConnector(self.conf)
except:
self.log(traceback.format_exc(),level='error')
raise
def initialize(self, storm_conf, context):
try:
self.log("WriterBolt INIT")
settings = all_settings.get_settings(
storm_conf['topology.deployment'])
self.conf = settings
self.connector = factory.getEntityDataConnector(self.conf)
except:
self.log("WriterBolt initialize error", level='error')
self.log(traceback.format_exc(), level='error')
raise
def initialize(self, stormconf, context):
try:
settings = all_settings.get_settings(stormconf['topology.deployment'])
self.topic = settings['crawler-out-topic'].encode()
self.conn_pool = settings['conn_pool'].encode()
self.log('KafkaDatawakeLookaheadSpout initialized with topic =' + self.topic + ' conn_pool=' + self.conn_pool)
self.kafka = KafkaClient(self.conn_pool)
self.consumer = SimpleConsumer(self.kafka, self.group, self.topic, max_buffer_size=None)
self.consumer.seek(0, 2) # move to the tail of the queue
except:
self.log("KafkaDatawakeLookaheadSpout initialize error", level='error')
self.log(traceback.format_exc(), level='error')
raise
def initialize(self,storm_conf, context):
try:
self.log("CrawlerBolt INIT")
settings = all_settings.get_settings(storm_conf['topology.deployment'])
self.topic = settings['crawler-out-topic'].encode()
self.conn_pool = settings['conn_pool'].encode()
self.log('CrawlerQueueWriter initialized with topic ='+self.topic+' conn_pool='+self.conn_pool)
self.kafka = KafkaClient(self.conn_pool)
self.producer = SimpleProducer(self.kafka, async=False)
except:
self.log("CrawlerBolt initialize error",level='error')
self.log(traceback.format_exc(),level='error')
raise
def initialize(self, storm_conf, context):
try:
self.log("CrawlerBolt INIT")
settings = all_settings.get_settings(storm_conf['topology.deployment'])
self.topic = settings['crawler-out-topic'].encode()
self.conn_pool = settings['conn_pool'].encode()
self.log('CrawlerQueueWriter initialized with topic =' + self.topic + ' conn_pool=' + self.conn_pool)
self.kafka = KafkaClient(self.conn_pool)
self.producer = SimpleProducer(self.kafka, async=False)
except:
self.log("CrawlerBolt initialize error", level='error')
self.log(traceback.format_exc(), level='error')
raise
def initialize(self, stormconf, context):
try:
settings = all_settings.get_settings(stormconf['topology.deployment'])
self.topic = settings['crawler-in-topic'].encode()
self.conn_pool = settings['conn_pool'].encode()
self.log('CrawlerSpout initialized with topic ='+self.topic+' conn_pool='+self.conn_pool)
self.kafka = KafkaClient(self.conn_pool)
self.kafka.ensure_topic_exists(self.topic)
self.consumer = SimpleConsumer(self.kafka,self.group,self.topic,max_buffer_size=None, fetch_size_bytes=2000000)
self.consumer.seek(0,2) # move to the tail of the queue
except:
self.log("CrawlerSpout initialize error",level='error')
self.log(traceback.format_exc(),level='error')
raise
