def open(self, access=READ):
if access == 0:
return self
if not self.conn:
self.conn = KafkaClient(hosts=self.hosts)
self.topic = self.conn.topics[self.name]
if access & READ:
if not self.consumer:
if self.group:
self.consumer = self.topic.get_balanced_consumer(
consumer_group=self.group,
auto_commit_enable=True,
reset_offset_on_start=True,
auto_offset_reset=OffsetType.LATEST,
zookeeper_connect=self.zookeepers)
else:
self.consumer = self.topic.get_simple_consumer(
auto_commit_enable=True,
reset_offset_on_start=True,
auto_offset_reset=OffsetType.LATEST)
func = import_function(
self.entry) # importing functions dynamically
self.func_list[self.entry] = func
self.thread = Thread(target=self._messageRecieving)
self.thread.start()
if access & WRITE:
if not self.producer:
self.producer = self.topic.get_producer(delivery_reports=False)
return self
def open(self,access=AccessMode.READ):
if access == 0:
return self
broker = "%s:%s" % (self.host, self.port)
# if self.conn is not None:
# return self
if not self.conn:
self.conn = Connection(broker, reconnect=True, tcp_nodelay=True)
self.conn.open()
self.ssn = self.conn.session()
if access & AccessMode.READ:
if not self.consumer:
self.consumer = self.ssn.receiver(self.address)
self.consumer.capacity = 4000
func = import_function(self.entry) # importing functions dynamically
self.func_list[self.entry] = func
self.thread = Thread(target=self._messageRecieving)
self.thread.start()
if access & AccessMode.WRITE:
if not self.producer:
self.producer = self.ssn.sender(self.address)
return self
def __init__(self, cfgs):
task = import_function(cfgs.get('entry'))
self.task_entry = task
self.cfgs = cfgs
self.interval = int(self.cfgs.get('interval', 10))
self.running = False
self.tasklet = None
def init(self, cfgs):
self.cfgs = cfgs
for cfg in self.cfgs:
if not cfg.get('enable', False):
continue
cls = import_function(cfg.get('module'))
adapter = cls(cfg)
res = adapter.open()
if res:
self.registerAdapter(adapter.name, adapter)
else:
instance.getLogger().error('adapter %s open failed!' %
adapter.name)
def initMapGrids(self):
region = map(float,self.cfgs.get('china_region').strip().split(','))
encode = self.cfgs.get('coordinate_encode','').strip()
if encode:
func = import_function(encode)
region[0],region[1] = func(region[0],region[1])
cfgs = self.cfgs.get('map_grids',[])
for cfg in cfgs:
name = cfg.get('name')
size = map(float,cfg.get('cell_size').strip().split('x'))
l1,l2 = map(int,cfg.get('scale_level').strip().split(','))
mapgrid = GlobalMap(name,size,region)
self.map_grids[name] = mapgrid
for level in range(l1,l2+1):
self.scale_levels[level] = mapgrid