def open(self, access=READ): if access == 0: return self if not self.conn: self.conn = KafkaClient(hosts=self.hosts) self.topic = self.conn.topics[self.name] if access & READ: if not self.consumer: if self.group: self.consumer = self.topic.get_balanced_consumer( consumer_group=self.group, auto_commit_enable=True, reset_offset_on_start=True, auto_offset_reset=OffsetType.LATEST, zookeeper_connect=self.zookeepers) else: self.consumer = self.topic.get_simple_consumer( auto_commit_enable=True, reset_offset_on_start=True, auto_offset_reset=OffsetType.LATEST) func = import_function( self.entry) # importing functions dynamically self.func_list[self.entry] = func self.thread = Thread(target=self._messageRecieving) self.thread.start() if access & WRITE: if not self.producer: self.producer = self.topic.get_producer(delivery_reports=False) return self
def open(self,access=AccessMode.READ): if access == 0: return self broker = "%s:%s" % (self.host, self.port) # if self.conn is not None: # return self if not self.conn: self.conn = Connection(broker, reconnect=True, tcp_nodelay=True) self.conn.open() self.ssn = self.conn.session() if access & AccessMode.READ: if not self.consumer: self.consumer = self.ssn.receiver(self.address) self.consumer.capacity = 4000 func = import_function(self.entry) # importing functions dynamically self.func_list[self.entry] = func self.thread = Thread(target=self._messageRecieving) self.thread.start() if access & AccessMode.WRITE: if not self.producer: self.producer = self.ssn.sender(self.address) return self
def __init__(self, cfgs): task = import_function(cfgs.get('entry')) self.task_entry = task self.cfgs = cfgs self.interval = int(self.cfgs.get('interval', 10)) self.running = False self.tasklet = None
def init(self, cfgs): self.cfgs = cfgs for cfg in self.cfgs: if not cfg.get('enable', False): continue cls = import_function(cfg.get('module')) adapter = cls(cfg) res = adapter.open() if res: self.registerAdapter(adapter.name, adapter) else: instance.getLogger().error('adapter %s open failed!' % adapter.name)
def initMapGrids(self): region = map(float,self.cfgs.get('china_region').strip().split(',')) encode = self.cfgs.get('coordinate_encode','').strip() if encode: func = import_function(encode) region[0],region[1] = func(region[0],region[1]) cfgs = self.cfgs.get('map_grids',[]) for cfg in cfgs: name = cfg.get('name') size = map(float,cfg.get('cell_size').strip().split('x')) l1,l2 = map(int,cfg.get('scale_level').strip().split(',')) mapgrid = GlobalMap(name,size,region) self.map_grids[name] = mapgrid for level in range(l1,l2+1): self.scale_levels[level] = mapgrid