Пример #1
0
def test_heartbeat_timeout(conn, mocker):
    mocker.patch('kafka.client_async.KafkaClient.check_version',
                 return_value=(0, 9))
    mocker.patch('time.time', return_value=1234)
    consumer = KafkaConsumer('foobar')
    mocker.patch.object(consumer._coordinator.heartbeat, 'ttl', return_value=0)
    assert consumer._next_timeout() == 1234
Пример #2
0
 def __init__(self, threadId, name, sKey, bootstrapServers, groupId, permissions, token, cipher = None, uid = None):
     threading.Thread.__init__(self)
     self.threadId = threadId
     self.name = name
     
     self.group = groupId;
     self.subKey = sKey;
     
     self.cipher = None if cipher is None else cipher;
             
     configs = Configs.consumerConfigs();
     configs["bootstrap_servers"] = bootstrapServers.split(',');
     configs["group_id"] = groupId;
     configs['enable_auto_commit'] = False;
     
     self.__isAlive = True
     
     home = expanduser("~")
     
     if uid == None:
         self.__consumer = KafkaConsumer(
             bootstrap_servers = configs["bootstrap_servers"],
             check_crcs = False,
             exclude_internal_topics = True,
             session_timeout_ms = 10000,
             reconnect_backoff_ms = 10000,
             heartbeat_interval_ms = 2000,
             retry_backoff_ms = 500,
             fetch_min_bytes = 64,
             fetch_max_wait_ms = 96,           
             enable_auto_commit = False,
             max_in_flight_requests_per_connection = 4,
             api_version = (0, 10),            
             group_id = groupId);
     else:
         self.__consumer = KafkaConsumer(
             bootstrap_servers = configs["bootstrap_servers"],
             check_crcs = False,
             exclude_internal_topics = True,
             session_timeout_ms = 10000,
             reconnect_backoff_ms = 10000,
             heartbeat_interval_ms = 2000,
             retry_backoff_ms = 500,
             fetch_min_bytes = 64,
             fetch_max_wait_ms = 96,           
             enable_auto_commit = False,
             max_in_flight_requests_per_connection = 4,
             security_protocol = 'SSL',
             ssl_check_hostname = False,
             ssl_keyfile = home + '/magistral/' + token + '/key.pem',
             ssl_cafile = home + '/magistral/' + token + '/ca.pem',
             ssl_certfile = home + '/magistral/' + token + '/certificate.pem',
             api_version = (0, 10),
             group_id = groupId);
     
     self.permissions = permissions;
     self.map = {}
     
     self.__offsets = {}
Пример #3
0
 def __init__(self, threadId, name, sKey, bootstrapServers, groupId, permissions, token, cipher = None, uid = None):
     threading.Thread.__init__(self)
     self.threadId = threadId
     self.name = name
     
     self.group = groupId;
     self.subKey = sKey;
     
     self.cipher = None if cipher is None else cipher;
             
     configs = Configs.consumerConfigs();
     configs["bootstrap_servers"] = bootstrapServers.split(',');
     configs["group_id"] = groupId;
     configs['enable_auto_commit'] = False;
     
     self.__isAlive = True
     
     home = expanduser("~")
     
     if uid == None:
         self.__consumer = KafkaConsumer(
             bootstrap_servers = configs["bootstrap_servers"],
             check_crcs = False,
             exclude_internal_topics = True,
             session_timeout_ms = 10000,
             reconnect_backoff_ms = 10000,
             heartbeat_interval_ms = 2000,
             retry_backoff_ms = 500,
             fetch_min_bytes = 64,
             fetch_max_wait_ms = 96,           
             enable_auto_commit = False,
             max_in_flight_requests_per_connection = 4,
             api_version = (0, 10),            
             group_id = groupId);
     else:
         self.__consumer = KafkaConsumer(
             bootstrap_servers = configs["bootstrap_servers"],
             check_crcs = False,
             exclude_internal_topics = True,
             session_timeout_ms = 10000,
             reconnect_backoff_ms = 10000,
             heartbeat_interval_ms = 2000,
             retry_backoff_ms = 500,
             fetch_min_bytes = 64,
             fetch_max_wait_ms = 96,           
             enable_auto_commit = False,
             max_in_flight_requests_per_connection = 4,
             security_protocol = 'SSL',
             ssl_check_hostname = False,
             ssl_keyfile = home + '/magistral/' + token + '/key.pem',
             ssl_cafile = home + '/magistral/' + token + '/ca.pem',
             ssl_certfile = home + '/magistral/' + token + '/certificate.pem',
             api_version = (0, 10),
             group_id = groupId);
     
     self.permissions = permissions;
     self.map = {}
     
     self.__offsets = {}
Пример #4
0
def test_consumer(kafka_broker, topic, version):
    # The `topic` fixture is included because
    # 0.8.2 brokers need a topic to function well
    consumer = KafkaConsumer(bootstrap_servers=get_connect_str(kafka_broker))
    consumer.poll(500)
    assert len(consumer._client._conns) > 0
    node_id = list(consumer._client._conns.keys())[0]
    assert consumer._client._conns[node_id].state is ConnectionStates.CONNECTED
    consumer.close()
Пример #5
0
def test_heartbeat_thread(kafka_broker, topic):
    group_id = 'test-group-' + random_string(6)
    consumer = KafkaConsumer(topic,
                             bootstrap_servers=get_connect_str(kafka_broker),
                             group_id=group_id,
                             heartbeat_interval_ms=500)

    # poll until we have joined group / have assignment
    while not consumer.assignment():
        consumer.poll(timeout_ms=100)

    assert consumer._coordinator.state is MemberState.STABLE
    last_poll = consumer._coordinator.heartbeat.last_poll
    last_beat = consumer._coordinator.heartbeat.last_send

    timeout = time.time() + 30
    while True:
        if time.time() > timeout:
            raise RuntimeError('timeout waiting for heartbeat')
        if consumer._coordinator.heartbeat.last_send > last_beat:
            break
        time.sleep(0.5)

    assert consumer._coordinator.heartbeat.last_poll == last_poll
    consumer.poll(timeout_ms=100)
    assert consumer._coordinator.heartbeat.last_poll > last_poll
    consumer.close()
Пример #6
0
def test_consumer(kafka_broker, version):

    # 0.8.2 brokers need a topic to function well
    if version >= (0, 8, 2) and version < (0, 9):
        topic(simple_client(kafka_broker))

    consumer = KafkaConsumer(bootstrap_servers=get_connect_str(kafka_broker))
    consumer.poll(500)
    assert len(consumer._client._conns) > 0
    node_id = list(consumer._client._conns.keys())[0]
    assert consumer._client._conns[node_id].state is ConnectionStates.CONNECTED
Пример #7
0
def test_consumer(kafka_broker, version):

    # 0.8.2 brokers need a topic to function well
    if version >= (0, 8, 2) and version < (0, 9):
        topic(simple_client(kafka_broker))

    consumer = KafkaConsumer(bootstrap_servers=get_connect_str(kafka_broker))
    consumer.poll(500)
    assert len(consumer._client._conns) > 0
    node_id = list(consumer._client._conns.keys())[0]
    assert consumer._client._conns[node_id].state is ConnectionStates.CONNECTED
Пример #8
0
def test_consumer_topics(kafka_broker, topic, version):
    consumer = KafkaConsumer(bootstrap_servers=get_connect_str(kafka_broker))
    # Necessary to drive the IO
    consumer.poll(500)
    assert topic in consumer.topics()
    assert len(consumer.partitions_for_topic(topic)) > 0
    consumer.close()
Пример #9
0
def test_heartbeat_thread(kafka_broker, topic):
    group_id = 'test-group-' + random_string(6)
    consumer = KafkaConsumer(topic,
                             bootstrap_servers=get_connect_str(kafka_broker),
                             group_id=group_id,
                             heartbeat_interval_ms=500)

    # poll until we have joined group / have assignment
    while not consumer.assignment():
        consumer.poll(timeout_ms=100)

    assert consumer._coordinator.state is MemberState.STABLE
    last_poll = consumer._coordinator.heartbeat.last_poll
    last_beat = consumer._coordinator.heartbeat.last_send

    timeout = time.time() + 30
    while True:
        if time.time() > timeout:
            raise RuntimeError('timeout waiting for heartbeat')
        if consumer._coordinator.heartbeat.last_send > last_beat:
            break
        time.sleep(0.5)

    assert consumer._coordinator.heartbeat.last_poll == last_poll
    consumer.poll(timeout_ms=100)
    assert consumer._coordinator.heartbeat.last_poll > last_poll
    consumer.close()
    def __init__(self):
        self.KAFKA_HOST_NAME = os.environ.get('KAFKA_HOST_NAME')
        self.KAFKA_PORT = os.environ.get('KAFKA_PORT')
        self.KAFKA_TOPIC_NAME = os.environ.get('KAFKA_TOPIC_NAME')

        self.kafka_twitter_consumer = KafkaConsumer(
            self.KAFKA_TOPIC_NAME,
            bootstrap_servers=f'{self.KAFKA_HOST_NAME}:{self.KAFKA_PORT}')
Пример #11
0
def fetch_alert(request):
    consumer = KafkaConsumer(bootstrap_servers='en1:9092,en2:9092,en3:9092',
                             auto_offset_reset='earliest')
    consumer.subscribe(['ApacheLogAlert'])

    time = int(request.GET.get('time', '0'))
    test = consumer.poll(10000)
    result = []
    for aa in test:
        cc = test.get(aa)
        for msg in cc:
            print msg

            try:
                int(msg.value) + 2
                int(msg.key) + 2
            except:
                continue

            if int(msg.key) <= time or int(msg.value) <= 0:
                continue
            t = datetime.datetime.fromtimestamp(
                int(msg.key) / 1e3).strftime('%Y-%m-%d %H:%M:%S')

            c = msg.value
            tmp = {t: c}
            result.append(tmp)
    consumer.close()
    return render('fetch_alert.mako', request, dict(data=result))
Пример #12
0
def fetch_timeline(request):
    consumer = KafkaConsumer('ApacheLogAlert',
                             bootstrap_servers='en1:9092,en2:9092,en3:9092',
                             enable_auto_commit=True,
                             auto_offset_reset='earliest')

    time = int(request.GET.get('time', 0))
    threshold = int(request.GET.get('threshold', -1))

    test = consumer.poll(5000)
    i = 0
    temp = {}
    result = []
    threshold_list = []
    for topic in test:
        cc = test.get(topic)
        for msg in cc:
            print msg
            x = msg.key.split(' ')[0]
            y = msg.value.split(',')[2]
            key = msg.value.split(',')[1]

            if int(x) > time and threshold > -1 and int(y) >= threshold:
                print x, time, y, threshold
                threshold_list.append({
                    'x': int(x),
                    'y': int(y),
                    'key': int(key)
                })

            if key in temp:
                temp[key].append({'x': int(x), 'y': int(y)})
            else:
                temp[key] = []
                temp[key].append({'x': int(x), 'y': int(y)})

    for key in temp:
        result.append({'name': key, 'data': temp.get(key)})

    for key in result:
        key['data'] = sorted(key['data'], key=lambda a: a['x'])

    consumer.close()
    return render('fetch_timeline.mako', request,
                  dict(data=result, threshold=threshold_list))
Пример #13
0
 def consumer_thread(i):
     assert i not in consumers
     assert i not in stop
     stop[i] = threading.Event()
     consumers[i] = KafkaConsumer(topic,
                                  bootstrap_servers=connect_str,
                                  heartbeat_interval_ms=500)
     while not stop[i].is_set():
         for tp, records in six.itervalues(consumers[i].poll()):
             messages[i][tp].extend(records)
     consumers[i].close()
     del consumers[i]
     del stop[i]
Пример #14
0
def test_consumer(kafka_broker, topic, version):
    # The `topic` fixture is included because
    # 0.8.2 brokers need a topic to function well
    consumer = KafkaConsumer(bootstrap_servers=get_connect_str(kafka_broker))
    consumer.poll(500)
    assert len(consumer._client._conns) > 0
    node_id = list(consumer._client._conns.keys())[0]
    assert consumer._client._conns[node_id].state is ConnectionStates.CONNECTED
    consumer.close()
    def historyForTimePeriod(self, topic, channel, start, end, limit = -1):
        
        out = []        
        
        try:
            kfkTopic = self.__subKey + "." + topic;
            x = TopicPartition(kfkTopic, channel);
            
            if self.uid == None:
                self.consumer = KafkaConsumer(bootstrap_servers = self.__bootstrap,
                        check_crcs = False,
                        exclude_internal_topics = True,
                        session_timeout_ms = 30000,
                        reconnect_backoff_ms = 10000,
                        heartbeat_interval_ms = 2000,
                        retry_backoff_ms = 500,
                        fetch_min_bytes = 32,
                        fetch_max_wait_ms = 96,           
                        enable_auto_commit = False,
                        max_partition_fetch_bytes = 65536,
                        max_in_flight_requests_per_connection = 4,
                        api_version = (0, 10));
            else:            
                home = expanduser("~")
                
                self.consumer = KafkaConsumer(bootstrap_servers = self.__bootstrap,
                        check_crcs = False,
                        exclude_internal_topics = True,
                        session_timeout_ms = 30000,
                        reconnect_backoff_ms = 10000,
                        heartbeat_interval_ms = 2000,
                        retry_backoff_ms = 500,
                        fetch_min_bytes = 32,
                        fetch_max_wait_ms = 96,           
                        enable_auto_commit = False,
                        max_partition_fetch_bytes = 65536,
                        max_in_flight_requests_per_connection = 4,
                        security_protocol = 'SSL',
                        ssl_check_hostname = False,
                        ssl_keyfile = home + '/magistral/' + self.__token + '/key.pem',
                        ssl_cafile = home + '/magistral/' + self.__token + '/ca.pem',
                        ssl_certfile = home + '/magistral/' + self.__token + '/certificate.pem',
                        api_version = (0, 10));
                        
            self.consumer = KafkaConsumer(bootstrap_servers = self.__bootstrap);
            self.consumer.assign([x]);
            
            self.consumer.seek_to_end();        
            last = self.consumer.position(x);
        
            position = last - 1000;
            
            found = False;
            while found == False:
                self.consumer.seek(x, position);
                data = self.consumer.poll(500);
                 
                if x not in data.keys() or len(data[x]) == 0: break;
                
                record = data[x][0];
                timestamp = record[3];
 
                if timestamp < start: 
                    found = True;
                    break;
                 
                position = position - 1000;
             
            self.consumer.close();
                       
            if self.uid == None:
                self.с = KafkaConsumer(bootstrap_servers = self.__bootstrap,
                        check_crcs = False,
                        exclude_internal_topics = True,
                        session_timeout_ms = 10000,
                        reconnect_backoff_ms = 10000,
                        heartbeat_interval_ms = 2000,
                        retry_backoff_ms = 500,
                        fetch_min_bytes = 32,
                        fetch_max_wait_ms = 96,           
                        enable_auto_commit = False,
                        max_partition_fetch_bytes = 65536,
                        max_in_flight_requests_per_connection = 4,
                        api_version = (0, 10));
            else:            
                home = expanduser("~")
                
                self.с = KafkaConsumer(bootstrap_servers = self.__bootstrap,
                        check_crcs = False,
                        exclude_internal_topics = True,
                        session_timeout_ms = 10000,
                        reconnect_backoff_ms = 10000,
                        heartbeat_interval_ms = 2000,
                        retry_backoff_ms = 500,
                        fetch_min_bytes = 32,
                        fetch_max_wait_ms = 96,           
                        enable_auto_commit = False,
                        max_partition_fetch_bytes = 65536,
                        max_in_flight_requests_per_connection = 4,
                        security_protocol = 'SSL',
                        ssl_check_hostname = False,
                        ssl_keyfile = home + '/magistral/' + self.__token + '/key.pem',
                        ssl_cafile = home + '/magistral/' + self.__token + '/ca.pem',
                        ssl_certfile = home + '/magistral/' + self.__token + '/certificate.pem',
                        api_version = (0, 10));  
                      
            self.с.assign([x]);
                       
            self.c.seek(x, position);                        
            data = self.с.poll(256);
            
            while (x in data.keys() and len(data[x]) > 0):
                
                for record in data[x] :
                    timestamp = record[3];
                    if timestamp < start: continue;
                    
                    index = record[2];
                    
                    if timestamp > end or index >= last - 1:  
                        self.с.close();                 
                        return out;
                                    
                    message = Message(record[0][41:], record[1], record[6], index, timestamp);
                    out.append(message); 
                    
                    if limit is not None and limit > 0 and len(out) >= limit:
                        self.с.close();                 
                        return out;                 
                    
                self.с.seek(x, position + len(data[x]));                        
                data = self.с.poll(256);
            
            return out;
        
        except:
            
            raise MagistralException("Exception during history invocation occurred");
Пример #16
0
def test_paused(kafka_broker, topic):
    consumer = KafkaConsumer(bootstrap_servers=get_connect_str(kafka_broker))
    topics = [TopicPartition(topic, 1)]
    consumer.assign(topics)
    assert set(topics) == consumer.assignment()
    assert set() == consumer.paused()

    consumer.pause(topics[0])
    assert set([topics[0]]) == consumer.paused()

    consumer.resume(topics[0])
    assert set() == consumer.paused()

    consumer.unsubscribe()
    assert set() == consumer.paused()
Пример #17
0
    def historyForTimePeriod(self, topic, channel, start, end, limit=-1):

        out = []

        try:
            kfkTopic = self.__subKey + "." + topic
            x = TopicPartition(kfkTopic, channel)

            if self.uid == None:
                self.consumer = KafkaConsumer(
                    bootstrap_servers=self.__bootstrap,
                    check_crcs=False,
                    exclude_internal_topics=True,
                    session_timeout_ms=30000,
                    reconnect_backoff_ms=10000,
                    heartbeat_interval_ms=2000,
                    retry_backoff_ms=500,
                    fetch_min_bytes=32,
                    fetch_max_wait_ms=96,
                    enable_auto_commit=False,
                    max_partition_fetch_bytes=65536,
                    max_in_flight_requests_per_connection=4,
                    api_version=(0, 10))
            else:
                home = expanduser("~")

                self.consumer = KafkaConsumer(
                    bootstrap_servers=self.__bootstrap,
                    check_crcs=False,
                    exclude_internal_topics=True,
                    session_timeout_ms=30000,
                    reconnect_backoff_ms=10000,
                    heartbeat_interval_ms=2000,
                    retry_backoff_ms=500,
                    fetch_min_bytes=32,
                    fetch_max_wait_ms=96,
                    enable_auto_commit=False,
                    max_partition_fetch_bytes=65536,
                    max_in_flight_requests_per_connection=4,
                    security_protocol='SSL',
                    ssl_check_hostname=False,
                    ssl_keyfile=home + '/magistral/' + self.__token +
                    '/key.pem',
                    ssl_cafile=home + '/magistral/' + self.__token + '/ca.pem',
                    ssl_certfile=home + '/magistral/' + self.__token +
                    '/certificate.pem',
                    api_version=(0, 10))

            self.consumer = KafkaConsumer(bootstrap_servers=self.__bootstrap)
            self.consumer.assign([x])

            self.consumer.seek_to_end()
            last = self.consumer.position(x)

            position = last - 1000

            found = False
            while found == False:
                self.consumer.seek(x, position)
                data = self.consumer.poll(500)

                if x not in data.keys() or len(data[x]) == 0: break

                record = data[x][0]
                timestamp = record[3]

                if timestamp < start:
                    found = True
                    break

                position = position - 1000

            self.consumer.close()

            if self.uid == None:
                self.с = KafkaConsumer(bootstrap_servers=self.__bootstrap,
                                       check_crcs=False,
                                       exclude_internal_topics=True,
                                       session_timeout_ms=10000,
                                       reconnect_backoff_ms=10000,
                                       heartbeat_interval_ms=2000,
                                       retry_backoff_ms=500,
                                       fetch_min_bytes=32,
                                       fetch_max_wait_ms=96,
                                       enable_auto_commit=False,
                                       max_partition_fetch_bytes=65536,
                                       max_in_flight_requests_per_connection=4,
                                       api_version=(0, 10))
            else:
                home = expanduser("~")

                self.с = KafkaConsumer(
                    bootstrap_servers=self.__bootstrap,
                    check_crcs=False,
                    exclude_internal_topics=True,
                    session_timeout_ms=10000,
                    reconnect_backoff_ms=10000,
                    heartbeat_interval_ms=2000,
                    retry_backoff_ms=500,
                    fetch_min_bytes=32,
                    fetch_max_wait_ms=96,
                    enable_auto_commit=False,
                    max_partition_fetch_bytes=65536,
                    max_in_flight_requests_per_connection=4,
                    security_protocol='SSL',
                    ssl_check_hostname=False,
                    ssl_keyfile=home + '/magistral/' + self.__token +
                    '/key.pem',
                    ssl_cafile=home + '/magistral/' + self.__token + '/ca.pem',
                    ssl_certfile=home + '/magistral/' + self.__token +
                    '/certificate.pem',
                    api_version=(0, 10))

            self.с.assign([x])

            self.c.seek(x, position)
            data = self.с.poll(256)

            while (x in data.keys() and len(data[x]) > 0):

                for record in data[x]:
                    timestamp = record[3]
                    if timestamp < start: continue

                    index = record[2]

                    if timestamp > end or index >= last - 1:
                        self.с.close()
                        return out

                    message = Message(record[0][41:], record[1], record[6],
                                      index, timestamp)
                    out.append(message)

                    if limit is not None and limit > 0 and len(out) >= limit:
                        self.с.close()
                        return out

                self.с.seek(x, position + len(data[x]))
                data = self.с.poll(256)

            return out

        except:

            raise MagistralException(
                "Exception during history invocation occurred")
Пример #18
0
class GroupConsumer(threading.Thread):
        
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.INFO)
     
    def __init__(self, threadId, name, sKey, bootstrapServers, groupId, permissions, token, cipher = None, uid = None):
        threading.Thread.__init__(self)
        self.threadId = threadId
        self.name = name
        
        self.group = groupId;
        self.subKey = sKey;
        
        self.cipher = None if cipher is None else cipher;
                
        configs = Configs.consumerConfigs();
        configs["bootstrap_servers"] = bootstrapServers.split(',');
        configs["group_id"] = groupId;
        configs['enable_auto_commit'] = False;
        
        self.__isAlive = True
        
        home = expanduser("~")
        
        if uid == None:
            self.__consumer = KafkaConsumer(
                bootstrap_servers = configs["bootstrap_servers"],
                check_crcs = False,
                exclude_internal_topics = True,
                session_timeout_ms = 10000,
                reconnect_backoff_ms = 10000,
                heartbeat_interval_ms = 2000,
                retry_backoff_ms = 500,
                fetch_min_bytes = 64,
                fetch_max_wait_ms = 96,           
                enable_auto_commit = False,
                max_in_flight_requests_per_connection = 4,
                api_version = (0, 10),            
                group_id = groupId);
        else:
            self.__consumer = KafkaConsumer(
                bootstrap_servers = configs["bootstrap_servers"],
                check_crcs = False,
                exclude_internal_topics = True,
                session_timeout_ms = 10000,
                reconnect_backoff_ms = 10000,
                heartbeat_interval_ms = 2000,
                retry_backoff_ms = 500,
                fetch_min_bytes = 64,
                fetch_max_wait_ms = 96,           
                enable_auto_commit = False,
                max_in_flight_requests_per_connection = 4,
                security_protocol = 'SSL',
                ssl_check_hostname = False,
                ssl_keyfile = home + '/magistral/' + token + '/key.pem',
                ssl_cafile = home + '/magistral/' + token + '/ca.pem',
                ssl_certfile = home + '/magistral/' + token + '/certificate.pem',
                api_version = (0, 10),
                group_id = groupId);
        
        self.permissions = permissions;
        self.map = {}
        
        self.__offsets = {}
        
        
    def recordsTotally(self, data):
        size = 0;
        for val in list(data.values()): 
            if len(val) > 0: size = size + len(val);
                               
        return size;
    
    def consumerRecord2Message(self, record):                    
        payload = record[6]
                                        
        if self.cipher is not None:
            try:
                payload = self.cipher.decrypt(payload)
            except:
                pass
                                        
        msg = Message(record[0][41:], record[1], payload, record[2], record[3])
        return msg

    def run(self):
        
        threadLock.acquire(False)
        
        while self.__isAlive:
            try:

                data = self.__consumer.poll(512);
                for values in list(data.values()):
                                         
                    for value in values:
                        msg = self.consumerRecord2Message(value);
                        listener = self.map[value[0]][msg.channel()];
                        if listener is not None: listener(msg);
                        
                if len(list(data.values())) > 0: self.__consumer.commit_async(); 
                    
            except:
                pass
              
        threadLock.release()

#   ////////////////////////////////////////////////////////////////////////////////////    

    def subscribe(self, topic, channel = -1, listener = None, callback = None):
        
        assert channel is not None and isinstance(channel, int), "Channel expected as int argument"        
        if (channel < -1): channel = -1;
                
        etopic = self.subKey + "." + topic;
                           
        self.logger.debug("Subscribe -> %s : %s | key = %s", topic, channel, self.subKey);
        
        if (self.permissions == None or len(self.permissions) == 0): 
            raise MagistralException("User has no permissions for topic [" + topic + "].");
        
        self.fch = [];
        
        for meta in self.permissions:             
            if (meta.topic() != topic): continue;  
            
            if channel == -1:
                self.fch = meta.channels();
            elif channel in meta.channels():                          
                self.fch = [ channel ];  
        
        if (len(self.fch) == 0): 
            npgex = "No permissions for topic [" + topic + "] granted";
            self.logger.error(npgex);                                
            raise MagistralException(npgex);
        
        if (self.map == None or etopic not in self.map): 
            self.map[etopic] = {}
        
#         // Assign Topic-partition pairs to listen
        
        tpas = [];
        for ch in self.fch:            
            tpas.append(TopicPartition(etopic, ch));
            if (listener is not None): self.map[etopic][ch] = listener
            
            ca = self.__consumer.assignment()
            if (ca is not None):
                for tp in ca: tpas.append(tp)
        
        self.__consumer.assign(tpas);       
                
        if callback is not None: 
            callback(self.__consumer.assignment());
            
        return self.__consumer.assignment();
        
        
    def unsubscribe(self, topic):
        self.consumer.assign([]);
        self.map.remove(topic);        

    def close(self):
        self.__isAlive = False
        self.__consumer.pause()
        self.__consumer.close()
        
    logging.getLogger('kafka.conn').setLevel(logging.FATAL)
    logging.getLogger('kafka.cluster').setLevel(logging.FATAL)
    logging.getLogger('kafka.consumer.group').setLevel(logging.INFO)    
    logging.getLogger('kafka.consumer.fetcher').setLevel(logging.INFO)
    logging.getLogger('kafka.coordinator.consumer').setLevel(logging.INFO)
    logging.getLogger('kafka.producer.record_accumulator').setLevel(logging.INFO)
Пример #19
0
class MagistralConsumer(object):

    __HISTORY_DATA_FETCH_SIZE_LIMIT = 10000

    def __init__(self,
                 pubKey,
                 subKey,
                 secretKey,
                 bootstrap,
                 token,
                 cipher=None,
                 uid=None):
        self.__pubKey = pubKey
        self.__subKey = subKey
        self.__secretKey = secretKey

        self.__token = token

        self.uid = uid

        self.__bootstrap = bootstrap.split(',')
        if cipher is not None: self.__cipher = cipher

    def history(self, topic, channel, records):

        messages = []

        if self.uid == None:
            self.consumer = KafkaConsumer(
                bootstrap_servers=self.__bootstrap,
                check_crcs=False,
                exclude_internal_topics=True,
                session_timeout_ms=10000,
                reconnect_backoff_ms=10000,
                heartbeat_interval_ms=2000,
                retry_backoff_ms=500,
                fetch_min_bytes=64,
                fetch_max_wait_ms=96,
                enable_auto_commit=False,
                max_in_flight_requests_per_connection=4,
                api_version=(0, 10))
        else:
            home = expanduser("~")

            self.consumer = KafkaConsumer(
                bootstrap_servers=self.__bootstrap,
                check_crcs=False,
                exclude_internal_topics=True,
                session_timeout_ms=10000,
                reconnect_backoff_ms=10000,
                heartbeat_interval_ms=2000,
                retry_backoff_ms=500,
                fetch_min_bytes=64,
                fetch_max_wait_ms=96,
                enable_auto_commit=False,
                max_in_flight_requests_per_connection=4,
                security_protocol='SSL',
                ssl_check_hostname=False,
                ssl_keyfile=home + '/magistral/' + self.__token + '/key.pem',
                ssl_cafile=home + '/magistral/' + self.__token + '/ca.pem',
                ssl_certfile=home + '/magistral/' + self.__token +
                '/certificate.pem',
                api_version=(0, 10))

        if (records > self.__HISTORY_DATA_FETCH_SIZE_LIMIT):
            records = self.__HISTORY_DATA_FETCH_SIZE_LIMIT

        kfkTopic = self.__subKey + "." + topic
        x = TopicPartition(kfkTopic, channel)

        self.consumer.assign([x])
        self.consumer.seek_to_end()
        last = self.consumer.position(x)

        pos = last - records if last > records else 0
        self.consumer.seek(x, pos)

        data = self.consumer.poll(256)

        endIsNotReached = True
        while endIsNotReached:

            if len(data.values()) == 0:
                return messages

            records = list(data.values())

            for record in records[0]:
                index = record[2]
                if index >= last - 1: endIsNotReached = False

                message = Message(record[0][41:], record[1], record[6], index,
                                  record[3])
                messages.append(message)

            if endIsNotReached == False:
                self.consumer.close()
                return messages

            pos = pos + len(messages)
            self.consumer.seek(x, pos)
            data = self.consumer.poll(256)

        self.consumer.close()

        return messages

    def historyForTimePeriod(self, topic, channel, start, end, limit=-1):

        out = []

        try:
            kfkTopic = self.__subKey + "." + topic
            x = TopicPartition(kfkTopic, channel)

            if self.uid == None:
                self.consumer = KafkaConsumer(
                    bootstrap_servers=self.__bootstrap,
                    check_crcs=False,
                    exclude_internal_topics=True,
                    session_timeout_ms=30000,
                    reconnect_backoff_ms=10000,
                    heartbeat_interval_ms=2000,
                    retry_backoff_ms=500,
                    fetch_min_bytes=32,
                    fetch_max_wait_ms=96,
                    enable_auto_commit=False,
                    max_partition_fetch_bytes=65536,
                    max_in_flight_requests_per_connection=4,
                    api_version=(0, 10))
            else:
                home = expanduser("~")

                self.consumer = KafkaConsumer(
                    bootstrap_servers=self.__bootstrap,
                    check_crcs=False,
                    exclude_internal_topics=True,
                    session_timeout_ms=30000,
                    reconnect_backoff_ms=10000,
                    heartbeat_interval_ms=2000,
                    retry_backoff_ms=500,
                    fetch_min_bytes=32,
                    fetch_max_wait_ms=96,
                    enable_auto_commit=False,
                    max_partition_fetch_bytes=65536,
                    max_in_flight_requests_per_connection=4,
                    security_protocol='SSL',
                    ssl_check_hostname=False,
                    ssl_keyfile=home + '/magistral/' + self.__token +
                    '/key.pem',
                    ssl_cafile=home + '/magistral/' + self.__token + '/ca.pem',
                    ssl_certfile=home + '/magistral/' + self.__token +
                    '/certificate.pem',
                    api_version=(0, 10))

            self.consumer = KafkaConsumer(bootstrap_servers=self.__bootstrap)
            self.consumer.assign([x])

            self.consumer.seek_to_end()
            last = self.consumer.position(x)

            position = last - 1000

            found = False
            while found == False:
                self.consumer.seek(x, position)
                data = self.consumer.poll(500)

                if x not in data.keys() or len(data[x]) == 0: break

                record = data[x][0]
                timestamp = record[3]

                if timestamp < start:
                    found = True
                    break

                position = position - 1000

            self.consumer.close()

            if self.uid == None:
                self.с = KafkaConsumer(bootstrap_servers=self.__bootstrap,
                                       check_crcs=False,
                                       exclude_internal_topics=True,
                                       session_timeout_ms=10000,
                                       reconnect_backoff_ms=10000,
                                       heartbeat_interval_ms=2000,
                                       retry_backoff_ms=500,
                                       fetch_min_bytes=32,
                                       fetch_max_wait_ms=96,
                                       enable_auto_commit=False,
                                       max_partition_fetch_bytes=65536,
                                       max_in_flight_requests_per_connection=4,
                                       api_version=(0, 10))
            else:
                home = expanduser("~")

                self.с = KafkaConsumer(
                    bootstrap_servers=self.__bootstrap,
                    check_crcs=False,
                    exclude_internal_topics=True,
                    session_timeout_ms=10000,
                    reconnect_backoff_ms=10000,
                    heartbeat_interval_ms=2000,
                    retry_backoff_ms=500,
                    fetch_min_bytes=32,
                    fetch_max_wait_ms=96,
                    enable_auto_commit=False,
                    max_partition_fetch_bytes=65536,
                    max_in_flight_requests_per_connection=4,
                    security_protocol='SSL',
                    ssl_check_hostname=False,
                    ssl_keyfile=home + '/magistral/' + self.__token +
                    '/key.pem',
                    ssl_cafile=home + '/magistral/' + self.__token + '/ca.pem',
                    ssl_certfile=home + '/magistral/' + self.__token +
                    '/certificate.pem',
                    api_version=(0, 10))

            self.с.assign([x])

            self.c.seek(x, position)
            data = self.с.poll(256)

            while (x in data.keys() and len(data[x]) > 0):

                for record in data[x]:
                    timestamp = record[3]
                    if timestamp < start: continue

                    index = record[2]

                    if timestamp > end or index >= last - 1:
                        self.с.close()
                        return out

                    message = Message(record[0][41:], record[1], record[6],
                                      index, timestamp)
                    out.append(message)

                    if limit is not None and limit > 0 and len(out) >= limit:
                        self.с.close()
                        return out

                self.с.seek(x, position + len(data[x]))
                data = self.с.poll(256)

            return out

        except:

            raise MagistralException(
                "Exception during history invocation occurred")
Пример #20
0
def test_heartbeat_timeout(conn, mocker):
    mocker.patch('kafka.client_async.KafkaClient.check_version', return_value = '0.9')
    mocker.patch('time.time', return_value = 1234)
    consumer = KafkaConsumer('foobar')
    mocker.patch.object(consumer._coordinator.heartbeat, 'ttl', return_value = 0)
    assert consumer._next_timeout() == 1234
 def history(self, topic, channel, records):
     
     messages = []
     
     if self.uid == None:
         self.consumer = KafkaConsumer(bootstrap_servers = self.__bootstrap,
                 check_crcs = False,
                 exclude_internal_topics = True,
                 session_timeout_ms = 10000,
                 reconnect_backoff_ms = 10000,
                 heartbeat_interval_ms = 2000,
                 retry_backoff_ms = 500,
                 fetch_min_bytes = 64,
                 fetch_max_wait_ms = 96,           
                 enable_auto_commit = False,
                 max_in_flight_requests_per_connection = 4,
                 api_version = (0, 10));
     else:            
         home = expanduser("~")
         
         self.consumer = KafkaConsumer(bootstrap_servers = self.__bootstrap,
                 check_crcs = False,
                 exclude_internal_topics = True,
                 session_timeout_ms = 10000,
                 reconnect_backoff_ms = 10000,
                 heartbeat_interval_ms = 2000,
                 retry_backoff_ms = 500,
                 fetch_min_bytes = 64,
                 fetch_max_wait_ms = 96,           
                 enable_auto_commit = False,
                 max_in_flight_requests_per_connection = 4,
                 security_protocol = 'SSL',
                 ssl_check_hostname = False,
                 ssl_keyfile = home + '/magistral/' + self.__token + '/key.pem',
                 ssl_cafile = home + '/magistral/' + self.__token + '/ca.pem',
                 ssl_certfile = home + '/magistral/' + self.__token + '/certificate.pem',
                 api_version = (0, 10));
         
     if (records > self.__HISTORY_DATA_FETCH_SIZE_LIMIT): records = self.__HISTORY_DATA_FETCH_SIZE_LIMIT;
         
     kfkTopic = self.__subKey + "." + topic;
     x = TopicPartition(kfkTopic, channel);
     
     self.consumer.assign([x]);
     self.consumer.seek_to_end();        
     last = self.consumer.position(x);
     
     pos = last - records if last > records else 0;
     self.consumer.seek(x, pos);
     
     data = self.consumer.poll(256);   
     
     endIsNotReached = True;
     while endIsNotReached:
         
         if len(data.values()) == 0:
             return messages;
         
         records = list(data.values())
         
         for record in records[0]:
             index = record[2];
             if index >= last - 1: endIsNotReached = False;
             
             message = Message(record[0][41:], record[1], record[6], index, record[3]);
             messages.append(message);
         
         if endIsNotReached == False: 
             self.consumer.close();
             return messages;
         
         pos = pos + len(messages)
         self.consumer.seek(x, pos);
         data = self.consumer.poll(256);
         
     self.consumer.close();
     
     return messages;
Пример #22
0
    def history(self, topic, channel, records):

        messages = []

        if self.uid == None:
            self.consumer = KafkaConsumer(
                bootstrap_servers=self.__bootstrap,
                check_crcs=False,
                exclude_internal_topics=True,
                session_timeout_ms=10000,
                reconnect_backoff_ms=10000,
                heartbeat_interval_ms=2000,
                retry_backoff_ms=500,
                fetch_min_bytes=64,
                fetch_max_wait_ms=96,
                enable_auto_commit=False,
                max_in_flight_requests_per_connection=4,
                api_version=(0, 10))
        else:
            home = expanduser("~")

            self.consumer = KafkaConsumer(
                bootstrap_servers=self.__bootstrap,
                check_crcs=False,
                exclude_internal_topics=True,
                session_timeout_ms=10000,
                reconnect_backoff_ms=10000,
                heartbeat_interval_ms=2000,
                retry_backoff_ms=500,
                fetch_min_bytes=64,
                fetch_max_wait_ms=96,
                enable_auto_commit=False,
                max_in_flight_requests_per_connection=4,
                security_protocol='SSL',
                ssl_check_hostname=False,
                ssl_keyfile=home + '/magistral/' + self.__token + '/key.pem',
                ssl_cafile=home + '/magistral/' + self.__token + '/ca.pem',
                ssl_certfile=home + '/magistral/' + self.__token +
                '/certificate.pem',
                api_version=(0, 10))

        if (records > self.__HISTORY_DATA_FETCH_SIZE_LIMIT):
            records = self.__HISTORY_DATA_FETCH_SIZE_LIMIT

        kfkTopic = self.__subKey + "." + topic
        x = TopicPartition(kfkTopic, channel)

        self.consumer.assign([x])
        self.consumer.seek_to_end()
        last = self.consumer.position(x)

        pos = last - records if last > records else 0
        self.consumer.seek(x, pos)

        data = self.consumer.poll(256)

        endIsNotReached = True
        while endIsNotReached:

            if len(data.values()) == 0:
                return messages

            records = list(data.values())

            for record in records[0]:
                index = record[2]
                if index >= last - 1: endIsNotReached = False

                message = Message(record[0][41:], record[1], record[6], index,
                                  record[3])
                messages.append(message)

            if endIsNotReached == False:
                self.consumer.close()
                return messages

            pos = pos + len(messages)
            self.consumer.seek(x, pos)
            data = self.consumer.poll(256)

        self.consumer.close()

        return messages
Пример #23
0
def test_paused(kafka_broker, topic):
    consumer = KafkaConsumer(bootstrap_servers=get_connect_str(kafka_broker))
    topics = [TopicPartition(topic, 1)]
    consumer.assign(topics)
    assert set(topics) == consumer.assignment()
    assert set() == consumer.paused()

    consumer.pause(topics[0])
    assert set([topics[0]]) == consumer.paused()

    consumer.resume(topics[0])
    assert set() == consumer.paused()

    consumer.unsubscribe()
    assert set() == consumer.paused()
class MagistralConsumer(object):
    
    __HISTORY_DATA_FETCH_SIZE_LIMIT = 10000;

    def __init__(self, pubKey, subKey, secretKey, bootstrap, token, cipher = None, uid = None):
        self.__pubKey = pubKey
        self.__subKey = subKey
        self.__secretKey = secretKey
        
        self.__token = token
        
        self.uid = uid
                
        self.__bootstrap = bootstrap.split(',')
        if cipher is not None: self.__cipher = cipher
    
    def history(self, topic, channel, records):
        
        messages = []
        
        if self.uid == None:
            self.consumer = KafkaConsumer(bootstrap_servers = self.__bootstrap,
                    check_crcs = False,
                    exclude_internal_topics = True,
                    session_timeout_ms = 10000,
                    reconnect_backoff_ms = 10000,
                    heartbeat_interval_ms = 2000,
                    retry_backoff_ms = 500,
                    fetch_min_bytes = 64,
                    fetch_max_wait_ms = 96,           
                    enable_auto_commit = False,
                    max_in_flight_requests_per_connection = 4,
                    api_version = (0, 10));
        else:            
            home = expanduser("~")
            
            self.consumer = KafkaConsumer(bootstrap_servers = self.__bootstrap,
                    check_crcs = False,
                    exclude_internal_topics = True,
                    session_timeout_ms = 10000,
                    reconnect_backoff_ms = 10000,
                    heartbeat_interval_ms = 2000,
                    retry_backoff_ms = 500,
                    fetch_min_bytes = 64,
                    fetch_max_wait_ms = 96,           
                    enable_auto_commit = False,
                    max_in_flight_requests_per_connection = 4,
                    security_protocol = 'SSL',
                    ssl_check_hostname = False,
                    ssl_keyfile = home + '/magistral/' + self.__token + '/key.pem',
                    ssl_cafile = home + '/magistral/' + self.__token + '/ca.pem',
                    ssl_certfile = home + '/magistral/' + self.__token + '/certificate.pem',
                    api_version = (0, 10));
            
        if (records > self.__HISTORY_DATA_FETCH_SIZE_LIMIT): records = self.__HISTORY_DATA_FETCH_SIZE_LIMIT;
            
        kfkTopic = self.__subKey + "." + topic;
        x = TopicPartition(kfkTopic, channel);
        
        self.consumer.assign([x]);
        self.consumer.seek_to_end();        
        last = self.consumer.position(x);
        
        pos = last - records if last > records else 0;
        self.consumer.seek(x, pos);
        
        data = self.consumer.poll(256);   
        
        endIsNotReached = True;
        while endIsNotReached:
            
            if len(data.values()) == 0:
                return messages;
            
            records = list(data.values())
            
            for record in records[0]:
                index = record[2];
                if index >= last - 1: endIsNotReached = False;
                
                message = Message(record[0][41:], record[1], record[6], index, record[3]);
                messages.append(message);
            
            if endIsNotReached == False: 
                self.consumer.close();
                return messages;
            
            pos = pos + len(messages)
            self.consumer.seek(x, pos);
            data = self.consumer.poll(256);
            
        self.consumer.close();
        
        return messages;
    
    def historyForTimePeriod(self, topic, channel, start, end, limit = -1):
        
        out = []        
        
        try:
            kfkTopic = self.__subKey + "." + topic;
            x = TopicPartition(kfkTopic, channel);
            
            if self.uid == None:
                self.consumer = KafkaConsumer(bootstrap_servers = self.__bootstrap,
                        check_crcs = False,
                        exclude_internal_topics = True,
                        session_timeout_ms = 30000,
                        reconnect_backoff_ms = 10000,
                        heartbeat_interval_ms = 2000,
                        retry_backoff_ms = 500,
                        fetch_min_bytes = 32,
                        fetch_max_wait_ms = 96,           
                        enable_auto_commit = False,
                        max_partition_fetch_bytes = 65536,
                        max_in_flight_requests_per_connection = 4,
                        api_version = (0, 10));
            else:            
                home = expanduser("~")
                
                self.consumer = KafkaConsumer(bootstrap_servers = self.__bootstrap,
                        check_crcs = False,
                        exclude_internal_topics = True,
                        session_timeout_ms = 30000,
                        reconnect_backoff_ms = 10000,
                        heartbeat_interval_ms = 2000,
                        retry_backoff_ms = 500,
                        fetch_min_bytes = 32,
                        fetch_max_wait_ms = 96,           
                        enable_auto_commit = False,
                        max_partition_fetch_bytes = 65536,
                        max_in_flight_requests_per_connection = 4,
                        security_protocol = 'SSL',
                        ssl_check_hostname = False,
                        ssl_keyfile = home + '/magistral/' + self.__token + '/key.pem',
                        ssl_cafile = home + '/magistral/' + self.__token + '/ca.pem',
                        ssl_certfile = home + '/magistral/' + self.__token + '/certificate.pem',
                        api_version = (0, 10));
                        
            self.consumer = KafkaConsumer(bootstrap_servers = self.__bootstrap);
            self.consumer.assign([x]);
            
            self.consumer.seek_to_end();        
            last = self.consumer.position(x);
        
            position = last - 1000;
            
            found = False;
            while found == False:
                self.consumer.seek(x, position);
                data = self.consumer.poll(500);
                 
                if x not in data.keys() or len(data[x]) == 0: break;
                
                record = data[x][0];
                timestamp = record[3];
 
                if timestamp < start: 
                    found = True;
                    break;
                 
                position = position - 1000;
             
            self.consumer.close();
                       
            if self.uid == None:
                self.с = KafkaConsumer(bootstrap_servers = self.__bootstrap,
                        check_crcs = False,
                        exclude_internal_topics = True,
                        session_timeout_ms = 10000,
                        reconnect_backoff_ms = 10000,
                        heartbeat_interval_ms = 2000,
                        retry_backoff_ms = 500,
                        fetch_min_bytes = 32,
                        fetch_max_wait_ms = 96,           
                        enable_auto_commit = False,
                        max_partition_fetch_bytes = 65536,
                        max_in_flight_requests_per_connection = 4,
                        api_version = (0, 10));
            else:            
                home = expanduser("~")
                
                self.с = KafkaConsumer(bootstrap_servers = self.__bootstrap,
                        check_crcs = False,
                        exclude_internal_topics = True,
                        session_timeout_ms = 10000,
                        reconnect_backoff_ms = 10000,
                        heartbeat_interval_ms = 2000,
                        retry_backoff_ms = 500,
                        fetch_min_bytes = 32,
                        fetch_max_wait_ms = 96,           
                        enable_auto_commit = False,
                        max_partition_fetch_bytes = 65536,
                        max_in_flight_requests_per_connection = 4,
                        security_protocol = 'SSL',
                        ssl_check_hostname = False,
                        ssl_keyfile = home + '/magistral/' + self.__token + '/key.pem',
                        ssl_cafile = home + '/magistral/' + self.__token + '/ca.pem',
                        ssl_certfile = home + '/magistral/' + self.__token + '/certificate.pem',
                        api_version = (0, 10));  
                      
            self.с.assign([x]);
                       
            self.c.seek(x, position);                        
            data = self.с.poll(256);
            
            while (x in data.keys() and len(data[x]) > 0):
                
                for record in data[x] :
                    timestamp = record[3];
                    if timestamp < start: continue;
                    
                    index = record[2];
                    
                    if timestamp > end or index >= last - 1:  
                        self.с.close();                 
                        return out;
                                    
                    message = Message(record[0][41:], record[1], record[6], index, timestamp);
                    out.append(message); 
                    
                    if limit is not None and limit > 0 and len(out) >= limit:
                        self.с.close();                 
                        return out;                 
                    
                self.с.seek(x, position + len(data[x]));                        
                data = self.с.poll(256);
            
            return out;
        
        except:
            
            raise MagistralException("Exception during history invocation occurred");
Пример #25
0
class GroupConsumer(threading.Thread):
        
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.INFO)
     
    def __init__(self, threadId, name, sKey, bootstrapServers, groupId, permissions, token, cipher = None, uid = None):
        threading.Thread.__init__(self)
        self.threadId = threadId
        self.name = name
        
        self.group = groupId;
        self.subKey = sKey;
        
        self.cipher = None if cipher is None else cipher;
                
        configs = Configs.consumerConfigs();
        configs["bootstrap_servers"] = bootstrapServers.split(',');
        configs["group_id"] = groupId;
        configs['enable_auto_commit'] = False;
        
        self.__isAlive = True
        
        home = expanduser("~")
        
        if uid == None:
            self.__consumer = KafkaConsumer(
                bootstrap_servers = configs["bootstrap_servers"],
                check_crcs = False,
                exclude_internal_topics = True,
                session_timeout_ms = 10000,
                reconnect_backoff_ms = 10000,
                heartbeat_interval_ms = 2000,
                retry_backoff_ms = 500,
                fetch_min_bytes = 64,
                fetch_max_wait_ms = 96,           
                enable_auto_commit = False,
                max_in_flight_requests_per_connection = 4,
                api_version = (0, 10),            
                group_id = groupId);
        else:
            self.__consumer = KafkaConsumer(
                bootstrap_servers = configs["bootstrap_servers"],
                check_crcs = False,
                exclude_internal_topics = True,
                session_timeout_ms = 10000,
                reconnect_backoff_ms = 10000,
                heartbeat_interval_ms = 2000,
                retry_backoff_ms = 500,
                fetch_min_bytes = 64,
                fetch_max_wait_ms = 96,           
                enable_auto_commit = False,
                max_in_flight_requests_per_connection = 4,
                security_protocol = 'SSL',
                ssl_check_hostname = False,
                ssl_keyfile = home + '/magistral/' + token + '/key.pem',
                ssl_cafile = home + '/magistral/' + token + '/ca.pem',
                ssl_certfile = home + '/magistral/' + token + '/certificate.pem',
                api_version = (0, 10),
                group_id = groupId);
        
        self.permissions = permissions;
        self.map = {}
        
        self.__offsets = {}
        
        
    def recordsTotally(self, data):
        size = 0;
        for val in data.values(): 
            if len(val) > 0: size = size + len(val);
                               
        return size;
    
    def consumerRecord2Message(self, record):                    
        payload = record[6]
                                        
        if self.cipher is not None:
            try:
                payload = self.cipher.decrypt(payload)
            except:
                pass
                                        
        msg = Message(record[0][41:], record[1], payload, record[2], record[3])
        return msg

    def run(self):
        
        threadLock.acquire(False)
        
        while self.__isAlive:
            try:

                data = self.__consumer.poll(512);
                for values in data.values():
                                         
                    for value in values:
                        msg = self.consumerRecord2Message(value);
                        listener = self.map[value[0]][msg.channel()];
                        if listener is not None: listener(msg);
                        
                if len(data.values()) > 0: self.__consumer.commit_async(); 
                    
            except:
                pass
              
        threadLock.release()

#   ////////////////////////////////////////////////////////////////////////////////////    

    def subscribe(self, topic, channel = -1, listener = None, callback = None):
        
        assert channel is not None and isinstance(channel, int), "Channel expected as int argument"        
        if (channel < -1): channel = -1;
                
        etopic = self.subKey + "." + topic;
                           
        self.logger.debug("Subscribe -> %s : %s | key = %s", topic, channel, self.subKey);
        
        if (self.permissions == None or len(self.permissions) == 0): 
            raise MagistralException("User has no permissions for topic [" + topic + "].");
        
        self.fch = [];
        
        for meta in self.permissions:             
            if (meta.topic() != topic): continue;  
            
            if channel == -1:
                self.fch = meta.channels();
            elif channel in meta.channels():                          
                self.fch = [ channel ];  
        
        if (len(self.fch) == 0): 
            npgex = "No permissions for topic [" + topic + "] granted";
            self.logger.error(npgex);                                
            raise MagistralException(npgex);
        
        if (self.map == None or etopic not in self.map): 
            self.map[etopic] = {}
        
#         // Assign Topic-partition pairs to listen
        
        tpas = [];
        for ch in self.fch:            
            tpas.append(TopicPartition(etopic, ch));
            if (listener is not None): self.map[etopic][ch] = listener
            
            ca = self.__consumer.assignment()
            if (ca is not None):
                for tp in ca: tpas.append(tp)
        
        self.__consumer.assign(tpas);       
                
        if callback is not None: 
            callback(self.__consumer.assignment());
            
        return self.__consumer.assignment();
        
        
    def unsubscribe(self, topic):
        self.consumer.assign([]);
        self.map.remove(topic);        

    def close(self):
        self.__isAlive = False
        self.__consumer.pause()
        self.__consumer.close()
        
    logging.getLogger('kafka.conn').setLevel(logging.FATAL)
    logging.getLogger('kafka.cluster').setLevel(logging.FATAL)
    logging.getLogger('kafka.consumer.group').setLevel(logging.INFO)    
    logging.getLogger('kafka.consumer.fetcher').setLevel(logging.INFO)
    logging.getLogger('kafka.coordinator.consumer').setLevel(logging.INFO)
    logging.getLogger('kafka.producer.record_accumulator').setLevel(logging.INFO)