def _create_consumer(self) -> None:
# noinspection PyPackageRequirements
import kafka
if "plaintext" in self.security_protocol.lower():
self.consumer = kafka.KafkaConsumer(
self.topic,
bootstrap_servers=self.host,
security_protocol="PLAINTEXT",
sasl_mechanism="PLAIN",
sasl_plain_username=self.sasl_username,
sasl_plain_password=self.sasl_password,
ssl_check_hostname=False,
)
elif "ssl" in self.security_protocol.lower():
self.consumer = kafka.KafkaConsumer(
self.topic,
bootstrap_servers=self.host,
security_protocol="SSL",
ssl_cafile=self.ssl_cafile,
ssl_certfile=self.ssl_certfile,
ssl_keyfile=self.ssl_keyfile,
ssl_check_hostname=self.ssl_check_hostname,
)
else:
raise ValueError(
f"Cannot initialise `kafka.KafkaConsumer` "
f"with security protocol '{self.security_protocol}'."
)
def create_kafka_consumer(self):
# Create kafka consumer. Wait until kafka broker becomes available if necessary
#
while self._consumer is None:
try:
logger.info(
'Kafka: starting a consumer for topics {}..'.format(
KAFKA_REQUEST_TOPICS))
if self._consumer_timeout_ms is not None: # TODO improve kafka initialization.
self._consumer = kafka.KafkaConsumer(
*self._topics,
bootstrap_servers=self._bootstrap_servers,
consumer_timeout_ms=self._consumer_timeout_ms)
else:
self._consumer = kafka.KafkaConsumer(
*self._topics,
bootstrap_servers=self._bootstrap_servers)
except:
logger.info("Failed to create kafka consumer. Error {}".format(
sys.exc_info()[0]))
logger.info("Waiting 5s before retrying..")
time.sleep(5)
logger.info("Consumer Created.")
while self._adminclient is None:
try:
self._adminclient = kafka.KafkaAdminClient(
bootstrap_servers=self._bootstrap_servers)
except:
logger.info(
"Failed to create kafka admin client. Error {}".format(
sys.exc_info()[0]))
logger.info("Waiting 5s before retrying..")
time.sleep(5)
logger.info('Kafka: creating topics..')
while self._topics_ready is False:
try:
self._topics_ready = True
for topic in self._topics:
topic_partitions = self._consumer.partitions_for_topic(
topic)
if topic_partitions is None:
self._adminclient.create_topics([
kafka.admin.NewTopic(
topic,
num_partitions=KAFKA_CONSUMER_PARTITIONS,
replication_factor=KAFKA_CONSUMER_REPLICAS)
])
self._topics_ready = False
time.sleep(2)
except:
self._topics_ready = False
logger.info("Failed to create kafka topics. Error {}".format(
sys.exc_info()))
logger.info("Waiting 2s before retrying..")
time.sleep(2)
self._adminclient.close()
logger.info("Kafka: Topics Created.")
def main():
parser = argparse.ArgumentParser(
description="Kafka client to get groups and topics status")
parser.add_argument(
"--server",
type=str,
metavar="HOST",
default="localhost",
help="Kafka bootstrap-server address",
)
parser.add_argument(
"--port",
type=int,
metavar="PORT",
default=9092,
help="Kafka bootstrap-server port",
)
parser.add_argument(
"--client",
type=str,
default="ch-kafka-python",
help="custom client id for this producer",
)
args = parser.parse_args()
config = {
"bootstrap_servers": f"{args.server}:{args.port}",
"client_id": args.client,
}
client = kafka.KafkaAdminClient(**config)
consumer = kafka.KafkaConsumer(**config)
cluster = client._client.cluster
topics = cluster.topics()
for topic in topics:
print(f'Topic "{topic}":', end="")
for partition in cluster.partitions_for_topic(topic):
tp = kafka.TopicPartition(topic, partition)
print(
f" {partition} (begin: {consumer.beginning_offsets([tp])[tp]}, end: {consumer.end_offsets([tp])[tp]})",
end="",
)
print()
groups = client.list_consumer_groups()
for group in groups:
print(f'Group "{group[0]}" ({group[1]}):')
consumer = kafka.KafkaConsumer(**config, group_id=group[0])
offsets = client.list_consumer_group_offsets(group[0])
for topic, offset in offsets.items():
print(
f"\t{topic.topic}[{topic.partition}]: {consumer.beginning_offsets([topic])[topic]}, {offset.offset}, {consumer.end_offsets([topic])[topic]}"
)
consumer.close()
client.close()
return 0
def main():
parser = argparse.ArgumentParser(
description='Kafka client to get groups and topics status')
parser.add_argument('--server',
type=str,
metavar='HOST',
default='localhost',
help='Kafka bootstrap-server address')
parser.add_argument('--port',
type=int,
metavar='PORT',
default=9092,
help='Kafka bootstrap-server port')
parser.add_argument('--client',
type=str,
default='ch-kafka-python',
help='custom client id for this producer')
args = parser.parse_args()
config = {
'bootstrap_servers': f'{args.server}:{args.port}',
'client_id': args.client,
}
client = kafka.KafkaAdminClient(**config)
consumer = kafka.KafkaConsumer(**config)
cluster = client._client.cluster
topics = cluster.topics()
for topic in topics:
print(f'Topic "{topic}":', end='')
for partition in cluster.partitions_for_topic(topic):
tp = kafka.TopicPartition(topic, partition)
print(
f' {partition} (begin: {consumer.beginning_offsets([tp])[tp]}, end: {consumer.end_offsets([tp])[tp]})',
end='')
print()
groups = client.list_consumer_groups()
for group in groups:
print(f'Group "{group[0]}" ({group[1]}):')
consumer = kafka.KafkaConsumer(**config, group_id=group[0])
offsets = client.list_consumer_group_offsets(group[0])
for topic, offset in offsets.items():
print(
f'\t{topic.topic}[{topic.partition}]: {consumer.beginning_offsets([topic])[topic]}, {offset.offset}, {consumer.end_offsets([topic])[topic]}'
)
consumer.close()
client.close()
return 0
def _create_producer(self) -> None:
import kafka
if self.security_protocol == "PLAINTEXT":
self.producer = kafka.KafkaConsumer(
self.topic,
bootstrap_servers=self.url,
client_id=self.client_id,
group_id=self.group_id,
security_protocol="PLAINTEXT",
ssl_check_hostname=False,
)
elif self.security_protocol == "SASL_PLAINTEXT":
self.producer = kafka.KafkaProducer(
bootstrap_servers=self.url,
value_serializer=lambda v: json.dumps(v).encode(
DEFAULT_ENCODING),
sasl_plain_username=self.sasl_username,
sasl_plain_password=self.sasl_password,
sasl_mechanism="PLAIN",
security_protocol=self.security_protocol,
)
elif self.security_protocol == "SSL":
self.producer = kafka.KafkaProducer(
bootstrap_servers=self.url,
value_serializer=lambda v: json.dumps(v).encode(
DEFAULT_ENCODING),
ssl_cafile=self.ssl_cafile,
ssl_certfile=self.ssl_certfile,
ssl_keyfile=self.ssl_keyfile,
ssl_check_hostname=False,
security_protocol=self.security_protocol,
)
elif self.security_protocol == "SASL_SSL":
self.producer = kafka.KafkaConsumer(
self.topic,
bootstrap_servers=self.url,
client_id=self.client_id,
group_id=self.group_id,
security_protocol="SASL_SSL",
sasl_mechanism="PLAIN",
sasl_plain_username=self.sasl_username,
sasl_plain_password=self.sasl_password,
ssl_cafile=self.ssl_cafile,
ssl_certfile=self.ssl_certfile,
ssl_keyfile=self.ssl_keyfile,
ssl_check_hostname=self.ssl_check_hostname,
)
else:
raise ValueError(
f"Cannot initialise `KafkaEventBroker`: "
f"Invalid `security_protocol` ('{self.security_protocol}').")
def connect(self, *args: Any, **kwargs: Any) -> kafka.KafkaConsumer:
if 'value_deserializer' not in self.config and 'value_deserializer' not in kwargs:
kwargs['value_deserializer'] = lambda x: json.loads(
x.decode('utf-8'))
if 'topic' in self.config:
_config = deepcopy(self.config)
_topic = _config.pop('topic')
self.session = kafka.KafkaConsumer(_topic, *args, **_config,
**kwargs)
else:
self.session = kafka.KafkaConsumer(*args, **self.config, **kwargs)
self._closed = False
return self.session
def _get_server(self):
if not self.kafka_server:
self.kafka_server = kafka.KafkaClient("%s:%s" %
(self.host, self.port))
self.kafka_consumer = kafka.KafkaConsumer(
self.topic,
bootstrap_servers=["%s:%s" % (self.host, self.port)])
def main(consumer_topic=TOPIC,
producer_topic=VALIDATION_TOPIC,
boot_servers=BOOT_SERVERS):
consumer = kafka.KafkaConsumer(consumer_topic,
bootstrap_servers=boot_servers,
auto_offset_reset="earliest",
value_deserializer=_deserializer)
producer = kafka.KafkaProducer(
bootstrap_servers=boot_servers,
value_serializer=lambda m: json.dumps(m).encode('utf-8'))
def validate_data(request_id, result="success"):
msg = {"request_id": request_id, "validation": result}
producer.send(producer_topic, msg)
for msg in consumer:
data = msg.value
if not data or not isinstance(data, dict):
# smth is wrong, not processing
continue
reports = fetch_report(data["url"])
if not reports:
# couldn't fetch report data, discarding message
validate_data(data.get("request_id"), result="handoff")
else:
LOG.info("Fetched and validated tgz from %s", data["url"])
# data is valid, waving green flag
validate_data(data.get("request_id"))
# process data (upload to db in our case)
for report in reports:
res = store_in_db(report)
LOG.info("Uploaded: %s", json.dumps(res).encode('utf-8'))
def offset_range_for_timestamp_range(brokers, start, end, topic):
"""Determine OffsetRange for a given timestamp range
Parameters
----------
client_config : ClientConfig
start : number
Unix timestamp in seconds
end : number
Unix timestamp in seconds
topic : str
Topic to fetch offsets for
Returns
-------
list of OffsetRange or None
Per-partition ranges of offsets to read
"""
consumer = kafka.KafkaConsumer(bootstrap_servers=brokers)
partitions = consumer.partitions_for_topic(topic)
if partitions is None:
# Topic does not exist.
return None
partitions = [kafka.TopicPartition(topic, p) for p in partitions]
o_start = offsets_for_times(consumer, partitions, start)
o_end = offsets_for_times(consumer, partitions, end)
return [OffsetRange(tp, o_start[tp], o_end[tp]) for tp in partitions]
def convert_offsets_to_timestamp(self, topic, kafka_offsets, kafka_hosts):
consumer = kafka.KafkaConsumer(bootstrap_servers=kafka_hosts,
group_id=None,
enable_auto_commit="false",
auto_offset_reset="none",
consumer_timeout_ms=1000)
timestamps = {}
for tp, offset in kafka_offsets.items():
consumer.assign([tp])
if not offset is None:
consumer.seek(tp, max(int(offset) - 1, 0))
try:
msg = next(consumer)
except (StopIteration, OffsetOutOfRangeError) as err:
error_msg = "Message does not exists in host <{}>: <{}>"
logger.error(error_msg.format(kafka_hosts, err))
else:
timestamps[tp] = msg.timestamp
else:
msg = "Partition: {} has ans invalid offset: {}"
logger.debug(msg.format(tp, offset))
consumer.close()
return timestamps
def __init__(self,config_file):
super().__init__()
self.args = AlphaConfig.loadConfig( os.path.join( AlphaPathLookUp.ConfigPath, config_file ) )
#self.is_ready = Event()
self.producer=kafka.KafkaProducer(bootstrap_servers=self.args.producer["servers"],value_serializer= lambda m: json.dumps(m).encode('ascii') )
self.consumer=kafka.KafkaConsumer(self.args.consumer["topic"], group_id=self.args.consumer["group_id"],
bootstrap_servers=self.args.consumer["servers"], value_deserializer= lambda m: json.loads(m.decode("ascii")))
def main():
parser = argparse.ArgumentParser(description='Kafka Producer client')
parser.add_argument('--server', type=str, metavar='HOST', default='localhost',
help='Kafka bootstrap-server address')
parser.add_argument('--port', type=int, metavar='PORT', default=9092,
help='Kafka bootstrap-server port')
parser.add_argument('--client', type=str, default='ch-kafka-python',
help='custom client id for this producer')
parser.add_argument('--topic', type=str, required=True,
help='name of Kafka topic to store in')
parser.add_argument('--group', type=str, required=True,
help='name of the consumer group')
args = parser.parse_args()
config = {
'bootstrap_servers': f'{args.server}:{args.port}',
'client_id': args.client,
'group_id': args.group,
}
client = kafka.KafkaConsumer(**config)
client.subscribe([args.topic])
pprint(client.poll(10000))
client.unsubscribe()
client.close()
def offset_range_for_timestamp_range(brokers,
start,
end,
topic=mjolnir.kafka.TOPIC_RESULT):
"""Determine OffsetRange for a given timestamp range
Parameters
----------
brokers : list of str
List of kafka broker hostport to bootstrap kafka connection with
start : number
Unix timestamp in seconds
end : number
Unix timestamp in seconds
topic : str
Kafka topic to retrieve offsets for
Returns
-------
list of pyspark.streaming.kafka.OffsetRange or None
Per-partition ranges of offsets to read
"""
consumer = kafka.KafkaConsumer(bootstrap_servers=brokers,
api_version=mjolnir.kafka.BROKER_VERSION)
partitions = consumer.partitions_for_topic(topic)
if partitions is None:
# Topic does not exist.
return None
partitions = [kafka.TopicPartition(topic, p) for p in partitions]
o_start = offsets_for_times(consumer, partitions, start)
o_end = offsets_for_times(consumer, partitions, end)
return [
OffsetRange(tp.topic, tp.partition, o_start[tp], o_end[tp])
for tp in partitions
]
def listKafkaTopics():
import kafka, sys, os
from array import array
consumer = kafka.KafkaConsumer(bootstrap_servers=['192.168.1.130:9092'])
kafkaTopics = []
kafkaRawTopic = consumer.topics()
kafkaTopics.append(kafkaRawTopic)
kafkaTopics2 = ["hello mr bubba. this is great!","jeeze bubba"]
kafkaTopics2.append(kafkaRawTopic)
kafkaTopics3 = kafkaTopics2[2]
#kafkaTopics3Encoded = kafkaTopics3.encode('utf-8')
#firstTopic = kafkaTopics[1]
#flash(firstTopic)
#resp = make_response(redirect(url_for('main.watchZookeeper')))
# global kafkaTopics3
#kafkaTopics3 = os.system('/home/joe/workspace/flasky/kafka-utils/list-topics.sh')
#kafkaTopics4 = kafkaTopics3[0]
#def subfunction():
# return True
#resp = make_response(flash(consumer.topics(0)))
#resp.set_cookie('show_followed', '', max_age=30*24*60*60)
#return jsonify(kafkaTopics2)
#return request.get_json(force=True)
#return json.dumps({'status':'OK', 'diskeyText' :kafkaTopics2})
return json.dumps({'status':'OK', 'diskeyText' :kafkaTopics2[1]})
def main():
consumer = kafka.KafkaConsumer(
config.get_config("KAFKA_TOPIC"),
auto_offset_reset="earliest",
bootstrap_servers=config.get_config("KAFKA_URI"),
client_id=config.get_config("KAFKA_CLIENT_ID"),
group_id=config.get_config("KAFKA_CONSUMER_GROUP"),
enable_auto_commit=True,
value_deserializer=lambda x: loads(x.decode("utf-8")),
security_protocol="SSL",
ssl_cafile=config.get_config("KAFKA_CAFILE"),
ssl_certfile=config.get_config("KAFKA_ACCESS_CERT"),
ssl_keyfile=config.get_config("KAFKA_ACCESS_KEY"),
)
db_repo = DB(config.get_config("PSQL_URI"))
try:
# this should be a seperate schema migration process instead
db_repo.create_hb_table()
# Short polling here
# Consider long-polling
for message in consumer:
message_value = message.value
try:
db_repo.insert_hb(**message_value)
except Exception as e:
print(f"failled to insert record {message_value}, err: {e}")
except Exception as e:
print(f"exception occured: {e}")
raise e
finally:
consumer.close(autocommit=True)
db_repo.close()
def consumer(args):
attempts = 1
ready = False
kafka_consumer = None
while not ready:
logging.info("starting kafka consumer (attempt %d)" % attempts)
attempts += 1
try:
kafka_consumer = kafka.KafkaConsumer(
args.topic, bootstrap_servers=args.brokers)
ready = True
finally:
pass
msg_count = 0
for msg in kafka_consumer:
if EXIT_EVENT.is_set():
logging.info("exiting upon request")
break
try:
msg_count = msg_count + 1
if msg_count % 100 == 0:
logging.info("scoring message %d" % msg_count)
score_text(json.loads(str(msg.value, "utf-8"))["text"])
except Exception as ex:
logging.error(repr(ex))
logging.info("exiting kafka consumer")
def __init__(self, config: dict) -> None:
try:
self._consumer = kafka.KafkaConsumer(
config['kafka']['topic'],
auto_offset_reset="earliest",
bootstrap_servers=config['kafka']['uri'],
security_protocol="SSL",
group_id="site-availability-group",
ssl_cafile="keys/ca.pem",
ssl_certfile="keys/service.cert",
ssl_keyfile="keys/service.key",
value_deserializer=lambda m: json.loads(m.decode('utf-8')),
enable_auto_commit=True)
except kafka.errors.NoBrokersAvailable:
raise Exception(
f"Unable to connect to Kafka at {config['kafka']['uri']}")
# Init PostgreSQL connection and do some sanity checking
try:
conn = psycopg2.connect(**config['postgres'])
conn.autocommit = True
self._conn = conn
except psycopg2.OperationalError:
raise Exception(
f"Unable to connect to PostgreSQL instance at {config}")
# Simple check to ensure that our working table exists
with self._conn.cursor() as cur:
try:
cur.execute('SELECT 1 FROM availability_data')
except psycopg2.errors.UndefinedTable:
raise Exception(
"'availability_data' table does not exist. Please create it using supplied schema.sql"
)
def _create_util_consumer(self, group_none=False):
consumer = kafka.KafkaConsumer(
bootstrap_servers=self._brokers,
group_id=None if group_none else self._group,
api_version=self._api_version,
**self._ssl_configs)
return consumer
def topology_consume_custom(server, tx, knowledge):
topic = 'oa.custom.{0}.{1}'.format(tx, knowledge)
consumer = kafka.KafkaConsumer(topic, bootstrap_servers=server)
for msg in consumer:
yield msg.value
def main(config):
"""Continuously copy events from kafka to postgresql"""
conn = psycopg2.connect(config.PG_DSN)
consumer = kafka.KafkaConsumer(
'webmonitor',
group_id='webmonitor',
auto_offset_reset='earliest',
enable_auto_commit=False,
value_deserializer=json_deserialize,
**config.KAFKA_OPTS
)
batch = []
cur = conn.cursor()
for msg in consumer:
if len(batch) >= config.BATCH_SIZE:
psycopg2.extras.execute_batch(cur, 'SELECT insert_event(%s, %s, %s, %s, %s, %s, %s)', batch)
batch = []
conn.commit()
consumer.commit()
timestamp = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(msg.timestamp/1000))
data = msg.value
ok = 'error' not in data
batch.append((timestamp, data['url'], ok, data.get('http_status'), data.get('response_time'), data.get('content_ok'), data.get('error')))
def test_consumer_kafka_python(conf: Config):
consumer = k.KafkaConsumer(
auto_offset_reset='earliest',
bootstrap_servers=conf["brokers"],
group_id="kafka-python-" + str(uuid.uuid1()),
enable_auto_commit=True,
key_deserializer=lambda k: k.decode('utf-8'),
value_deserializer=lambda m: json.loads(m.decode('utf-8')))
consumer.subscribe(conf["topic"])
consumer.poll(0)
log.info("Resetting offset to beginning...")
consumer.seek_to_beginning()
start = datetime.datetime.now()
_counter = 0
while _counter < conf["num_messages"]:
data = consumer.poll(timeout_ms=500)
for tp, messages in data.items():
for message in messages:
_counter += 1
if _counter >= conf["num_messages"]:
break
end = datetime.datetime.now()
log.debug("Consumed %s messages", _counter)
return TestResult(start=start, end=end, num_messages=_counter)
def run(self):
consumer = kafka.KafkaConsumer(bootstrap_servers=config.KAFKA_SERVERS)
consumer.subscribe([config.KAFKA_TOPIC])
if config.LOG_DEBUG:
for message in consumer:
print("{0}\n\n".format(message))
def stopKafkaConsumer():
import kafka
topic = "cryBert"
consumer = kafka.KafkaConsumer(bootstrap_servers='192.168.1.130:9092')
consumer.unsubscribe()
consumer.close()
return json.dumps({'status': 'OK','topic': topic})
def entry_point():
params = parse_args()
if params.cp == ConsumerProducer.PRODUCER:
producer = kafka.KafkaProducer(
bootstrap_servers=[params.addr],
value_serializer=lambda x: json.dumps(x).encode('utf-8'))
while True:
for i in range(0, sys.maxsize):
data = {'number': i}
producer.send(params.topic, value=data)
time.sleep(1)
else:
consumer = kafka.KafkaConsumer(
params.topic,
bootstrap_servers=[params.addr],
group_id='my-group',
value_deserializer=lambda x: json.loads(x.decode('utf-8')))
for message in consumer:
message = message.value
print("message is %s" % message)
pass
def consumer(args):
# connect to kafka
brokers = args.kafka_brokers.split(",")
kafka_consumer = kafka.KafkaConsumer(args.topic, bootstrap_servers=brokers)
# if debug not set, write libpcap global header
if args.debug == 0:
sys.stdout.write(global_header())
# start packet capture
packet_count = 0
for msg in kafka_consumer:
# if debug not set, write the packet header and packet
if args.debug == 0:
sys.stdout.write(packet_header(msg.value, msg.key))
sys.stdout.write(msg.value)
elif packet_count % args.debug == 0:
print 'Packet: count=%s dt=%s topic=%s' % (
packet_count, to_date(unpack_ts(msg.key)), args.topic)
print to_hex(msg.value)
packet_count += 1
if args.packet_count > 0 and packet_count >= args.packet_count:
break
def main_loop():
files = []
print('Connected to AMQ Streams...')
try:
while True:
consumer_received = kafka.KafkaConsumer(
'file-received',
bootstrap_servers='my-cluster-kafka-bootstrap:9092',
consumer_timeout_ms=10000)
for message in consumer_received:
str_message = bytes.decode(message.value)
print(str_message)
# if its stupid but it works... well this is still stupid
filename = str(str_message.split(': ')[1:])
filename = filename.replace("'", '')
filename = filename.replace('[', '')
filename = filename.replace(']', '')
files.append(filename)
if len(files) == 2:
get_files(files)
files = []
except Exception as e:
print('Stream timed out, restarting...')
pass
def update_stats(topic, mycollection):
consumer = kafka.KafkaConsumer(topic, bootstrap_servers=[kafka_server])
for message in consumer:
data = (message.value).decode('utf-8')
print(data)
data = data.split(' ', 2)
mid = data[0] + ':' + data[1]
server_stats = data[2].split(' ')
load = {}
load['machineID'] = mid
load["username"] = server_stats[0]
load["password"] = server_stats[1]
load["free_cpu"] = float(server_stats[2])
load["free_mem"] = float(server_stats[3])
load["number_of_events_per_sec"] = int(server_stats[4])
load["free_RAM"] = float(server_stats[5])
load["temperature"] = float(server_stats[6])
load["n_cores"] = int(server_stats[7])
load["load_average_1"] = float(server_stats[8])
load["load_average_5"] = float(server_stats[9])
load["load_average_15"] = float(server_stats[10])
load["timestamp"] = getCurrTimestamp()
if checkIntoDB(mycollection, mid):
updateIntoDB(mycollection, mid, load)
print(mid, '- updated record into registry.')
else:
rec = insertIntoDB(mycollection, load)
print(mid, '- new record inserted into registry with id :', rec)
def consumer(name, instance, host, port):
consumer_latency = prometheus_client.Histogram(
'consumer_latency', 'consumer latency', ['app', 'instance', 'topic'])
consumer_counter = prometheus_client.Counter('consumer_counter',
'consumer counter',
['app', 'instance', 'topic'])
for _ in range(100):
try:
consumer = kafka.KafkaConsumer(bootstrap_servers='{}:{}'.format(
host, port),
group_id=name,
auto_offset_reset='earliest',
enable_auto_commit=False)
break
except Exception as e:
print('ERROR', e)
time.sleep(1)
def consume(topic, function):
consumer.subscribe([topic])
for message in consumer:
consumer_counter.labels(app=name, instance=instance,
topic=topic).inc()
with consumer_latency.labels(app=name,
instance=instance,
topic=topic).time():
function(json.loads(message.value.decode('utf-8')))
consumer.commit()
return consume
def start_kafka(self):
"""
Метод запуска kafka - для публичных очередей
Вызывается после успешного запуска Kombu API
"""
try:
if self.kafka_active:
logging.warning("Kafka already consuming, returning")
return
self.kafka_active = True
self.kafka_consumer = kafka.KafkaConsumer(
'cw3-offers',
'cw3-deals',
'cw3-duels',
'cw3-sex_digest',
'cw3-yellow_pages',
'cw3-au_digest',
bootstrap_servers=['digest-api.chtwrs.com:9092'],
auto_offset_reset='earliest',
enable_auto_commit=True,
group_id='cactiw_cw3_group_id',
value_deserializer=lambda x: json.loads(x.decode('utf-8')))
self.start_kafka_consuming()
except Exception:
logging.exception("Can not start kafka: {}".format(
traceback.format_exc()))
dispatcher.bot.send_message(chat_id=SUPER_ADMIN_ID,
text="Невозможно запустить kafka API")
def start(self):
print('[RequestDB Init] Sleeping...')
time.sleep(configs.sleep_interval)
print('[RequestDB Init] Init Successfully!')
while True:
try:
consumer = kafka.KafkaConsumer(self.kafka_topic,
group_id=self.group_id
,bootstrap_servers=[self.kafka_host])
for msg in consumer:
info_str = msg.value
if not info_str or info_str is None:
time.sleep(configs.call_interval)
continue
info_str = str(info_str, encoding = "utf-8")
info = json.loads(info_str)
client = pymongo.MongoClient(host=self.mongo_host, port=self.mongo_port)
db = client[self.mongo_db]
auth_ans = db.authenticate(name=self.mongo_user, password=self.mongo_pwd)
request_record = copy.deepcopy(info)
request_record['status'] = 'unfinished'
if db[configs.app_speaker_request_table].find_one({'id':request_record['id']}):
request_record['status'] = 'recovered'
db[configs.app_speaker_request_table].save(request_record)
else:
insert_result = db[configs.app_speaker_request_table].insert_one(request_record)
print('request has been written in db. info: ', info_str)
except Exception as e:
traceback.print_exc()
time.sleep(configs.call_interval)
continue
