class AIMSDownsamplingTCPServerConsumer:
LOG_FORMAT ="{} UTC_TS\t"\
"{}"
INTERVAL = 60
DELAY = 0
MAX_CONNECTION = 32
def __init__(self, kafka_host, kafka_port, tcp_host, tcp_port, topic,
log_topic):
self.kafka_host = kafka_host
self.kafka_port = kafka_port
self.tcp_host = tcp_host
self.tcp_port = tcp_port
self.topic = topic
self.log_topic = log_topic
self.consumer = KafkaConsumer(
topic,
bootstrap_servers=["{}:{}".format(kafka_host, kafka_port)],
enable_auto_commit=False,
max_poll_records=1024 * 1024,
max_partition_fetch_bytes=1024 * 1024 * 100)
self.producer = KafkaProducer(
bootstrap_servers=["{}:{}".format(kafka_host, kafka_port)])
self.connections = {}
self.sample_end_time = self.get_end_time(time())
self.lastPolled = []
def run(self):
self.log("running")
asyncio.run(self._async_run())
async def _async_run(self):
tcpServer = await asyncio.start_server(self.connection_handler,
self.tcp_host, self.tcp_port)
await asyncio.gather(tcpServer.serve_forever(), self.poll_from_kafka())
async def connection_handler(self, reader, writer):
addr = str(writer.get_extra_info("peername"))
addr = str(writer.get_extra_info("peername"))
# A new connection, but we can accept no more
if addr not in self.connections and \
len(self.connections)>=self.MAX_CONNECTION:
self.refuse_client(addr, writer)
return
# Add connection
self.add_client(addr, writer)
# Read data from connection
remaining_data = b""
try:
while True:
data = await reader.read(1) # 1024*8 bytes
if not data:
break
except BrokenPipeError:
"""
Catches connecton reset by peer when we are sending the batched data,
which is also when we cannot check for reader. The broken connection
on the writer side will ultimately lead to BrokenPipeError on the
reader side. Hence
"""
pass
finally:
self.remove_client(addr)
async def poll_from_kafka(self):
polled = self.consumer.poll(timeout_ms=self.INTERVAL * 1000 / 2)
self.lastPolled = polled
while True:
t = time()
if t >= self.sample_end_time + self.DELAY:
polled = self.consumer.poll(timeout_ms=self.INTERVAL * 1000 /
2)
lastPolled = self.lastPolled
start_time = self.sample_end_time - self.INTERVAL
end_time = self.sample_end_time
self.lastPolled = polled
self.sample_end_time = self.get_end_time(time())
if len(self.connections) != 0:
# run on lastPolled first to hit the cache
parsed_records = self.get_parsed_records(lastPolled) + \
self.get_parsed_records(polled)
parsed_records = list(
filter(
lambda rec: rec["observation_date_time"] is
not None, parsed_records))
ds_records = self.down_sample(parsed_records, start_time,
end_time)
messages = [rec["message"] for rec in ds_records]
for addr in self.connections.keys():
await self.send_or_ignore_message(addr, messages)
await asyncio.sleep(0.1)
def get_parsed_records(self, polled):
# Create cache
if "_get_parsed_records__polled" not in self.__dict__:
self._get_parsed_records__polled = []
if "_get_parsed_records__ret" not in self.__dict__:
self._get_parsed_records__ret = []
# Cache hit
if self._get_parsed_records__polled == polled:
return self._get_parsed_records__ret
# Cache not hit
self._get_parsed_records__polled = polled
self._get_parsed_records__ret = []
records = []
for recordList in polled.values():
records.extend([rec.value for rec in recordList])
for rec in records:
self._get_parsed_records__ret.append(self.parse_hl7(rec))
return self._get_parsed_records__ret
def parse_hl7(self, message):
segments = message.decode(errors="ignore") \
.strip() \
.split(MESSAGE_SEGMENT_END_BYTE)
location = None
date_time = None
observation_types = []
observation_type = None
for seg in segments:
fields = seg.split('|')
if fields[0] == "PV1":
try:
location = fields[3]
except IndexError:
pass
if fields[0] == "OBR":
try:
date_time = mktime(strptime(fields[7], "%Y%m%d%H%M%S"))
except IndexError:
pass
if fields[0] == "OBX":
try:
observation_types.append(fields[13])
except IndexError:
observation_types.append(None)
observation_type_set = set(observation_types)
if len(observation_type_set)==1 and \
"APERIODIC" in observation_type_set:
observation_type = "aperiodic"
if len(observation_type_set)==1 and \
None in observation_type_set:
observation_type = "default"
return {
"assigned_patient_location": location,
"observation_date_time": date_time,
"observation_type": observation_type,
"message": message
}
def down_sample(self, parsed_records, start_time, end_time):
dt = [rec["observation_date_time"] for rec in parsed_records]
records = []
tmp = {}
sorted_records = sorted(parsed_records,
key=lambda rec: rec["observation_date_time"])
for rec in sorted_records:
date_time = rec["observation_date_time"]
location = rec["assigned_patient_location"]
observation_type = rec["observation_type"]
message = rec["message"]
if date_time<start_time or \
date_time>=end_time:
continue
tmp[location] = tmp.get(location, {})
tmp[location][observation_type] = rec
for d in tmp.values():
for rec in d.values():
records.append(rec)
return records
def log(self, msg):
self.producer.send( self.log_topic,
self.LOG_FORMAT.format( datetime.now().timestamp(),
msg
) \
.encode()
)
def get_end_time(self, current_time):
interval = self.INTERVAL
return current_time - current_time % interval + interval
async def send_or_ignore_message(self, addr, messages):
writer = self.connections[addr]
try:
for msg in messages:
writer.write(msg)
await writer.drain()
except ConnectionResetError:
"""
The error is not thrown reliably. If a connection is broken, and
one try to
writer.write(record)
await writer.drain()
This error may not manifest. It is thrown more often when one try
to repeatedly write to and drain a broken connection.
"""
self.remove_client(addr)
def refuse_client(self, addr, writer):
self.log("{} refused".format(addr))
writer.close()
def add_client(self, addr, writer):
if addr not in self.connections:
self.log("{} accepted".format(addr))
self.connections[addr] = writer
else:
self.remove_client(addr)
self.add_client(addr)
def remove_client(self, addr):
if addr in self.connections:
self.log("{} closed".format(addr))
writer = self.connections.pop(addr)
try:
writer.close()
except ConnectionResetError:
pass
def cleanup(self):
self.log("shutdown")
for addr in self.connections.keys():
self.remove_client(addr)
self.producer.flush()
self.producer.close()
def getMsgData(topic, group, result, maxsize):
try:
saveResult = SaveDataResult()
saveResult.guid = str(uuid.uuid4())
saveResult.CreateDate = datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S")
msgInfos = []
result.guid = saveResult.guid
result.topic_messages = []
consumer = KafkaConsumer(bootstrap_servers=tmpbootstrap_servers,
enable_auto_commit=False,
group_id=group)
# Get all partitions by topic
par = consumer.partitions_for_topic(topic)
now_count = 0
for p in par:
tp = TopicPartition(topic, p)
consumer.assign([tp])
print(tp)
info = MsgPartitionInfo()
# Get committed offset
print('start to get committed offset.....')
try:
committed = consumer.committed(tp) or 0
except Exception, e_commit:
print(str(e_commit))
# Move consumer to end to get the last position
consumer.seek_to_end(tp)
last_offset = consumer.position(tp)
# Move consumer to beginning to get the first position
consumer.seek_to_beginning()
now_offset = consumer.position(tp)
from_offset = committed
if from_offset is None:
from_offset = now_offset
if from_offset < now_offset:
from_offset = now_offset
info.partition_ID = tp.partition
info.get_last_offset = last_offset
msgInfos.append(info)
print("[%s] partition(%s) -> now:%s, last:%s, committed:%s" %
(tp.topic, tp.partition, now_offset, last_offset, committed))
# Get msg from position to offset
while (from_offset < last_offset) and (now_count < maxsize):
consumer.seek(tp, from_offset)
polldata = consumer.poll(100)
from_offset += 1
now_count += 1
print('now_count=' + str(now_count))
result.topic_messages.append(polldata[tp][0].value)
saveResult.MsgInfo = json.dumps(msgInfos,
default=encode_MsgPartitionInfo,
ensure_ascii=False)
print(saveResult.MsgInfo)
consumer.close()
saveResult.message = "Success"
saveResult.Code = 200
producer = KafkaProducer(bootstrap_servers=tmpbootstrap_servers)
producer.send(topic + "_log",
json.dumps(saveResult, default=encode_SaveDataResult))
producer.flush()
class IBUSStreamingDownsamplingConsumer:
LOG_FORMAT ="{} UTC_TS\t"\
"{}"
def __init__(self, kafkaHost, kafkaPort, tcpHost, tcpPort, group_id, topic,
logTopic, interval):
self.kafkaHost = kafkaHost
self.kafkaPort = kafkaPort
self.tcpHost = tcpHost
self.tcpPort = tcpPort
self.group_id = group_id
self.topic = topic
self.logTopic = logTopic
self.interval = int(interval)
self.consumer = KafkaConsumer(
topic,
bootstrap_servers=["{}:{}".format(kafkaHost, kafkaPort)],
group_id=group_id,
enable_auto_commit=False)
self.producer = KafkaProducer(
bootstrap_servers=["{}:{}".format(kafkaHost, kafkaPort)])
self.tcpWriter = None
def getTopicPartitions(self):
self.consumer.topics() #This ensures local cache is updated with
# information about partitions, offsets etc.
pids = self.consumer.partitions_for_topic(self.topic)
tps = [TopicPartition(self.topic, pid) for pid in pids]
return tps
def getTopicPartitionsCommittedPositions(self):
tps = self.getTopicPartitions()
ret = [(tp, self.consumer.committed(tp)) for tp in tps]
return ret
async def tcp_server_handler(self, reader, writer):
addr = str(writer.get_extra_info("socket").getpeername())
if self.tcpWriter is not None:
self.log("refused " + addr)
writer.write(b"Connection limit reached; connection refused.")
writer.close()
return
self.log("accepted " + addr)
self.tcpWriter = writer
t1 = asyncio.create_task(self.poll_from_Kafka(writer))
try:
while True:
data = await reader.read(1) # 1024*16 bytes
if not data:
break
except BrokenPipeError:
"""
Catches connecton reset by peer when we are sending the batched data,
which is also when we cannot check for reader. The broken connection
on the writer side will ultimately lead to BrokenPipeError on the
reader side. Hence
"""
pass
finally:
t1.cancel()
self.log("closed " + addr)
writer.close()
self.tcpWriter = None
async def poll_from_Kafka(self, writer):
while True:
prevPos = self.getTopicPartitionsCommittedPositions()
polled = self.consumer.poll(timeout_ms=1000)
records = [
record.value for recordList in polled.values()
for record in recordList
]
try:
for record in records:
writer.write(record)
await writer.drain()
except ConnectionResetError:
"""
The error is not thrown reliably. If a connection is broken, and
one try to
writer.write(record)
await writer.drain()
This error may not manifest. It is thrown more often when one try
to repeatedly write to and drain a broken connection.
"""
print("Last batch not fully sent, not commited.")
for tp, pos in prevPos:
self.consumer.seek(tp, pos)
break
else:
self.consumer.commit()
await asyncio.sleep(self.interval)
def log(self, msg):
self.producer.send( self.logTopic,
self.LOG_FORMAT.format( datetime.now().timestamp(),
msg
) \
.encode()
)
def cleanup(self):
self.log("shutdown")
self.consumer.close()
self.producer.flush()
self.producer.close()
def run(self):
self.log("running")
asyncio.run(self._async_run())
async def _async_run(self):
tcpServer = await asyncio.start_server(self.tcp_server_handler,
self.tcpHost, self.tcpPort)
await tcpServer.serve_forever()