def process_message(self, username, message):
self.from_user = username
at_ = re.findall(r'\@(\b.+?\b)', message)
if at_:
at_ = at_[0]
message = message.replace(at_, "")
msg = TwitchFormatter.format(message)
if not len(msg):
return
now = time.time()
self.processed += 1
self.message_bin.append(msg)
self.messages_sec = self.processed / (now - self.start_time)
log.info("{} in bin, {:.002f} msg/sec".format(len(self.message_bin),
self.messages_sec))
if ((len(self.message_bin) >= self.chatter_level)
and ((now - self.last_chatter) > self.chatter_level)):
self.direct_message = False
self.trigger(self._get_random_message())
elif at_ == config.get('TWITCH_NICK'):
self.direct_message = True
message = message.strip('@' + config.get('TWITCH_NICK'))
self.trigger(message, forced=False)
'''
Use this script to debug a kafka topic.
'''
import os
import sys
import json
sys.path.append(os.path.dirname(os.path.realpath(__name__)))
from talis import config
from talis import log
from kafka import KafkaConsumer
if __name__ == "__main__":
kafka_consumer = KafkaConsumer(
config.get("topic", config.get('KAFKA_TOPIC')),
bootstrap_servers=config.get("KAFKA_BOOTSTRAP_HOST"),
auto_offset_reset=config.get('auto_offset_reset', 'latest'))
for msg in kafka_consumer:
data = json.loads(msg.value)
print(msg)
from talis import config
from talis import log
from talis import push_queue
from talis import dequeue
from kafka import KafkaConsumer
from kafka import KafkaProducer
from talis import twitch_schema
if __name__ == "__main__":
chat_queue = queue.Queue()
bot_message_queue = queue.Queue()
stop_event = threading.Event()
kafka_consumer = KafkaConsumer(
config.get("KAFKA_TOPIC"),
bootstrap_servers=config.get("KAFKA_BOOTSTRAP_HOST"),
auto_offset_reset="latest")
kafka_producer = KafkaProducer(
bootstrap_servers=config.get('KAFKA_BOOTSTRAP_HOST'),
value_serializer=lambda v: json.dumps(v).encode('utf-8'))
# Pushes commands to the bot from the
# bot_message_queue
kp_thread = threading.Thread(target=dequeue,
args=(kafka_producer,
config.get('KAFKA_BOT_MESSAGE_TOPIC'),
bot_message_queue),
name="Kafka Chat Producer")
kp_thread.setDaemon(True)
# TODO: Add compression
# TODO: Add customer arg
'''
import queue
import threading
import os
import sys
import json
sys.path.append(os.path.dirname(os.path.realpath(__name__)))
from talis import config
from talis import log
from kafka import KafkaConsumer
if __name__ == "__main__":
kafka_consumer = KafkaConsumer(
config.get("KAFKA_TOPIC"),
bootstrap_servers=config.get("KAFKA_BOOTSTRAP_HOST"),
auto_offset_reset='earliest',
consumer_timeout_ms=300
)
with open('data/kafka_output.txt', 'w') as file:
for msg in kafka_consumer:
data = json.loads(msg.value)
message = data.get('message')
file.write(f"{message}\n")
for nlp processing
'''
import os
import sys
import json
sys.path.append(os.path.dirname(os.path.realpath(__name__)))
from talis import config
from talis import log
from talis import TwitchFormatter
from kafka import KafkaConsumer
if __name__ == "__main__":
print(config.get('KAFKA_TOPIC'))
twitch_formatter = TwitchFormatter()
consumer = KafkaConsumer(
config.get('KAFKA_TOPIC'),
bootstrap_servers=config.get('KAFKA_BOOTSTRAP_HOST'),
auto_offset_reset=config.get('auto_offset_reset', 'earliest'),
consumer_timeout_ms=1000)
with open('data/twitch_doc_tmp.txt', 'w') as fh:
for msg in consumer:
data = json.loads(msg.value)
message = data.get('message')
formatted = twitch_formatter.format(message)
if len(formatted):
from kafka import KafkaConsumer
from kafka import KafkaProducer
if __name__ == "__main__":
config.add_oauth()
log.setLevel(config.log_level())
log.info("=== Twitch Bot Started ===")
chat_queue = queue.Queue()
bot_message_queue = queue.Queue()
admin_command_queue = queue.Queue()
stop_event = threading.Event()
kafka_consumer = KafkaConsumer(
config.get("KAFKA_BOT_MESSAGE_TOPIC"),
bootstrap_servers=config.get("KAFKA_BOOTSTRAP_HOST"),
auto_offset_reset="latest")
admin_kafka_consumer = KafkaConsumer(
"central_control",
bootstrap_servers=config.get("KAFKA_BOOTSTRAP_HOST"),
auto_offset_reset="latest")
kafka_producer = KafkaProducer(
bootstrap_servers=config.get('KAFKA_BOOTSTRAP_HOST'),
value_serializer=lambda v: json.dumps(v).encode('utf-8'))
kc_thread = threading.Thread(target=push_queue,
args=(kafka_consumer, bot_message_queue,
stop_event),
from talis import push_queue
from talis import dequeue
from talis import SpamFilter
from talis import twitch_schema
from kafka import KafkaConsumer
from kafka import KafkaProducer
if __name__ == "__main__":
# The commands (spam) to send to the botKappa
bot_message_queue = queue.Queue()
stop_event = threading.Event()
kafka_consumer = KafkaConsumer(
config.get("KAFKA_TOPIC"),
bootstrap_servers=config.get("KAFKA_BOOTSTRAP_HOST"),
auto_offset_reset="latest")
kafka_producer = KafkaProducer(
bootstrap_servers=config.get('KAFKA_BOOTSTRAP_HOST'),
value_serializer=lambda v: json.dumps(v).encode('utf-8'))
# Pushes commands to the bot from the
# bot_message_queue
kp_thread = threading.Thread(target=dequeue,
args=(kafka_producer,
config.get('KAFKA_BOT_MESSAGE_TOPIC'),
bot_message_queue),
name="Kafka Chat Producer")
kp_thread.setDaemon(True)
import json
import re
sys.path.append(os.path.dirname(os.path.realpath(__name__)))
from talis import config
from talis import log
from kafka import KafkaProducer
from kafka import KafkaConsumer
from talis import twitch_schema
if __name__ == "__main__":
kafka_producer = KafkaProducer(
bootstrap_servers=config.get('KAFKA_BOOTSTRAP_HOST'),
value_serializer=lambda v: json.dumps(v).encode('utf-8'))
try:
while True:
command = input("What Message?\n")
central_control = re.findall(r'^(:join: )(.+)', command)
print(central_control)
if central_control and len(central_control[0]) == 2:
print("found parsed ADMIN command")
central_control = central_control[0]
command = central_control[0]
channel = central_control[1]
kafka_topic = "central_control"
data = twitch_schema.as_dict(config.get('TWITCH_CHANNEL'),
