def process_message(self, username, message): self.from_user = username at_ = re.findall(r'\@(\b.+?\b)', message) if at_: at_ = at_[0] message = message.replace(at_, "") msg = TwitchFormatter.format(message) if not len(msg): return now = time.time() self.processed += 1 self.message_bin.append(msg) self.messages_sec = self.processed / (now - self.start_time) log.info("{} in bin, {:.002f} msg/sec".format(len(self.message_bin), self.messages_sec)) if ((len(self.message_bin) >= self.chatter_level) and ((now - self.last_chatter) > self.chatter_level)): self.direct_message = False self.trigger(self._get_random_message()) elif at_ == config.get('TWITCH_NICK'): self.direct_message = True message = message.strip('@' + config.get('TWITCH_NICK')) self.trigger(message, forced=False)
''' Use this script to debug a kafka topic. ''' import os import sys import json sys.path.append(os.path.dirname(os.path.realpath(__name__))) from talis import config from talis import log from kafka import KafkaConsumer if __name__ == "__main__": kafka_consumer = KafkaConsumer( config.get("topic", config.get('KAFKA_TOPIC')), bootstrap_servers=config.get("KAFKA_BOOTSTRAP_HOST"), auto_offset_reset=config.get('auto_offset_reset', 'latest')) for msg in kafka_consumer: data = json.loads(msg.value) print(msg)
from talis import config from talis import log from talis import push_queue from talis import dequeue from kafka import KafkaConsumer from kafka import KafkaProducer from talis import twitch_schema if __name__ == "__main__": chat_queue = queue.Queue() bot_message_queue = queue.Queue() stop_event = threading.Event() kafka_consumer = KafkaConsumer( config.get("KAFKA_TOPIC"), bootstrap_servers=config.get("KAFKA_BOOTSTRAP_HOST"), auto_offset_reset="latest") kafka_producer = KafkaProducer( bootstrap_servers=config.get('KAFKA_BOOTSTRAP_HOST'), value_serializer=lambda v: json.dumps(v).encode('utf-8')) # Pushes commands to the bot from the # bot_message_queue kp_thread = threading.Thread(target=dequeue, args=(kafka_producer, config.get('KAFKA_BOT_MESSAGE_TOPIC'), bot_message_queue), name="Kafka Chat Producer") kp_thread.setDaemon(True)
# TODO: Add compression # TODO: Add customer arg ''' import queue import threading import os import sys import json sys.path.append(os.path.dirname(os.path.realpath(__name__))) from talis import config from talis import log from kafka import KafkaConsumer if __name__ == "__main__": kafka_consumer = KafkaConsumer( config.get("KAFKA_TOPIC"), bootstrap_servers=config.get("KAFKA_BOOTSTRAP_HOST"), auto_offset_reset='earliest', consumer_timeout_ms=300 ) with open('data/kafka_output.txt', 'w') as file: for msg in kafka_consumer: data = json.loads(msg.value) message = data.get('message') file.write(f"{message}\n")
for nlp processing ''' import os import sys import json sys.path.append(os.path.dirname(os.path.realpath(__name__))) from talis import config from talis import log from talis import TwitchFormatter from kafka import KafkaConsumer if __name__ == "__main__": print(config.get('KAFKA_TOPIC')) twitch_formatter = TwitchFormatter() consumer = KafkaConsumer( config.get('KAFKA_TOPIC'), bootstrap_servers=config.get('KAFKA_BOOTSTRAP_HOST'), auto_offset_reset=config.get('auto_offset_reset', 'earliest'), consumer_timeout_ms=1000) with open('data/twitch_doc_tmp.txt', 'w') as fh: for msg in consumer: data = json.loads(msg.value) message = data.get('message') formatted = twitch_formatter.format(message) if len(formatted):
from kafka import KafkaConsumer from kafka import KafkaProducer if __name__ == "__main__": config.add_oauth() log.setLevel(config.log_level()) log.info("=== Twitch Bot Started ===") chat_queue = queue.Queue() bot_message_queue = queue.Queue() admin_command_queue = queue.Queue() stop_event = threading.Event() kafka_consumer = KafkaConsumer( config.get("KAFKA_BOT_MESSAGE_TOPIC"), bootstrap_servers=config.get("KAFKA_BOOTSTRAP_HOST"), auto_offset_reset="latest") admin_kafka_consumer = KafkaConsumer( "central_control", bootstrap_servers=config.get("KAFKA_BOOTSTRAP_HOST"), auto_offset_reset="latest") kafka_producer = KafkaProducer( bootstrap_servers=config.get('KAFKA_BOOTSTRAP_HOST'), value_serializer=lambda v: json.dumps(v).encode('utf-8')) kc_thread = threading.Thread(target=push_queue, args=(kafka_consumer, bot_message_queue, stop_event),
from talis import push_queue from talis import dequeue from talis import SpamFilter from talis import twitch_schema from kafka import KafkaConsumer from kafka import KafkaProducer if __name__ == "__main__": # The commands (spam) to send to the botKappa bot_message_queue = queue.Queue() stop_event = threading.Event() kafka_consumer = KafkaConsumer( config.get("KAFKA_TOPIC"), bootstrap_servers=config.get("KAFKA_BOOTSTRAP_HOST"), auto_offset_reset="latest") kafka_producer = KafkaProducer( bootstrap_servers=config.get('KAFKA_BOOTSTRAP_HOST'), value_serializer=lambda v: json.dumps(v).encode('utf-8')) # Pushes commands to the bot from the # bot_message_queue kp_thread = threading.Thread(target=dequeue, args=(kafka_producer, config.get('KAFKA_BOT_MESSAGE_TOPIC'), bot_message_queue), name="Kafka Chat Producer") kp_thread.setDaemon(True)
import json import re sys.path.append(os.path.dirname(os.path.realpath(__name__))) from talis import config from talis import log from kafka import KafkaProducer from kafka import KafkaConsumer from talis import twitch_schema if __name__ == "__main__": kafka_producer = KafkaProducer( bootstrap_servers=config.get('KAFKA_BOOTSTRAP_HOST'), value_serializer=lambda v: json.dumps(v).encode('utf-8')) try: while True: command = input("What Message?\n") central_control = re.findall(r'^(:join: )(.+)', command) print(central_control) if central_control and len(central_control[0]) == 2: print("found parsed ADMIN command") central_control = central_control[0] command = central_control[0] channel = central_control[1] kafka_topic = "central_control" data = twitch_schema.as_dict(config.get('TWITCH_CHANNEL'),