def process_message(self, username, message): self.from_user = username at_ = re.findall(r'\@(\b.+?\b)', message) if at_: at_ = at_[0] message = message.replace(at_, "") msg = TwitchFormatter.format(message) if not len(msg): return now = time.time() self.processed += 1 self.message_bin.append(msg) self.messages_sec = self.processed / (now - self.start_time) log.info("{} in bin, {:.002f} msg/sec".format(len(self.message_bin), self.messages_sec)) if ((len(self.message_bin) >= self.chatter_level) and ((now - self.last_chatter) > self.chatter_level)): self.direct_message = False self.trigger(self._get_random_message()) elif at_ == config.get('TWITCH_NICK'): self.direct_message = True message = message.strip('@' + config.get('TWITCH_NICK')) self.trigger(message, forced=False)
def connect(self): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) connect_host = "irc.twitch.tv" connect_port = 6667 try: s.connect((connect_host, connect_port)) except (Exception, IOError): print("Unable to create a socket to %s:%s" % (connect_host, connect_port)) raise # unexpected, because it is a blocking socket s.send(('PASS %s\r\n' % self.oauth).encode('utf-8')) s.send(('NICK %s\r\n' % self.username).encode('utf-8')) log.debug("Sent PASS and NICK") received = s.recv(1024).decode() log.debug("{}".format(received.strip("\r\n"))) if not TwitchChat._logged_in_successful(received): raise IOError("Twitch did not accept the username-oauth " "combination") else: log.debug("Connected. Taking blocking socket into non-blocking") fcntl.fcntl(s, fcntl.F_SETFL, os.O_NONBLOCK) if self.s is not None: log.info("Closed socket :(") self.s.close() self.s = s self.join_channel(self.channel) while self.current_channel != self.channel: self.twitch_receive_messages()
def waiting(self): self.elapsed_chatter_time = time.time() if (self.elapsed_chatter_time - self.last_chatter > self.max_waiting_time and self.last_chatter != 0): if len(self.message_bin): log.info("I've waited long enough ({:.02f}/secs). " "I'm chatting.".format(self.elapsed_chatter_time - self.last_chatter)) self.trigger(self._get_random_message(), forced=True)
def calculate_unique_distribution(self): bin_len = len(self.message_bin) counter = collections.Counter( map(lambda x: x.lower(), self.message_bin)) most_common_count = counter.most_common(1)[0][1] r = most_common_count / bin_len log.info("{0:.2f}% of {1:.2f}% threshold".format( r * 100, self.unique_threshold * 100)) return r
def collapse_message(self, msg): m_split = msg.lower().split(" ") bin_len = len(m_split) unique = len(list(set(map(lambda x: x.lower(), m_split)))) if bin_len > 1 and unique == 1: log.info("DUPLICATE SPAM {0}".format(msg, m_split[0])) return m_split[0] else: return msg
def process_spam(self, message): end_time = time.time() diff = end_time - self.start_time if len(self.message_bin) < self.minimum_population: log.info("accumulating bin {0:.2f}/seconds".format(diff)) self.message_bin.append(self.collapse_message(message)) unique_perc = self.calculate_unique_distribution() if len(self.message_bin) > self.minimum_population: self.message_bin.pop(0) if unique_perc > self.unique_threshold: if diff > self.distribution_length_ms: self.triggered = True self.set_message()
def run(self): while not self.stop_event.is_set(): received = self.twitch_receive_messages() if received: username = received[0]["username"] msg = received[0]["message"] channel = received[0]["channel"].replace("#", "") try: data = { 'channel': channel, 'username': username, 'message': msg } self.chat_queue.put_nowait(data) log.info("({0}) {1}: {2}".format(channel, username, msg)) except: self.close() while not self.command_queue.empty(): data = self.command_queue.get_nowait() if data is None: return try: data = json.loads(data) message = data.get('message') self.send_chat_message(message) log.debug("===Sent chat message {}===".format(message)) except: raise self.sent += 1 self.command_queue.task_done() while not self.admin_command_queue.empty(): data = self.admin_command_queue.get_nowait() if data is None: return try: data = json.loads(data) message = data.get('message') # TODO: assuming join command self.notify_channel(message) log.debug("===ADMIN JOIN CHANNEL {}===".format(message)) except: raise self.admin_command_queue.task_done() time.sleep(0.01)
def nlp_answer(data, question, bot_message_queue, doc_file): nlp_answer.response = None nlp_answer.algo = TFIDF() nlp_answer.at_threshold = 0.10 log.info("Calculating Response...") # dont think will work.. # @cached(cache=TTLCache(maxsize=2000000, ttl=600)) def load_doc(file_name): file = open(file_name, 'r') nlp_answer.algo.set_doc(file.read().split("\n")) file.close() load_doc(doc_file) try: response = nlp_answer.algo.answer(question) except: raise if response is not None: # TODO: need a way to get global config # username = None # if data.get('username'): # username = data.get('username') # else: # r = random.randint(1, 100) / 100 # if r <= nlp_answer.at_threshold: # username = config.get('TWITCH_CHANNEL') # # if username: # response = ( # '@' + username + " " + response # ) log.info("New Response: {}".format(response)) data_to_send = twitch_schema.as_dict(data.get('channel'), response) bot_message_queue.put_nowait(data_to_send) else: log.info("No Response found.")
import threading import json from talis import config from talis import log from talis import push_queue from talis import dequeue from talis import TwitchChat from kafka import KafkaConsumer from kafka import KafkaProducer if __name__ == "__main__": config.add_oauth() log.setLevel(config.log_level()) log.info("=== Twitch Bot Started ===") chat_queue = queue.Queue() bot_message_queue = queue.Queue() admin_command_queue = queue.Queue() stop_event = threading.Event() kafka_consumer = KafkaConsumer( config.get("KAFKA_BOT_MESSAGE_TOPIC"), bootstrap_servers=config.get("KAFKA_BOOTSTRAP_HOST"), auto_offset_reset="latest") admin_kafka_consumer = KafkaConsumer( "central_control", bootstrap_servers=config.get("KAFKA_BOOTSTRAP_HOST"), auto_offset_reset="latest")
def wiki_answer(data, question, bot_message_queue): wiki_answer.wiki_answer = Wikipedia() wiki_answer.algo = TFIDF() wiki_answer.response = None try: subject = subject_parser(question) log.info("Found subject {}".format(subject)) if not subject: return log.info("Received subject {}".format(subject)) wiki_answer.algo.set_data(wiki_answer.wiki_answer.get_content(subject)) log.info("Processed subject") log.info("Getting Answer") wiki_answer.response = wiki_answer.algo.answer(question) log.info("Got Answer {}".format(wiki_answer.response)) except: raise if wiki_answer.response: data_answer = twitch_schema.as_dict( data.get('channel'), wiki_answer.response ) bot_message_queue.put_nowait(data_answer) log.info('Wiki processed') else: log.info("No response {}".format(wiki_answer.response))
try: kp_thread.start() while not stop_event.is_set(): for msg in kafka_consumer: data = json.loads(msg.value) message = data.get('message') split_message = re.findall(r'(^!q )(.+)', message) if split_message and len(split_message[0]) == 2: split_message = split_message[0] command = split_message[0].strip() question = split_message[1] log.info("Question: {}".format(question)) threading.Thread(target=wiki_answer, args=( data, question, bot_message_queue, ), name="wiki consumer thread").start() except (KeyboardInterrupt, SystemExit): stop_event.set() raise except: stop_event.set() raise
config.get('KAFKA_BOT_MESSAGE_TOPIC'), bot_message_queue), name="Kafka Chat Producer") kp_thread.setDaemon(True) try: kp_thread.start() spam = SpamFilter(config.get('minimum_population', 4), config.get('unique_threshold', .5), config.get('distribution_length_sec', 1)) while not stop_event.is_set(): for msg in kafka_consumer: data = json.loads(msg.value) message = data.get('message') log.info("Got latest message {}".format(message)) spam.process_spam(message) if spam.triggered: log.info("Found spam") send_to_bot = twitch_schema.as_dict( data.get('channel'), spam.message) bot_message_queue.put_nowait(send_to_bot) spam.reset() except (KeyboardInterrupt, SystemExit): stop_event.set() raise except: stop_event.set() raise
def set_message(self): counter = collections.Counter(self.message_bin) self.message = counter.most_common(1)[0][0] log.info("====== SPAM TRIGGER ======") log.info("{0}".format(self.message)) log.info("====== END TRIGGER ======")
def _get_random_message(self): random_int = random.randint(0, len(self.message_bin) - 1) log.info("Found rand int for message {}".format(random_int)) return self.message_bin[random_int]
args=( kafka_producer, config.get('KAFKA_BOT_MESSAGE_TOPIC'), bot_message_queue ), name="Kafka Chat Producer" ) kp_thread.setDaemon(True) try: kp_thread.start() while not stop_event.is_set(): for msg in kafka_consumer: data = json.loads(msg.value) command = data.get('message') log.info("Got latest message {}".format(command)) if command in commands.keys(): log.info("Found comand") response = commands[command] send_to_bot = twitch_schema.as_dict( data.get('channel'), response ) bot_message_queue.put_nowait(send_to_bot) except (KeyboardInterrupt, SystemExit): stop_event.set() pass except: stop_event.set() raise
def leave_channel(self, channel): self.s.send(('PART #%s\r\n' % channel).encode('utf-8')) self.current_channel = None log.info("LEFT {}".format(channel))
def join_channel(self, channel): self.s.send(('JOIN #%s\r\n' % channel).encode('utf-8')) self.current_channel = channel log.info("JOINED {}".format(channel))
# bot_message_queue kp_thread = threading.Thread(target=dequeue, args=(kafka_producer, config.get('KAFKA_BOT_MESSAGE_TOPIC'), bot_message_queue), name="Kafka Chat Producer") kp_thread.setDaemon(True) try: kp_thread.start() users = [] while not stop_event.is_set(): for msg in kafka_consumer: data = json.loads(msg.value) username = data.get('username') command = data.get('message') if username not in users: send_to_bot = twitch_schema.as_dict( data.get('channel'), "/timeout {}".format(username.lower())) log.info("Timed out {}".format(username)) bot_message_queue.put_nowait(send_to_bot) users.append(username) except (KeyboardInterrupt, SystemExit): stop_event.set() pass except: stop_event.set() raise
def trigger(self, question, forced=False): self.triggered = True self.question = question self.forced_chatter = forced log.info("Analyzing Message: {}".format(question))