def process_message(self, username, message):
self.from_user = username
at_ = re.findall(r'\@(\b.+?\b)', message)
if at_:
at_ = at_[0]
message = message.replace(at_, "")
msg = TwitchFormatter.format(message)
if not len(msg):
return
now = time.time()
self.processed += 1
self.message_bin.append(msg)
self.messages_sec = self.processed / (now - self.start_time)
log.info("{} in bin, {:.002f} msg/sec".format(len(self.message_bin),
self.messages_sec))
if ((len(self.message_bin) >= self.chatter_level)
and ((now - self.last_chatter) > self.chatter_level)):
self.direct_message = False
self.trigger(self._get_random_message())
elif at_ == config.get('TWITCH_NICK'):
self.direct_message = True
message = message.strip('@' + config.get('TWITCH_NICK'))
self.trigger(message, forced=False)
def connect(self):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
connect_host = "irc.twitch.tv"
connect_port = 6667
try:
s.connect((connect_host, connect_port))
except (Exception, IOError):
print("Unable to create a socket to %s:%s" %
(connect_host, connect_port))
raise # unexpected, because it is a blocking socket
s.send(('PASS %s\r\n' % self.oauth).encode('utf-8'))
s.send(('NICK %s\r\n' % self.username).encode('utf-8'))
log.debug("Sent PASS and NICK")
received = s.recv(1024).decode()
log.debug("{}".format(received.strip("\r\n")))
if not TwitchChat._logged_in_successful(received):
raise IOError("Twitch did not accept the username-oauth "
"combination")
else:
log.debug("Connected. Taking blocking socket into non-blocking")
fcntl.fcntl(s, fcntl.F_SETFL, os.O_NONBLOCK)
if self.s is not None:
log.info("Closed socket :(")
self.s.close()
self.s = s
self.join_channel(self.channel)
while self.current_channel != self.channel:
self.twitch_receive_messages()
def waiting(self):
self.elapsed_chatter_time = time.time()
if (self.elapsed_chatter_time - self.last_chatter >
self.max_waiting_time and self.last_chatter != 0):
if len(self.message_bin):
log.info("I've waited long enough ({:.02f}/secs). "
"I'm chatting.".format(self.elapsed_chatter_time -
self.last_chatter))
self.trigger(self._get_random_message(), forced=True)
def calculate_unique_distribution(self):
bin_len = len(self.message_bin)
counter = collections.Counter(
map(lambda x: x.lower(), self.message_bin))
most_common_count = counter.most_common(1)[0][1]
r = most_common_count / bin_len
log.info("{0:.2f}% of {1:.2f}% threshold".format(
r * 100, self.unique_threshold * 100))
return r
def collapse_message(self, msg):
m_split = msg.lower().split(" ")
bin_len = len(m_split)
unique = len(list(set(map(lambda x: x.lower(), m_split))))
if bin_len > 1 and unique == 1:
log.info("DUPLICATE SPAM {0}".format(msg, m_split[0]))
return m_split[0]
else:
return msg
def process_spam(self, message):
end_time = time.time()
diff = end_time - self.start_time
if len(self.message_bin) < self.minimum_population:
log.info("accumulating bin {0:.2f}/seconds".format(diff))
self.message_bin.append(self.collapse_message(message))
unique_perc = self.calculate_unique_distribution()
if len(self.message_bin) > self.minimum_population:
self.message_bin.pop(0)
if unique_perc > self.unique_threshold:
if diff > self.distribution_length_ms:
self.triggered = True
self.set_message()
def run(self):
while not self.stop_event.is_set():
received = self.twitch_receive_messages()
if received:
username = received[0]["username"]
msg = received[0]["message"]
channel = received[0]["channel"].replace("#", "")
try:
data = {
'channel': channel,
'username': username,
'message': msg
}
self.chat_queue.put_nowait(data)
log.info("({0}) {1}: {2}".format(channel, username, msg))
except:
self.close()
while not self.command_queue.empty():
data = self.command_queue.get_nowait()
if data is None:
return
try:
data = json.loads(data)
message = data.get('message')
self.send_chat_message(message)
log.debug("===Sent chat message {}===".format(message))
except:
raise
self.sent += 1
self.command_queue.task_done()
while not self.admin_command_queue.empty():
data = self.admin_command_queue.get_nowait()
if data is None:
return
try:
data = json.loads(data)
message = data.get('message')
# TODO: assuming join command
self.notify_channel(message)
log.debug("===ADMIN JOIN CHANNEL {}===".format(message))
except:
raise
self.admin_command_queue.task_done()
time.sleep(0.01)
def nlp_answer(data, question, bot_message_queue, doc_file):
nlp_answer.response = None
nlp_answer.algo = TFIDF()
nlp_answer.at_threshold = 0.10
log.info("Calculating Response...")
# dont think will work..
# @cached(cache=TTLCache(maxsize=2000000, ttl=600))
def load_doc(file_name):
file = open(file_name, 'r')
nlp_answer.algo.set_doc(file.read().split("\n"))
file.close()
load_doc(doc_file)
try:
response = nlp_answer.algo.answer(question)
except:
raise
if response is not None:
# TODO: need a way to get global config
# username = None
# if data.get('username'):
# username = data.get('username')
# else:
# r = random.randint(1, 100) / 100
# if r <= nlp_answer.at_threshold:
# username = config.get('TWITCH_CHANNEL')
#
# if username:
# response = (
# '@' + username + " " + response
# )
log.info("New Response: {}".format(response))
data_to_send = twitch_schema.as_dict(data.get('channel'), response)
bot_message_queue.put_nowait(data_to_send)
else:
log.info("No Response found.")
import threading
import json
from talis import config
from talis import log
from talis import push_queue
from talis import dequeue
from talis import TwitchChat
from kafka import KafkaConsumer
from kafka import KafkaProducer
if __name__ == "__main__":
config.add_oauth()
log.setLevel(config.log_level())
log.info("=== Twitch Bot Started ===")
chat_queue = queue.Queue()
bot_message_queue = queue.Queue()
admin_command_queue = queue.Queue()
stop_event = threading.Event()
kafka_consumer = KafkaConsumer(
config.get("KAFKA_BOT_MESSAGE_TOPIC"),
bootstrap_servers=config.get("KAFKA_BOOTSTRAP_HOST"),
auto_offset_reset="latest")
admin_kafka_consumer = KafkaConsumer(
"central_control",
bootstrap_servers=config.get("KAFKA_BOOTSTRAP_HOST"),
auto_offset_reset="latest")
def wiki_answer(data, question, bot_message_queue):
wiki_answer.wiki_answer = Wikipedia()
wiki_answer.algo = TFIDF()
wiki_answer.response = None
try:
subject = subject_parser(question)
log.info("Found subject {}".format(subject))
if not subject:
return
log.info("Received subject {}".format(subject))
wiki_answer.algo.set_data(wiki_answer.wiki_answer.get_content(subject))
log.info("Processed subject")
log.info("Getting Answer")
wiki_answer.response = wiki_answer.algo.answer(question)
log.info("Got Answer {}".format(wiki_answer.response))
except:
raise
if wiki_answer.response:
data_answer = twitch_schema.as_dict(
data.get('channel'),
wiki_answer.response
)
bot_message_queue.put_nowait(data_answer)
log.info('Wiki processed')
else:
log.info("No response {}".format(wiki_answer.response))
try:
kp_thread.start()
while not stop_event.is_set():
for msg in kafka_consumer:
data = json.loads(msg.value)
message = data.get('message')
split_message = re.findall(r'(^!q )(.+)', message)
if split_message and len(split_message[0]) == 2:
split_message = split_message[0]
command = split_message[0].strip()
question = split_message[1]
log.info("Question: {}".format(question))
threading.Thread(target=wiki_answer,
args=(
data,
question,
bot_message_queue,
),
name="wiki consumer thread").start()
except (KeyboardInterrupt, SystemExit):
stop_event.set()
raise
except:
stop_event.set()
raise
config.get('KAFKA_BOT_MESSAGE_TOPIC'),
bot_message_queue),
name="Kafka Chat Producer")
kp_thread.setDaemon(True)
try:
kp_thread.start()
spam = SpamFilter(config.get('minimum_population', 4),
config.get('unique_threshold', .5),
config.get('distribution_length_sec', 1))
while not stop_event.is_set():
for msg in kafka_consumer:
data = json.loads(msg.value)
message = data.get('message')
log.info("Got latest message {}".format(message))
spam.process_spam(message)
if spam.triggered:
log.info("Found spam")
send_to_bot = twitch_schema.as_dict(
data.get('channel'), spam.message)
bot_message_queue.put_nowait(send_to_bot)
spam.reset()
except (KeyboardInterrupt, SystemExit):
stop_event.set()
raise
except:
stop_event.set()
raise
def set_message(self):
counter = collections.Counter(self.message_bin)
self.message = counter.most_common(1)[0][0]
log.info("====== SPAM TRIGGER ======")
log.info("{0}".format(self.message))
log.info("====== END TRIGGER ======")
def _get_random_message(self):
random_int = random.randint(0, len(self.message_bin) - 1)
log.info("Found rand int for message {}".format(random_int))
return self.message_bin[random_int]
args=(
kafka_producer,
config.get('KAFKA_BOT_MESSAGE_TOPIC'),
bot_message_queue
),
name="Kafka Chat Producer"
)
kp_thread.setDaemon(True)
try:
kp_thread.start()
while not stop_event.is_set():
for msg in kafka_consumer:
data = json.loads(msg.value)
command = data.get('message')
log.info("Got latest message {}".format(command))
if command in commands.keys():
log.info("Found comand")
response = commands[command]
send_to_bot = twitch_schema.as_dict(
data.get('channel'),
response
)
bot_message_queue.put_nowait(send_to_bot)
except (KeyboardInterrupt, SystemExit):
stop_event.set()
pass
except:
stop_event.set()
raise
def leave_channel(self, channel):
self.s.send(('PART #%s\r\n' % channel).encode('utf-8'))
self.current_channel = None
log.info("LEFT {}".format(channel))
def join_channel(self, channel):
self.s.send(('JOIN #%s\r\n' % channel).encode('utf-8'))
self.current_channel = channel
log.info("JOINED {}".format(channel))
# bot_message_queue
kp_thread = threading.Thread(target=dequeue,
args=(kafka_producer,
config.get('KAFKA_BOT_MESSAGE_TOPIC'),
bot_message_queue),
name="Kafka Chat Producer")
kp_thread.setDaemon(True)
try:
kp_thread.start()
users = []
while not stop_event.is_set():
for msg in kafka_consumer:
data = json.loads(msg.value)
username = data.get('username')
command = data.get('message')
if username not in users:
send_to_bot = twitch_schema.as_dict(
data.get('channel'),
"/timeout {}".format(username.lower()))
log.info("Timed out {}".format(username))
bot_message_queue.put_nowait(send_to_bot)
users.append(username)
except (KeyboardInterrupt, SystemExit):
stop_event.set()
pass
except:
stop_event.set()
raise
def trigger(self, question, forced=False):
self.triggered = True
self.question = question
self.forced_chatter = forced
log.info("Analyzing Message: {}".format(question))
