def __init__(self):
super().__init__()
self._store = InMemoryStore()
# process and delete self.messages every minute
self.cron = Cron()
self.cron.add(call=self._archive)
self.cron.start()
# publish a summary of all cancer messages grouped by minute and channel
self.zmq_context = zmq.Context()
self.pubsub_socket = self.zmq_context.socket(zmq.PUB)
self.pubsub_socket.bind(Config.get('monitor.socket.cancer_summary'))
logger.info("bound publish socket to %s", Config.get('monitor.socket.cancer_summary'))
# respond to live cancer requests
self.cancer_socket = self.zmq_context.socket(zmq.REP)
self.cancer_socket.bind(Config.get('monitor.socket.cancer_request'))
logger.info("bound cancer socket to %s", Config.get('monitor.socket.cancer_request'))
# TODO: use asyncio
t = threading.Thread(target=self._handle_cancer_request)
t.daemon = True
t.start()
logger.info("started handle cancer request thread")
def test_defaults_defaults(self):
Config.defaults()
self.assertEqual('expose' in Config.config, True)
self.assertEqual('record' in Config.config, True)
self.assertEqual('monitor' in Config.config, True)
self.assertEqual('chat' in Config.config['monitor'], True)
self.assertEqual('unknown' not in Config.config, True)
def _connect(self):
self.socket = self.context.socket(zmq.REQ)
read_socket = Config.get('monitor.socket.read.cancer_request') or Config.get('monitor.socket.cancer_request')
self.socket.connect(read_socket)
self.poller.register(self.socket, zmq.POLLIN)
logger.info("connected cancer request socket to %s", read_socket)
def test_defaults_defaults(self):
Config.defaults()
self.assertEqual('expose' in Config.config, True)
self.assertEqual('record' in Config.config, True)
self.assertEqual('monitor' in Config.config, True)
self.assertEqual('chat' in Config.config['monitor'], True)
self.assertEqual('unknown' not in Config.config, True)
def __init__(self):
super().__init__()
self._store = InMemoryStore()
# process and delete self.messages every minute
self.cron = Cron()
self.cron.add(call=self._archive)
self.cron.start()
# publish a summary of all cancer messages grouped by minute and channel
self.zmq_context = zmq.Context()
self.pubsub_socket = self.zmq_context.socket(zmq.PUB)
summary_socket = Config.get('monitor.socket.cancer_summary')
self.pubsub_socket.bind(summary_socket)
logger.info("bound publish socket to %s", summary_socket)
# respond to live cancer requests
self.cancer_socket = self.zmq_context.socket(zmq.REP)
request_socket = Config.get('monitor.socket.cancer_request')
self.cancer_socket.bind(request_socket)
logger.info("bound cancer socket to %s", request_socket)
# TODO: use asyncio
t = threading.Thread(target=self._handle_cancer_request)
t.daemon = True
t.start()
logger.info("started handle cancer request thread")
def main():
# monitor chat channels
parser = argparse.ArgumentParser()
parser.add_argument('-l',
'--log',
dest='loglevel',
help="set the level of messages to display")
parser.add_argument('-c',
'--config',
dest='config',
help="load configuration from this file")
parser.add_argument(
'--viewers',
dest='viewers',
default=0,
type=int,
help="minimum viewer count to monitor channels (default: 0)")
args = parser.parse_args()
if args.config:
Config.load(args.config)
setup_logger(logger, args.loglevel, Config, "monitor.log")
# start monitoring forever
from twitchcancer.chat.chat import run
run(args)
def onOpen(self):
self.sendMessage('CAP REQ :twitch.tv/tags twitch.tv/commands twitch.tv/membership'.encode())
self.sendMessage('PASS {0}'.format(Config.get("monitor.chat.password").lower()).encode())
self.sendMessage('NICK {0}'.format(Config.get("monitor.chat.username").lower()).encode())
for channel in self.channels:
self.sendMessage('JOIN {0}'.format(channel).encode())
logger.info("auto-joining %s", channel)
def onOpen(self): # noqa
self.sendMessage('CAP REQ :twitch.tv/membership'.encode())
self.sendMessage('PASS {0}'.format(
Config.get("monitor.chat.password").lower()).encode())
self.sendMessage('NICK {0}'.format(
Config.get("monitor.chat.username").lower()).encode())
for channel in self.channels:
self.sendMessage('JOIN {0}'.format(channel).encode())
logger.info("joining %s", channel)
def __init__(self):
super().__init__()
self._store = PersistentStore()
# subscribe to summaries from the publisher socket
self.context = zmq.Context()
self.socket = self.context.socket(zmq.SUB)
self.socket.setsockopt(zmq.SUBSCRIBE, b"summary")
self.socket.connect(Config.get("monitor.socket.cancer_summary"))
logger.info("connected summary socket to %s", Config.get("monitor.socket.cancer_summary"))
def setUp(self):
try:
self.client = pymongo.MongoClient(
host=Config.get('record.mongodb.host'),
port=int(Config.get('record.mongodb.port')),
connectTimeoutMS=100,
serverSelectionTimeoutMS=100)
self.client.server_info()
self.db = self.client[self.database_name]
except pymongo.errors.ServerSelectionTimeoutError:
raise unittest.SkipTest(
"couldn't connect to a test database at mongodb://localhost:27017/"
)
def __init__(self):
super().__init__()
self._store = PersistentStore()
# subscribe to summaries from the publisher socket
self.context = zmq.Context()
self.socket = self.context.socket(zmq.SUB)
self.socket.setsockopt(zmq.SUBSCRIBE, b'summary')
read_socket = Config.get(
'monitor.socket.read.cancer_summary') or Config.get(
'monitor.socket.cancer_summary')
self.socket.connect(read_socket)
logger.info("connected summary socket to %s", read_socket)
def request(cls, url: str) -> dict:
""" Makes a call to Twitch's API
"""
headers = {
'Accept': 'application/vnd.twitchtv.v5+json',
'Client-ID': Config.get('monitor.chat.clientid'),
'User-agent': 'twitchcancer/python'
}
response = requests.get(url, headers=headers)
response.raise_for_status()
return response.json()
def main():
# persist data generated by monitor
parser = argparse.ArgumentParser()
parser.add_argument('-l',
'--log',
dest='loglevel',
help="set the level of messages to display")
parser.add_argument('-c',
'--config',
dest='config',
help="load configuration from this file")
args = parser.parse_args()
if args.config:
Config.load(args.config)
setup_logger(logger, args.loglevel, Config, "record.log")
# start recording forever
storage = Storage()
storage.record()
def __init__(self, host, port):
super().__init__()
self.host = host
self.port = int(port)
# queue used to hold messages coming from the producer (IRC client) and going into our consumer
self.messages = queue.Queue()
# thread started on connect only
self.client_thread = None
# create an IRCClient for this server
try:
self.client = IRCClient({
'username': Config.get("monitor.chat.username"),
'password': Config.get("monitor.chat.password"),
'server': self.host,
'port': self.port
})
except IRCConfigurationError:
raise ServerConfigurationError("Invalid configuration for {0}:{1}".format(self.host, self.port))
def __init__(self):
super().__init__()
client = pymongo.MongoClient(
host=Config.get('record.mongodb.host'),
port=int(Config.get('record.mongodb.port'))
)
self.db = client[Config.get('record.mongodb.database')]
# unique index on channel + date
#self.db.leaderboard.create_index()
#self.db.monthly_leaderboard.create_index()
#self.db.daily_leaderboard.create_index()
# 0.2.0 -> 0.3.0: channel name is in leaderboard.channel instead of leaderboard._id
for record in self.db.leaderboard.find({'channel': None}):
self.db.leaderboard.update_one(
{'_id': record['_id']},
{ '$set': {'channel': record['_id'] }}
)
# 0.2.0 -> 0.3.0: moved total.date to date
for record in self.db.leaderboard.find({'date': None}):
self.db.leaderboard.update_one(
{'_id': record['_id']},
{ '$set': {'date': record['total']['date'] }}
)
# ease access to parallel collections
self._collections = {
'all': self.db.leaderboard,
'monthly': self.db.monthly_leaderboard,
'daily': self.db.daily_leaderboard,
}
logger.info('using mongodb://%s:%s/%s', Config.get('record.mongodb.host'), Config.get('record.mongodb.port'), self.db.name)
def __init__(self):
super().__init__()
client = pymongo.MongoClient(
host=Config.get('record.mongodb.host'),
port=int(Config.get('record.mongodb.port'))
)
self.db = client[Config.get('record.mongodb.database')]
# unique index on channel + date
# self.db.leaderboard.create_index()
# self.db.monthly_leaderboard.create_index()
# self.db.daily_leaderboard.create_index()
# ease access to parallel collections
self._collections = {
'all': self.db.leaderboard,
'monthly': self.db.monthly_leaderboard,
'daily': self.db.daily_leaderboard,
}
logger.info('using mongodb://%s:%s/%s', Config.get('record.mongodb.host'), Config.get('record.mongodb.port'),
self.db.name)
def run(args):
use_ssl = Config.get('expose.websocket.pem') != ""
# use an ssl prefix if we have a pem file
if use_ssl:
prefix = "wss"
else:
prefix = "ws"
# build the full URL of the web socket end-point
url = "{0}://{1}:{2}".format(prefix, Config.get('expose.websocket.host'),
Config.get('expose.websocket.port'))
logger.info("starting web-socket server at %s", url)
factory = WebSocketServerFactory(url)
factory.protocol = PubSubProtocol
# setup the main event loop for network i/o
loop = asyncio.get_event_loop()
# create an ssl context if we need one
if use_ssl:
context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
context.load_cert_chain(Config.get('expose.websocket.pem'))
logger.debug("using ssl")
else:
context = None
logger.debug("not using ssl")
coro = loop.create_server(factory,
host=Config.get('expose.websocket.host'),
port=Config.get('expose.websocket.port'),
ssl=context)
server = loop.run_until_complete(coro)
# setup publishers coroutines
publishers = create_publishers()
loop.run_until_complete(publishers)
try:
loop.run_forever()
except KeyboardInterrupt:
pass
finally:
publishers.close()
server.close()
loop.close()
def _load_json(cls, url):
try:
request = urllib.request.Request(url)
request.add_header('Client-ID', Config.get('monitor.chat.clientid'))
with urllib.request.urlopen(request) as response:
binary = response.read()
string = binary.decode()
data = json.loads(string)
return data
except urllib.error.URLError as e:
logger.warning("Twitch API request failed with URLError %s", e)
return None
except urllib.error.HTTPError as e:
logger.warning("Twitch API request failed with HTTPError %s", e)
return None
except HTTPException as e:
logger.warning("Twitch API request failed with HTTPException %s", e)
return None
except ValueError as e:
logger.warning("Twitch API response was not json %s", e)
return None
def run(args):
use_ssl = Config.get('expose.websocket.pem') != ""
# use an ssl prefix if we have a pem file
if use_ssl:
prefix = "wss"
else:
prefix = "ws"
# build the full URL of the web socket end-point
url = "{0}://{1}:{2}".format(prefix, Config.get('expose.websocket.host'), Config.get('expose.websocket.port'))
logger.info("starting web-socket server at %s", url)
factory = WebSocketServerFactory(url)
factory.protocol = PubSubProtocol
# setup the main event loop for network i/o
loop = asyncio.get_event_loop()
# create an ssl context if we need one
if use_ssl:
context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
context.load_cert_chain(Config.get('expose.websocket.pem'))
logger.debug("using ssl")
else:
context = None
logger.debug("not using ssl")
coro = loop.create_server(factory, host=Config.get('expose.websocket.host'), port=Config.get('expose.websocket.port'), ssl=context)
server = loop.run_until_complete(coro)
# setup publishers coroutines
publishers = create_publishers()
loop.run_until_complete(publishers)
try:
loop.run_forever()
except KeyboardInterrupt:
pass
finally:
publishers.close()
server.close()
loop.close()
def test_get_single_value(self):
self.assertEqual(Config.get("level1.level2.level3"), "value")
def test_update_ignore_unknown_key(self):
Config.update({"level1": {"level2": {"unknown": "merged"}}})
self.assertEqual('unknown' not in Config.config['level1']['level2'], True)
def test_update_existing_untouched(self):
Config.update({})
self.assertEqual(Config.config['level1']['level2']['level3'], "value")
def test_update_single_value(self):
Config.update({"level1": {"level2": {"level3": "merged"}}})
self.assertEqual(Config.config['level1']['level2']['level3'], "merged")
def test_get_compound_value(self):
self.assertEqual(Config.get("level1.level2"), {"level3": "value"})
def test_get_single_value(self):
self.assertEqual(Config.get("level1.level2.level3"), "value")
def test_get_missing_key(self):
self.assertEqual(Config.get("missing.key.is.missing"), {})
def test_get_missing_key(self):
self.assertEqual(Config.get("missing.key.is.missing"), {})
def test_get_compound_value(self):
self.assertEqual(Config.get("level1.level2"), {"level3": "value"})
def test_update_single_value(self):
Config.update({"level1": {"level2": {"level3": "merged"}}})
self.assertEqual(Config.config['level1']['level2']['level3'], "merged")
def test_update_existing_untouched(self):
Config.update({})
self.assertEqual(Config.config['level1']['level2']['level3'], "value")
def test_update_ignore_unknown_key(self):
Config.update({"level1": {"level2": {"unknown": "merged"}}})
self.assertEqual('unknown' not in Config.config['level1']['level2'],
True)
