def __init__(self, channels_amount, channels, log_filename=None):
self.bots = []
self.channels_amount = channels_amount
self.log_filename = log_filename
self.channels = channels
self.db_logger = DatabaseLogger(settings.DATABASE['HOST'],
settings.DATABASE['NAME'],
settings.DATABASE['USER'],
settings.DATABASE['PASSWORD'])
def run(self):
db_logger = DatabaseLogger(DATABASE['HOST'], DATABASE['NAME'],
DATABASE['USER'], DATABASE['PASSWORD'])
data = []
print "I'll begin the scrapping."
while True:
twitch_api_url = 'https://api.twitch.tv/kraken/streams/?stream_type=live&limit=100&offset=' + str(
self.getOffset()) #&language=en
try:
data = (json.loads(
requests.get(twitch_api_url,
headers=self.getHeaders()).text)['streams'])
except (ValueError, ConnectionError, KeyError, SSLError):
time.sleep(5)
except KeyboardInterrupt:
print "Closing scrapper gracefully!"
db_logger.close()
for stream in data:
if not db_logger.track_exists(stream['channel']['name'],
self.batch):
db_logger.log_stream(self.batch, stream)
self.increaseTotalViews()
print "Collecting Data! Values: ", self.getTotalViews()
print "Current page Value: ", self.getOffset()
self.nextOffset()
if not data:
print "Finished Batch!"
self.resetOffset()
self.nextBatch()
def __init__(self, channels_amount, channels, log_filename=None):
self.bots = []
self.channels_amount = channels_amount
self.log_filename = log_filename
self.channels = channels
self.db_logger = DatabaseLogger(settings.DATABASE['HOST'],
settings.DATABASE['NAME'],
settings.DATABASE['USER'],
settings.DATABASE['PASSWORD'])
def _create_bot(self, name, channels):
conn = IRCConnection(settings.IRC['SERVER'], settings.IRC['PORT'],
settings.IRC['NICK'], settings.IRC['PASSWORD'],
self.log_filename)
if settings.USERS is not None:
bot_db_logger = DatabaseLogger(settings.DATABASE['HOST'],
settings.DATABASE['NAME'],
settings.DATABASE['USER'],
settings.DATABASE['PASSWORD'],
settings.USERS)
else:
bot_db_logger = DatabaseLogger(settings.DATABASE['HOST'],
settings.DATABASE['NAME'],
settings.DATABASE['USER'],
settings.DATABASE['PASSWORD'])
bot = TwitchBot(name, conn, bot_db_logger, Queue.Queue(),
self.log_filename)
bot.daemon = True
bot.connect_and_join_channels(channels)
bot.start()
return bot
class TwitchManager:
"""
Manages a series of irc chat bots that log the most popular channels on
Twitch.TV. The list of most popular channels is updated every 60 seconds.
"""
CHANNELS_PER_BOT = 25
SECONDS_BETWEEN_UPDATE_STREAMS = 120
SECONDS_BETWEEN_CREATE_BOTS = 30
def __init__(self, channels_amount, channels, log_filename=None):
self.bots = []
self.channels_amount = channels_amount
self.log_filename = log_filename
self.channels = channels
self.db_logger = DatabaseLogger(settings.DATABASE['HOST'],
settings.DATABASE['NAME'],
settings.DATABASE['USER'],
settings.DATABASE['PASSWORD'])
def _create_bot(self, name, channels):
conn = IRCConnection(settings.IRC['SERVER'], settings.IRC['PORT'],
settings.IRC['NICK'], settings.IRC['PASSWORD'],
self.log_filename)
bot_db_logger = DatabaseLogger(settings.DATABASE['HOST'],
settings.DATABASE['NAME'],
settings.DATABASE['USER'],
settings.DATABASE['PASSWORD'])
bot = TwitchBot(name, conn, bot_db_logger, Queue.Queue(),
self.log_filename)
bot.daemon = True
bot.connect_and_join_channels(channels)
bot.start()
return bot
def _create_bots(self, channels):
"""
Creates bots to log the given list of channels.
Twitch limits how many channels can be joined per connection, so we
create just enough bots to log all the desired streams. There's also
a timeout between opening connections otherwise Twitch disconnects
all bots.
"""
channels_joined = 0
while channels_joined < len(channels):
self.bots.append(
self._create_bot(
'Bot %i' % len(self.bots),
channels[channels_joined:channels_joined +
self.CHANNELS_PER_BOT]))
channels_joined += self.CHANNELS_PER_BOT
time.sleep(self.SECONDS_BETWEEN_CREATE_BOTS)
def _update_bot_channels(self, bot, new_channels):
channels_to_remove = list(set(bot.channels) - set(new_channels))
channels_to_add = list(set(new_channels) - set(bot.channels))
if channels_to_remove:
bot.command_queue.put(('part', channels_to_remove))
if channels_to_add:
bot.command_queue.put(('join', channels_to_add))
def _log_streams(self, streams):
for stream in streams:
self.db_logger.log_stream_stats(stream)
def _run_popular_streams_loop(self):
streams = get_top_streams(self.channels_amount)
channels = get_channel_names(streams)
self._create_bots(channels)
self._log_streams(streams)
while True:
time.sleep(self.SECONDS_BETWEEN_UPDATE_STREAMS)
streams = get_top_streams(self.channels_amount)
channels = get_channel_names(streams)
i, channels_joined = 0, 0
while channels_joined < self.channels_amount:
self._update_bot_channels(
self.bots[i], channels[channels_joined:channels_joined +
self.CHANNELS_PER_BOT])
i += 1
channels_joined += self.CHANNELS_PER_BOT
self._log_streams(streams)
def _run_static_streams_loop(self):
self._create_bots(self.channels)
while True:
time.sleep(self.SECONDS_BETWEEN_UPDATE_STREAMS)
#streams = get_top_streams(self.channels_amount)
#self._log_streams(streams)
def run_log_loop(self):
"""
Creates the logger bots and update which stream they log every 60
seconds.
"""
if self.channels:
self._run_static_streams_loop()
else:
self._run_popular_streams_loop()
def stop_bot(self):
for bot in self.bots:
bot.join()
def testDB():
db = DatabaseLogger('localhost', 'streams', 'streams', 'admin123')
print db.track_exists('boxbox', 1)
class TwitchManager:
"""
Manages a series of irc chat bots that log the most popular channels on
Twitch.TV. The list of most popular channels is updated every 60 seconds.
"""
CHANNELS_PER_BOT = 20
SECONDS_BETWEEN_UPDATE_STREAMS = 60
SECONDS_BETWEEN_CREATE_BOTS = 15
def __init__(self, channels_amount, channels, log_filename=None):
self.bots = []
self.channels_amount = channels_amount
self.log_filename = log_filename
self.channels = channels
self.db_logger = DatabaseLogger(settings.DATABASE['HOST'],
settings.DATABASE['NAME'],
settings.DATABASE['USER'],
settings.DATABASE['PASSWORD'])
def _create_bot(self, name, channels):
conn = IRCConnection(settings.IRC['SERVER'],
settings.IRC['PORT'],
settings.IRC['NICK'],
settings.IRC['PASSWORD'],
self.log_filename)
bot_db_logger = DatabaseLogger(settings.DATABASE['HOST'],
settings.DATABASE['NAME'],
settings.DATABASE['USER'],
settings.DATABASE['PASSWORD'])
bot = TwitchBot(name, conn, bot_db_logger, Queue.Queue(), self.log_filename)
bot.daemon = True
bot.connect_and_join_channels(channels)
bot.start()
return bot
def _create_bots(self, channels):
"""
Creates bots to log the given list of channels.
Twitch limits how many channels can be joined per connection, so we
create just enough bots to log all the desired streams. There's also
a timeout between opening connections otherwise Twitch disconnects
all bots.
"""
channels_joined = 0
while channels_joined < len(channels):
self.bots.append(
self._create_bot('Bot %i' % len(self.bots),
channels[channels_joined:channels_joined + self.CHANNELS_PER_BOT]))
channels_joined += self.CHANNELS_PER_BOT
time.sleep(self.SECONDS_BETWEEN_CREATE_BOTS)
def _update_bot_channels(self, bot, new_channels):
channels_to_remove = list(set(bot.channels) - set(new_channels))
channels_to_add = list(set(new_channels) - set(bot.channels))
if channels_to_remove:
bot.command_queue.put(('part', channels_to_remove))
if channels_to_add:
bot.command_queue.put(('join', channels_to_add))
def _log_streams(self, streams):
for stream in streams:
self.db_logger.log_stream_stats(stream)
def _run_popular_streams_loop(self):
streams = get_top_streams(self.channels_amount)
channels = get_channel_names(streams)
self._create_bots(channels)
self._log_streams(streams)
while True:
time.sleep(self.SECONDS_BETWEEN_UPDATE_STREAMS)
streams = get_top_streams(self.channels_amount)
channels = get_channel_names(streams)
i, channels_joined = 0, 0
while channels_joined < self.channels_amount:
self._update_bot_channels(self.bots[i],
channels[channels_joined:channels_joined + self.CHANNELS_PER_BOT])
i += 1
channels_joined += self.CHANNELS_PER_BOT
self._log_streams(streams)
def _run_static_streams_loop(self):
self._create_bots(self.channels)
streams = get_streams_from_channels(self.channels)
streams = [stream for stream in streams if stream]
self._log_streams(streams)
while True:
time.sleep(self.SECONDS_BETWEEN_UPDATE_STREAMS)
streams = get_streams_from_channels(self.channels)
streams = [stream for stream in streams if stream]
self._log_streams(streams)
def run_log_loop(self):
"""
Creates the logger bots and update which stream they log every 60
seconds.
"""
if self.channels:
self._run_static_streams_loop()
else:
self._run_popular_streams_loop()
def stop_bot(self):
for bot in self.bots:
bot.join()