def run(self):
db_logger = DatabaseLogger(DATABASE['HOST'], DATABASE['NAME'],
DATABASE['USER'], DATABASE['PASSWORD'])
data = []
print "I'll begin the scrapping."
while True:
twitch_api_url = 'https://api.twitch.tv/kraken/streams/?stream_type=live&limit=100&offset=' + str(
self.getOffset()) #&language=en
try:
data = (json.loads(
requests.get(twitch_api_url,
headers=self.getHeaders()).text)['streams'])
except (ValueError, ConnectionError, KeyError, SSLError):
time.sleep(5)
except KeyboardInterrupt:
print "Closing scrapper gracefully!"
db_logger.close()
for stream in data:
if not db_logger.track_exists(stream['channel']['name'],
self.batch):
db_logger.log_stream(self.batch, stream)
self.increaseTotalViews()
print "Collecting Data! Values: ", self.getTotalViews()
print "Current page Value: ", self.getOffset()
self.nextOffset()
if not data:
print "Finished Batch!"
self.resetOffset()
self.nextBatch()
def testDB():
db = DatabaseLogger('localhost', 'streams', 'streams', 'admin123')
print db.track_exists('boxbox', 1)