CDN_ORIGIN = 'https://berrytube.tv/berrymotes'
factory = UserscriptEmotesProcessorFactory(single_emotes_filename=os.path.join('..', 'single_emotes', '{}', '{}.png'),
apng_dir=os.path.join('..', 'images'),
apng_url=CDN_ORIGIN + '/images/{}/{}')
scraper = BMScraper(factory)
scraper.user = os.environ['REDDIT_USERNAME']
scraper.password = os.environ['REDDIT_PASSWORD']
scraper.subreddits = subreddits
scraper.legacy_subreddits = legacy_subreddits
scraper.image_blacklist = image_blacklist
scraper.nsfw_subreddits = nsfw_subreddits
scraper.emote_info = emote_info
scraper.rate_limit_lock = TokenBucket(15, 30)
scraper.tags_data = requests.get(CDN_ORIGIN + "/data/tags.js").json()
start = time.time()
scraper.scrape()
logger.info("Finished scrape in {}.".format(time.time() - start))
def output(basename, data):
with open(basename, 'wb') as fh:
fh.write(data)
fname = basename + '.gz'
try:
with gzip.open(fname, 'wb') as fh:
fh.write(data)
except:
logger.exception('Unable to gzip emote data')
import os
factory = UserscriptEmotesProcessorFactory(single_emotes_filename=os.path.join('..', 'single_emotes', '{}', '{}.png'),
apng_dir=os.path.join('..', 'images'),
apng_url='http://berrymotes.com/images/{}/{}')
scraper = BMScraper(factory)
scraper.user = '******'
scraper.password = '******'
scraper.subreddits = subreddits
scraper.image_blacklist = image_blacklist
scraper.nsfw_subreddits = nsfw_subreddits
scraper.emote_info = emote_info
scraper.rate_limit_lock = TokenBucket(15, 30)
scraper.tags_data = requests.get("http://btc.berrytube.tv/berrymotes/data.js").json()
start = time.time()
scraper.scrape()
logger.info("Finished scrape in {}.".format(time.time() - start))
f = open(os.path.join('..', 'js', 'berrymotes_data.js'), 'wb')
json = dumps(scraper.emotes, separators=(',', ':'))
f.write(''.join(["var berryEmotes=", json, ";"]))
f.close()
f = open(os.path.join('..', 'js', 'berrymotes_json_data.json'), 'wb')
f.write(dumps(scraper.emotes, separators=(',', ':')))
f.close()
