logger.warn('Brotli is not available')
brotli = None
from data import *
from json import dumps
import os
CDN_ORIGIN = 'https://berrytube.tv/berrymotes'
factory = UserscriptEmotesProcessorFactory(single_emotes_filename=os.path.join('..', 'single_emotes', '{}', '{}.png'),
apng_dir=os.path.join('..', 'images'),
apng_url=CDN_ORIGIN + '/images/{}/{}')
scraper = BMScraper(factory)
scraper.user = os.environ['REDDIT_USERNAME']
scraper.password = os.environ['REDDIT_PASSWORD']
scraper.subreddits = subreddits
scraper.legacy_subreddits = legacy_subreddits
scraper.image_blacklist = image_blacklist
scraper.nsfw_subreddits = nsfw_subreddits
scraper.emote_info = emote_info
scraper.rate_limit_lock = TokenBucket(15, 30)
scraper.tags_data = requests.get(CDN_ORIGIN + "/data/tags.js").json()
start = time.time()
scraper.scrape()
logger.info("Finished scrape in {}.".format(time.time() - start))
def output(basename, data):
with open(basename, 'wb') as fh:
fh.write(data)
# --------------------------------------------------------------------
import logging
logger = logging.basicConfig(level=logging.WARN)
from bmscraper import BMScraper, AndroidEmotesProcessorFactory
from bmscraper.ratelimiter import TokenBucket
from data import *
from json import dumps
import os
import gzip
factory = AndroidEmotesProcessorFactory(single_emotes_filename=os.path.join( '..', 'single_emotes', '{}', '{}.png'))
scraper = BMScraper(factory)
scraper.user = '******'
scraper.password = '******'
scraper.subreddits = subreddits
scraper.image_blacklist = image_blacklist_android
scraper.nsfw_subreddits = nsfw_subreddits
scraper.emote_info = emote_info
scraper.rate_limit_lock = TokenBucket(15, 30)
scraper.scrape()
f = gzip.open(os.path.join('..', 'single_emotes', 'emotes.json.gz'), 'wb')
f.write(dumps(factory.emotes, separators=(',', ': ')))
f.close()
for subreddit in subreddits:
subreddit_emotes = [x for x in factory.emotes if x['sr'] == subreddit]
subreddit_emotes = sorted(subreddit_emotes, key = lambda x: x['image'])
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
from bmscraper import BMScraper, UserscriptEmotesProcessorFactory
from data import *
from json import dumps, loads
import os
factory = UserscriptEmotesProcessorFactory(single_emotes_filename=os.path.join('..', 'single_emotes', '{}', '{}.png'),
apng_dir=os.path.join('..', 'images'),
apng_url='http://berrymotes.com/images/{}/{}')
scraper = BMScraper(factory)
scraper.user = '******'
scraper.password = '******'
scraper.subreddits = subreddits
scraper.image_blacklist = image_blacklist
scraper.nsfw_subreddits = nsfw_subreddits
scraper.emote_info = emote_info
scraper.rate_limit_lock = TokenBucket(15, 30)
scraper.tags_data = requests.get("http://btc.berrytube.tv/berrymotes/data.js").json()
start = time.time()
scraper.scrape()
logger.info("Finished scrape in {}.".format(time.time() - start))
f = open(os.path.join('..', 'js', 'berrymotes_data.js'), 'wb')
json = dumps(scraper.emotes, separators=(',', ':'))
f.write(''.join(["var berryEmotes=", json, ";"]))
f.close()