# COPYING for more details.
#
# --------------------------------------------------------------------
import logging
logger = logging.basicConfig(level=logging.WARN)
from bmscraper import BMScraper, AndroidEmotesProcessorFactory
from bmscraper.ratelimiter import TokenBucket
from data import *
from json import dumps
import os
import gzip
factory = AndroidEmotesProcessorFactory(single_emotes_filename=os.path.join( '..', 'single_emotes', '{}', '{}.png'))
scraper = BMScraper(factory)
scraper.user = '******'
scraper.password = '******'
scraper.subreddits = subreddits
scraper.image_blacklist = image_blacklist_android
scraper.nsfw_subreddits = nsfw_subreddits
scraper.emote_info = emote_info
scraper.rate_limit_lock = TokenBucket(15, 30)
scraper.scrape()
f = gzip.open(os.path.join('..', 'single_emotes', 'emotes.json.gz'), 'wb')
f.write(dumps(factory.emotes, separators=(',', ': ')))
f.close()
for subreddit in subreddits:
import brotli
except ImportError:
logger.warn('Brotli is not available')
brotli = None
from data import *
from json import dumps
import os
CDN_ORIGIN = 'https://berrytube.tv/berrymotes'
factory = UserscriptEmotesProcessorFactory(single_emotes_filename=os.path.join('..', 'single_emotes', '{}', '{}.png'),
apng_dir=os.path.join('..', 'images'),
apng_url=CDN_ORIGIN + '/images/{}/{}')
scraper = BMScraper(factory)
scraper.user = os.environ['REDDIT_USERNAME']
scraper.password = os.environ['REDDIT_PASSWORD']
scraper.subreddits = subreddits
scraper.legacy_subreddits = legacy_subreddits
scraper.image_blacklist = image_blacklist
scraper.nsfw_subreddits = nsfw_subreddits
scraper.emote_info = emote_info
scraper.rate_limit_lock = TokenBucket(15, 30)
scraper.tags_data = requests.get(CDN_ORIGIN + "/data/tags.js").json()
start = time.time()
scraper.scrape()
logger.info("Finished scrape in {}.".format(time.time() - start))
def output(basename, data):
#
# --------------------------------------------------------------------
import logging
logger = logging.basicConfig(level=logging.WARN)
from bmscraper import BMScraper, AndroidEmotesProcessorFactory
from bmscraper.ratelimiter import TokenBucket
from data import *
from json import dumps
import os
import gzip
factory = AndroidEmotesProcessorFactory(
single_emotes_filename=os.path.join('..', 'single_emotes', '{}', '{}.png'))
scraper = BMScraper(factory)
scraper.user = os.environ['REDDIT_USERNAME']
scraper.password = os.environ['REDDIT_PASSWORD']
scraper.subreddits = subreddits
scraper.legacy_subreddits = legacy_subreddits
scraper.image_blacklist = image_blacklist_android
scraper.nsfw_subreddits = nsfw_subreddits
scraper.emote_info = emote_info
scraper.rate_limit_lock = TokenBucket(15, 30)
scraper.scrape()
f = gzip.open(os.path.join('..', 'single_emotes', 'emotes.json.gz'), 'wb')
f.write(dumps(factory.emotes, separators=(',', ': ')))
f.close()
# COPYING for more details.
#
# --------------------------------------------------------------------
import logging
logger = logging.basicConfig(level=logging.WARN)
from bmscraper import BMScraper, AndroidEmotesProcessorFactory
from bmscraper.ratelimiter import TokenBucket
from data import *
from json import dumps
import os
import gzip
factory = AndroidEmotesProcessorFactory(single_emotes_filename=os.path.join( '..', 'single_emotes', '{}', '{}.png'))
scraper = BMScraper(factory)
scraper.user = '******'
scraper.password = '******'
scraper.subreddits = subreddits
scraper.user_agent = 'linux:Ponymote harvester:v2.1 (by /u/Trellmor)'
scraper.image_blacklist = image_blacklist_android
scraper.emote_info = emote_info
scraper.rate_limit_lock = TokenBucket(15, 30)
scraper.scrape()
def save_if_new(filename, data):
data_old = ''
if (os.path.exists(filename)):
f = gzip.open(filename, 'r')
data_old = f.read()
from bmscraper.ratelimiter import TokenBucket
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
from bmscraper import BMScraper, UserscriptEmotesProcessorFactory
from data import *
from json import dumps, loads
import os
factory = UserscriptEmotesProcessorFactory(single_emotes_filename=os.path.join('..', 'single_emotes', '{}', '{}.png'),
apng_dir=os.path.join('..', 'images'),
apng_url='http://berrymotes.com/images/{}/{}')
scraper = BMScraper(factory)
scraper.user = '******'
scraper.password = '******'
scraper.subreddits = subreddits
scraper.image_blacklist = image_blacklist
scraper.nsfw_subreddits = nsfw_subreddits
scraper.emote_info = emote_info
scraper.rate_limit_lock = TokenBucket(15, 30)
scraper.tags_data = requests.get("http://btc.berrytube.tv/berrymotes/data.js").json()
start = time.time()
scraper.scrape()
logger.info("Finished scrape in {}.".format(time.time() - start))
f = open(os.path.join('..', 'js', 'berrymotes_data.js'), 'wb')
json = dumps(scraper.emotes, separators=(',', ':'))
