def run_for_subreddits():
api_subreddits = get_subreddits_all()
for api_subreddit in api_subreddits:
subreddit_name = api_subreddit['name']
praw_subreddit = reddit.subreddit(subreddit_name)
start_date = '2018-01-01'
run(subreddit_name, praw_subreddit, start_date)
def run_for_subreddit(subreddit_name):
api_subreddit = get_subreddit_by_name(subreddit_name)
if not api_subreddit:
print('Subreddit with name %s does not exist on server' %
subreddit_name)
return
praw_subreddit = reddit.subreddit(subreddit_name)
start_date = api_subreddit['startDate']
run(subreddit_name, praw_subreddit, start_date)
def check_for_flairs(args):
sub = str(args.sub[0]).strip('[]\' ')
flair_count = 0
if (args.flair):
# Characters must be stripped due to the way that ArgParse handles user inputs.
flair_entered = str(args.flair[0]).strip('[]\' ')
sub = str(args.sub[0])
print(type(sub))
this_sub = reddit.subreddit(sub)
for flair in this_sub.flair(limit=None):
if flair['flair_text'] == flair_entered:
print('Match')
flair_count += 1
if (args.css):
cs = str(args.css[0]).strip('[]\' ')
# Characters must be stripped due to the way that ArgParse handles user inputs.
this_sub = reddit.subreddit(sub)
for flair in this_sub.flair(limit=None):
if flair['flair_css_class'] == cs:
print('Match')
flair_count += 1
return print('Flair Count: ', flair_count)
def analyze_subreddit(subreddit: str) -> dict:
logger.info("Looking for posts in /r/%s", subreddit)
result = {
"analyzed": [],
"is_old": [],
"is_self": [],
"is_unsupported": [],
"was_analyzed": [],
"error": [],
}
for submission in list(reddit.subreddit(subreddit).new(limit=25)):
data = {"id": submission.id, "title": submission.title}
if filters.is_self(submission):
result["is_self"].append(data)
continue
if filters.is_old(submission):
result["is_old"].append(data)
continue
if filters.is_unsupported(submission):
result["is_unsupported"].append(data)
continue
if filters.was_analyzed(submission):
result["was_analyzed"].append(data)
continue
try:
analyze_submission(submission)
sleep(1)
result["analyzed"].append(data)
except Exception as exc:
data["error"] = str(exc)
logging.error(exc, exc_info=True)
result["errors"].append(data)
return result
from api import Api
from database import SQLite3Database
from logger import logger
from reddit import reddit
from subreddits import SUBREDDITS
from utils import unix_timestamp_now, datetime_string_now
server = Api()
logger.info('Starting active user and subscriber counts script...')
for subreddit_name in SUBREDDITS:
praw_subreddit = reddit.subreddit(subreddit_name)
active_user_count = praw_subreddit.active_user_count
subscriber_count = praw_subreddit.subscribers
unix_timestamp = unix_timestamp_now()
datetime_string = datetime_string_now()
active_user_counts_db = SQLite3Database('active_user_counts.db')
result = active_user_counts_db.cursor.execute(
'''
INSERT INTO active_user_counts (subreddit_name, count, timestamp)
VALUES (?, ?, ?)
''', (subreddit_name, active_user_count, unix_timestamp))
active_user_counts_db.conn.commit()
active_user_counts_db.close()
response = server.create_active_user_count(subreddit_name,
active_user_count,
datetime_string)
def stream_submissions(sub_name, handler):
for submission in reddit.subreddit(subreddit_name).stream.submissions():
handler(submission)
def get_by_time_filter(sub_name, time_filter, limit, handler):
for submission in reddit.subreddit(subreddit_name).top(
time_filter=time_filter, limit=limit):
handler(submission)
from reddit import reddit
import os
import re
from haiku import haiku
if not os.path.isfile("posts_replied_to.txt"):
posts_replied_to = []
else:
with open("posts_replied_to.txt", "r") as f:
posts_replied_to = f.read()
posts_replied_to = posts_replied_to.split("\n")
posts_replied_to = filter(None, posts_replied_to)
subreddit = reddit.subreddit('test')
for submission in subreddit.hot(limit=5):
if submission.id not in posts_replied_to:
if re.search("give me a shitty haiku", submission.title,
re.IGNORECASE):
haiku.run()
submission.reply(haiku.body)
print "Bot replying to : ", submission.title
posts_replied_to.append(submission.id)
with open("posts_replied_to.txt", "w") as f:
for post_id in posts_replied_to:
f.write(post_id + "\n")
def main():
import os
import time
from collections import deque
import logging, logging.handlers
from pathlib import Path
import praw, prawcore
from reddit import reddit
from strategy import process_subsmission, process_inbox_item
from config import target_subreddits
script_path = Path(__file__).resolve()
os.chdir(script_path.parent)
prawcore_logger = logging.getLogger('prawcore')
prawcore_logger.setLevel(logging.DEBUG)
prawcore_logger.disabled = True
log_file = script_path.parent / 'log' / 'prawcore.log'
if log_file.parent.is_dir():
log_format = '%(asctime)s %(levelname)s %(funcName)s:%(lineno)d | %(message)s'
rfh_config = {
'filename': log_file,
'encoding': 'utf-8',
'maxBytes': 5 * 1024 * 1024, # 5 megabytes
'backupCount': 8
}
rfh = logging.handlers.RotatingFileHandler(**rfh_config)
rfh.setFormatter(logging.Formatter(log_format))
prawcore_logger.addHandler(rfh)
prawcore_logger.info('Log ({}): {}'.format(prawcore_logger.name,
log_file.absolute()))
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
#logger.addHandler(logging.StreamHandler())
#logger.disabled = True
log_file = script_path.parent / 'log' / script_path.with_suffix(
'.log').name
if log_file.parent.is_dir():
log_format = '%(asctime)s %(levelname)s %(funcName)s:%(lineno)d | %(message)s'
rfh_config = {
'filename': log_file,
'encoding': 'utf-8',
'maxBytes': 5 * 1024 * 1024, # 5 megabytes
'backupCount': 8
}
rfh = logging.handlers.RotatingFileHandler(**rfh_config)
rfh.setFormatter(logging.Formatter(log_format))
logger.addHandler(rfh)
logger.info('Log ({}): {}'.format(logger.name, log_file.absolute()))
if reddit.read_only:
raise RuntimeError('a read-write reddit instance is required')
subreddit = reddit.subreddit('+'.join(target_subreddits))
start_time = time.time()
__ = ['submission', 'inbox']
check_time = dict.fromkeys(__, start_time)
seen_deque = dict.fromkeys(__, deque(maxlen=100))
control_checkpoint_progression = lambda d: max(0, .5 * (d - 10))
while True:
try:
for submission in subreddit.stream.submissions(pause_after=-1):
if submission is None:
break
if submission.id in seen_deque['submission']:
logger.debug('Skip: seen item: t3_{}'.format(
submission.id))
continue
if submission.created_utc < check_time['submission']:
if submission.created_utc < start_time:
logger.debug(
'Skip: item was submitted before bot started: t3_{}'
.format(submission.id))
else:
logger.debug(
'Skip: timestamp was supplanted: t3_{}'.format(
submission.id))
continue
check_time['submission'] += control_checkpoint_progression(
submission.created_utc - check_time['submission'])
seen_deque['submission'].append(submission.id)
process_subsmission(submission)
for item in reddit.inbox.stream(pause_after=-1):
if item is None:
break
if item.id in seen_deque['inbox']:
logger.debug('[Inbox] Skip: seen item: t4_{}'.format(
item.id))
continue
if item.created_utc < check_time['inbox']:
if item.created_utc < start_time:
logger.debug(
'[Inbox] Skip: item was submitted before bot started: t4_{}'
.format(item.id))
else:
logger.debug(
'[Inbox] Skip: timestamp was supplanted: t4_{}'.
format(item.id))
continue
check_time['inbox'] += control_checkpoint_progression(
item.created_utc - check_time['inbox'])
seen_deque['inbox'].append(item.id)
process_inbox_item(item)
except (praw.exceptions.PRAWException,
prawcore.exceptions.PrawcoreException) as e:
if isinstance(e, praw.exceptions.APIException):
if e.error_type == 'RATELIMIT':
logger.info('Exception: ratelimit exceeded: {}'.format(
e.message))
time.sleep(11 * 60)
else:
logger.warning(
'Exception: unhandled PRAW APIException exception:',
exc_info=True)
elif isinstance(e, prawcore.exceptions.ResponseException):
logger.info('Exception: ResponseException: {}'.format(
e.response))
time.sleep(5)
elif isinstance(e, prawcore.exceptions.RequestException):
logger.info('Exception: RequestException: {}'.format(
e.original_exception))
time.sleep(5)
else:
logger.warning('Exception: unhandled PRAW exception:',
exc_info=True)
except Exception:
logger.error('Exception: unhandled exception:', exc_info=True)