def get_blacklisted_users():
'''
Returns:
List of blacklisted users from wiki
Raises:
Exception if anything goes wrong, page times out, etc
'''
Reddit.wait()
r = Reddit.httpy.get('http://www.reddit.com/r/AmateurArchives/wiki/banned.json')
json = loads(r)
wiki = json['data']['content_md']
lines = wiki.split('\r\n')
blacklisted = []
for line in lines:
if not '|' in line:
continue
fields = line.split('|')
if len(fields) != 5:
continue
if fields[1] in ['username', ':--']:
continue
for user in fields[1].replace('/u/', '').split('/'):
user = user.strip()
if user == '': continue
blacklisted.append(user)
return blacklisted
def get_top_submissions():
titles = "<ul>"
reddit = Reddit()
reddit.open_reddit_read()
submissions = reddit.top_submissions('u_hansknecht', 5)
for entry in submissions:
titles += "<li>{0}</li>".format(entry.title)
titles += "</ul>"
return titles
def is_valid_request(child, db, log):
'''
Ensures request is from an account older than MINIMUM_REQUESTER_AGE days,
and the accounts last request was over MINIMUM_REQUEST_DAYS days ago.
If not, removes the request and comments with the reason for removal
Returns:
True if post is valid request,
False if request is not valid and was removed.
'''
if type(child) != Post: return True
request_is_valid = False
# Check if last request was < MINIMUM_REQUEST_DAYS days ago
now = timegm(gmtime())
for (date, permalink) in db.select('date, permalink', 'amarch_requests', 'username = ?', [child.author]):
if date + (3600 * 24 * AmArch.MINIMUM_REQUEST_DAYS) > now:
# Last request was < MINIMUM_REQUEST_DAYS days ago, check if the request was 'removed'
post = Reddit.get(permalink)
if post.banned_by == None:
# Last request was < MINIMUM_REQUEST_DAYS days ago, wasn't removed
child.remove(mark_as_spam=False)
log('AmArch.is_valid_request: Request < %d days old: %s' % (AmArch.MINIMUM_REQUEST_DAYS, child.permalink()))
body = '## Rule: [Requests must be at least %d days apart](/r/AmateurArchives/about/sidebar)\n\n' % AmArch.MINIMUM_REQUEST_DAYS
body += 'The [**last request**](%s) from your account was submitted %s' % (permalink, Reddit.utc_timestamp_to_hr(post.created))
response = child.reply(body)
response.distinguish()
child.flair('last req < %dd' % AmArch.MINIMUM_REQUEST_DAYS)
return False
else:
# XXX OPTIMIZATION
# Last request was > MINIMUM_REQUEST_DAYS days ago but was removed
# Therefore: User account must be > MINIMUM_REQUESTER_AGE days old
request_is_valid = True
if not request_is_valid:
# Check if user is < MINIMUM_REQUESTER_AGE days old
user = Reddit.get_user_info(child.author)
if user.created > now - (3600 * 24 * AmArch.MINIMUM_REQUESTER_AGE):
child.remove(mark_as_spam=False)
log('AmArch.is_valid_request: Requester /u/%s < %d days old: %s' % (child.author, AmArch.MINIMUM_REQUESTER_AGE, child.permalink()))
body = '## Rule: [Requests must be from accounts more than %d days old](/r/AmateurArchives/about/sidebar)\n\n' % AmArch.MINIMUM_REQUESTER_AGE
body += 'The account (/u/%s) was created %s.' % (child.author, Reddit.utc_timestamp_to_hr(user.created))
response = child.reply(body)
response.distinguish()
child.flair('user < %dd' % AmArch.MINIMUM_REQUESTER_AGE)
return False
# Request is valid. Add it to the database for checking in the future
log('AmArch.is_valid_request: Allowing request from /u/%s' % child.author)
if db.count('amarch_requests', 'username = ?', [child.author]) == 0:
db.insert('amarch_requests', (child.author, child.created, child.permalink()))
else:
db.update('amarch_requests', 'date = ?, permalink = ?', 'username = ?', [child.created, child.permalink(), child.author])
return True
def backfill_posts(legacy=True):
(username, password) = db.get_credentials('reddit')
reddit.login(username, password)
where = ''
if legacy:
where = 'where legacy = 1'
cur = db.conn.cursor()
query = '''
select id, userid, title, selftext, url, subreddit, over_18, created, legacy, permalink, ups, downs
from posts
%s
order by id
''' % where
total = 0
ids_to_fetch = []
# Store existing values in dict
for (postid, userid, title, selftext, url, subreddit, over_18, created,
legacy, permalink, ups, downs) in cur.execute(query):
ids_to_fetch.append(str(postid))
if len(ids_to_fetch) >= 99:
total += len(ids_to_fetch)
ids_to_fetch.append('1234')
url = 'http://www.reddit.com/by_id/t3_%s.json' % ',t3_'.join(
ids_to_fetch)
try:
posts = reddit.get(url)
except HTTPError, e:
print 'HTTPError: %s' % str(e)
posts = []
for post in posts:
oldpost = {}
oldpost['title'] = post.title
oldpost['url'] = post.url
oldpost['selftext'] = post.selftext
oldpost['subreddit'] = post.subreddit
oldpost['created'] = int(post.created)
oldpost['permalink'] = post.permalink()
oldpost['over_18'] = int(post.over_18)
oldpost['legacy'] = 0
oldpost['id'] = post.id.rjust(6, '0')
oldpost['ups'] = post.ups
oldpost['downs'] = post.downs
Reddit.debug('updating post %s by %s' % (post.id, post.author))
update_post(oldpost)
db.conn.commit()
ids_to_fetch = list()
print 'running total: %d' % total
def __init__(self):
# Single file that all output is written to, to track usage
self.exit_if_already_started()
self.db = DB() # Database instance
log_level = self.db.get_config('log_level', default='user')
if log_level == 'none':
self.root_log = open(devnull, 'w')
else:
self.root_log = open(
path.join(ImageUtils.get_root(), 'history.log'), 'a')
self.logger = self.root_log # Logger used by helper classes
self.reddit = Reddit()
self.excluded_subs = self.db.get_excluded_subreddits()
def backfill_users():
q_users = '''
select id,username
from users
where deleted = 0
'''
cur = db.conn.cursor()
execur = cur.execute(q_users)
ids_and_users = execur.fetchall() # Get list of users + ids
index = 0
for (userid, username) in ids_and_users:
index += 1
print('(%d/%d) updating %s...' % (index, len(ids_and_users), username)),
stdout.flush()
try:
ui = Reddit.get_user_info(username)
except Exception as e:
print(str(e))
continue
q_user = '''
update users
set
created = %d,
username = "******"
where id = %d
''' % (ui.created, ui.name, userid)
cur.execute(q_user)
print('done')
cur.close()
def backfill_users():
q_users = '''
select id,username
from users
where deleted = 0
'''
cur = db.conn.cursor()
execur = cur.execute(q_users)
ids_and_users = execur.fetchall() # Get list of users + ids
index = 0
for (userid, username) in ids_and_users:
index += 1
print '(%d/%d) updating %s...' % (index, len(ids_and_users), username),
stdout.flush()
try:
ui = Reddit.get_user_info(username)
except Exception, e:
print str(e)
continue
q_user = '''
update users
set
created = %d,
username = "******"
where id = %d
''' % (ui.created, ui.name, userid)
cur.execute(q_user)
print 'done'
def backfill_posts(legacy=True):
(username, password) = db.get_credentials('reddit')
reddit.login(username, password)
where = ''
if legacy:
where = 'where legacy = 1'
cur = db.conn.cursor()
query = '''
select id, userid, title, selftext, url, subreddit, over_18, created, legacy, permalink, ups, downs
from posts
%s
order by id
''' % where
total = 0
ids_to_fetch = []
# Store existing values in dict
for (postid, userid, title, selftext, url, subreddit, over_18, created, legacy, permalink, ups, downs) in cur.execute(query):
ids_to_fetch.append(str(postid))
if len(ids_to_fetch) >= 99:
total += len(ids_to_fetch)
ids_to_fetch.append('1234')
url = 'http://www.reddit.com/by_id/t3_%s.json' % ',t3_'.join(ids_to_fetch)
try:
posts = reddit.get(url)
except HTTPError, e:
print 'HTTPError: %s' % str(e)
posts = []
for post in posts:
oldpost = {}
oldpost['title'] = post.title
oldpost['url'] = post.url
oldpost['selftext'] = post.selftext
oldpost['subreddit'] = post.subreddit
oldpost['created'] = int(post.created)
oldpost['permalink'] = post.permalink()
oldpost['over_18'] = int(post.over_18)
oldpost['legacy'] = 0
oldpost['id'] = post.id.rjust(6, '0')
oldpost['ups'] = post.ups
oldpost['downs'] = post.downs
Reddit.debug('updating post %s by %s' % (post.id, post.author))
update_post(oldpost)
db.conn.commit()
ids_to_fetch = list()
print 'running total: %d' % total
def __init__(self): # Single file that all output is written to, to track usage self.exit_if_already_started() self.root_log = open(path.join(ImageUtils.get_root(), 'history.log'), 'a') self.logger = self.root_log # Logger used by helper classes self.db = DB() # Database instance self.reddit = Reddit() self.excluded_subs = self.db.get_excluded_subreddits()
def __init__(self):
# Single file that all output is written to, to track usage
self.exit_if_already_started()
self.root_log = open(path.join(ImageUtils.get_root(), 'history.log'),
'a')
self.logger = self.root_log # Logger used by helper classes
self.db = DB() # Database instance
self.reddit = Reddit()
try:
(username, password) = self.db.get_credentials('reddit')
try:
self.reddit.login(username, password)
except Exception, e:
self.debug('__init__: failed to login to reddit: %s' % str(e))
except Exception, e:
self.debug('__init__: failed to get reddit credentials: %s' %
str(e))
class Gonewild(object):
def __init__(self):
# Single file that all output is written to, to track usage
self.exit_if_already_started()
self.root_log = open(path.join(ImageUtils.get_root(), 'history.log'), 'a')
self.logger = self.root_log # Logger used by helper classes
self.db = DB() # Database instance
self.reddit = Reddit()
try:
(username, password) = self.db.get_credentials('reddit')
try:
self.reddit.login(username, password)
except Exception, e:
self.debug('__init__: failed to login to reddit: %s' % str(e))
except Exception, e:
self.debug('__init__: failed to get reddit credentials: %s' % str(e))
def get_blacklisted_urls():
'''
Returns:
List of blacklisted urls from wiki
Raises:
Exception if anything goes wrong, page times out, etc
'''
Reddit.wait()
r = Reddit.httpy.get('http://www.reddit.com/r/AmateurArchives/wiki/illicit.json')
json = loads(r)
wiki = json['data']['content_md']
illicit = []
for url in Reddit.httpy.between(r, '](http://', ')'):
if not 'imgur' in url: continue
url = 'http://%s' % url
illicit.append(url)
return illicit
def backfill_comments():
(username, password) = db.get_credentials('reddit')
reddit.login(username, password)
cur = db.conn.cursor()
query = '''
select
id,
userid,
postid,
subreddit,
text,
created,
legacy,
permalink,
ups,
downs
from comments
where legacy = 1
order by id
'''
execur = cur.execute(query)
results = execur.fetchall()
for (commentid,
userid,
postid,
subreddit,
text,
created,
legacy,
permalink,
ups,
downs) in results:
# Get comment from reddit
post = Reddit.get('http://www.reddit.com/comments/%s/_/%s' % (postid, commentid))
if len(post.comments) > 0:
comment = post.comments[0]
# Update db
query = '''
update comments
set
postid = ?,
subreddit = ?,
text = ?,
created = ?,
permalink = ?,
legacy = 0,
ups = ?,
downs = ?
where
id = ?
'''
cur.execute(query, (postid, subreddit, text, created, permalink, legacy, ups, downs, commentid) )
db.commit()
cur.close()
def __init__(self):
# Single file that all output is written to, to track usage
self.exit_if_already_started()
self.db = DB() # Database instance
log_level = self.db.get_config('log_level', default='user')
if log_level == 'none':
self.root_log = open(devnull, 'w')
else:
self.root_log = open(path.join(ImageUtils.get_root(), 'history.log'), 'a')
self.logger = self.root_log # Logger used by helper classes
self.reddit = Reddit()
self.excluded_subs = self.db.get_excluded_subreddits()
def backfill_comments():
(username, password) = db.get_credentials('reddit')
reddit.login(username, password)
cur = db.conn.cursor()
query = '''
select
id,
userid,
postid,
subreddit,
text,
created,
legacy,
permalink,
ups,
downs
from comments
where legacy = 1
order by id
'''
execur = cur.execute(query)
results = execur.fetchall()
for (commentid, userid, postid, subreddit, text, created, legacy,
permalink, ups, downs) in results:
# Get comment from reddit
post = Reddit.get('http://www.reddit.com/comments/%s/_/%s' %
(postid, commentid))
if len(post.comments) > 0:
comment = post.comments[0]
# Update db
query = '''
update comments
set
postid = ?,
subreddit = ?,
text = ?,
created = ?,
permalink = ?,
legacy = 0,
ups = ?,
downs = ?
where
id = ?
'''
cur.execute(query, (postid, subreddit, text, created, permalink,
legacy, ups, downs, commentid))
db.commit()
cur.close()
class Expunger:
deleteMsg = "Deleted"
shouldDelete = False
verboseMode = False
def __init__(self, username, password, clientId, clientSecret):
self.reddit = Reddit(username, password, clientId, clientSecret)
def run(self):
allComments = self.reddit.getComments()
if self.reddit.login() is True:
for commentId in allComments:
deleteMsg = self.deleteMsg + " " + self.randomString(8)
editSuccess = self.reddit.editComment(commentId, deleteMsg)
print(commentId + " - Edit " +
("Successful" if editSuccess else "Failed"))
if self.verboseMode:
print("New comment body: \"" + deleteMsg + "\"")
print("Operation completed")
else:
print("Login failed - check your details")
def setDeleteMsg(self, msg):
self.deleteMsg = msg
def setVerboseMode(self, value):
self.verboseMode = value
def randomString(self, length):
return "".join(
random.choice(string.ascii_lowercase + string.ascii_uppercase +
string.digits) for _ in range(length))
class Test_Reddit_test(unittest.TestCase):
def setUp(self):
self.reddit = Reddit()
return super().setUp()
def test_connect_as_read(self):
self.reddit.open_reddit_read()
self.assertTrue(self.reddit.is_read_only())
def test_connect_as_write(self):
self.reddit.open_reddit_write()
self.assertFalse(self.reddit.is_read_only())
def test_returns_generator(self):
self.reddit.open_reddit_read()
top_submissions = 5
sut = self.reddit.top_submissions('u_hansknecht', top_submissions)
count = 0
for submission in sut:
count += 1
print(submission.title)
self.assertEqual(count, top_submissions,
"The top ten submissions were not collected")
def __init__(self):
self.data = Data()
self.teams = Teams()
self.scoreboard = Scoreboard()
self.reddit = Reddit('nba')
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from HackerNews import HackerNews
from Reddit import Reddit
from RSS import RSS
from Tweet import Tweet
# CONSTANTS
RETRIES = 10 # Number of retries on URL failure
WAIT_TIME = 20 * 60 # Time to wait between posting (in seconds)
tweet = Tweet()
if (tweet.check_last_tweet()):
news = HackerNews(WAIT_TIME)
news.tweet_stories(5)
reddit = Reddit(WAIT_TIME)
reddit.tweet_stories(3)
rss = RSS(WAIT_TIME)
rss.tweet_stories(2)
class Gonewild(object):
def __init__(self):
# Single file that all output is written to, to track usage
self.exit_if_already_started()
self.db = DB() # Database instance
log_level = self.db.get_config('log_level', default='user')
if log_level == 'none':
self.root_log = open(devnull, 'w')
else:
self.root_log = open(
path.join(ImageUtils.get_root(), 'history.log'), 'a')
self.logger = self.root_log # Logger used by helper classes
self.reddit = Reddit()
self.excluded_subs = self.db.get_excluded_subreddits()
def debug(self, text):
tstamp = strftime('[%Y-%m-%dT%H:%M:%SZ]', gmtime())
text = '%s Gonewild: %s' % (tstamp, text)
self.root_log.write('%s\n' % text)
if self.logger != self.root_log:
self.logger.write('%s\n' % text)
stderr.write('%s\n' % text)
def user_already_added(self, user):
return self.db.user_already_added(user)
def user_has_gone_wild(self, user):
# Look at last 100 submissions
try:
children = self.reddit.get_user('%s/submitted' % user, max_pages=1)
except Exception:
# User is 404
return False
for child in children:
if type(child) == Post:
if 'gonewild' in child.subreddit.lower() or \
'gw' in child.subreddit.lower() or \
'asstastic' in child.subreddit.lower() or \
'girlsgone' in child.subreddit.lower() or \
'gone' in child.subreddit.lower():
return True
return False
def add_excluded_subreddit(self, subreddit):
return self.db.add_excluded_subreddit(subreddit)
def setup_loggers_for_user(self, user):
# Create directories if needed
user_dir = path.join(ImageUtils.get_root(), 'content', user)
ImageUtils.create_subdirectories(user_dir)
# Setup logger
log_level = self.db.get_config('log_level', default='user')
if log_level == 'none': self.logger = open(devnull, 'w')
elif log_level == 'user':
self.logger = open(path.join(user_dir, 'history.log'), 'a')
elif log_level == 'global':
self.logger = self.root_log
self.db.logger = self.logger
ImageUtils.logger = self.logger
self.reddit.logger = self.logger
def restore_loggers(self):
log_level = self.db.get_config('log_level', default='user')
if log_level == 'user':
self.logger.close()
self.logger = self.root_log
self.db.logger = self.logger
ImageUtils.logger = self.logger
self.reddit.logger = self.logger
def is_excluded_child(self, child):
if child.subreddit.lower() in [x.lower() for x in self.excluded_subs]:
self.debug(
'''%s: poll_user: Ignoring post/comment in excluded subreddit ("%s")
Permalink: %s
Ignored: %s''' %
(child.author, child.subreddit, child.permalink(), str(child)))
return True
return False
def get_and_process_urls_from_child(self, child):
urls = self.get_urls(child)
try:
if type(child) == Post:
self.db.add_post(child)
elif type(child) == Comment:
self.db.add_comment(child)
except Exception, e:
if 'already exists' not in str(e):
self.debug('%s: poll_user: %s' % (child.author, str(e)))
return # If we can't add the post/comment to DB, skip it
if len(urls) > 0:
self.debug('%s: poll_user: found %d url(s) in child %s' %
(child.author, len(urls), child.permalink()))
for url_index, url in enumerate(urls):
self.process_url(url, url_index, child)
def gonewild_check(child, json, db, log):
'''
Check if image(s) in child were posted to gonewild
and the child author is not the original gonewild author.
Remove child and explain in comment.
Args:
child: reddit child (post)
json: results from rarchives
Returns:
True if post originated on gonewild, False otherwise.
'''
# Disable this on /r/AmateurArchives
if child.subreddit.lower() == 'amateurarchives': return False
# XXX Only enforce posts by gonewild users (not comments)
for post in json['posts']:
if post['subreddit'].lower() == 'gonewild' and \
post['author'].lower() != child.author.lower() and \
post['author'] != '[deleted]':
# Child author is not the gonewild user, image *is* though
# Remove and comment the source
child.remove(mark_as_spam=False)
log('Rarchives.gonewild_check: Removed %s - matches /u/%s @ http://reddit.com%s' % (child.permalink(), post['author'], post['permalink']))
title = post['title'].replace('[', '\\[').replace('(', '\\(').replace(']', '\\]').replace(')', '\\)').replace('*', '')
body = '## This post was removed because it is a repost of a /r/gonewild submission:\n\n'
body += '* **/u/%s** submitted [*%s*](%s) %s' % (post['author'], title, post['permalink'], Reddit.utc_timestamp_to_hr(post['created']))
try:
response = child.reply(body)
response.distinguish()
child.flair('Gonewild repost: /u/%s' % post['author'])
except Exception, e:
log('Rarchives.gonewild_check: Error while replying to %s : %s' % (child.permalink(), str(e)))
# Update 'log_amarch' db table
db.insert('log_amarch', ('removed', child.permalink(), timegm(gmtime()), 'gonewild repost of /u/%s: %s' % (post['author'], post['permalink'])))
return True
SK = Skyrock(json['login'], json['pass'], json['text'], json['link'],
json['datetime'], json['page_post'], json['groupe_post'],
json['title'], json['image_link'], json['hashtags'],
json['gropes_links'], json['priority'])
SK.post()
print(SK.get_bugrep())
if json['id'] == 'wh':
WH = WeHear(json['login'], json['pass'], json['text'], json['link'],
json['datetime'], json['page_post'], json['groupe_post'],
json['title'], json['image_link'], json['hashtags'],
json['gropes_links'], json['priority'])
WH.post()
print(WH.get_bugrep())
if json['id'] == 'rd':
RD = Reddit(json['login'], json['pass'], json['text'], json['link'],
json['datetime'], json['page_post'], json['groupe_post'],
json['title'], json['image_link'], json['hashtags'],
json['gropes_links'], json['priority'])
RD.post()
print(RD.get_bugrep())
if json['id'] == 'li':
LI = LinkedIn(json['login'], json['pass'], json['text'], json['link'],
json['datetime'], json['page_post'], json['groupe_post'],
json['title'], json['image_link'], json['hashtags'],
json['gropes_links'], json["priority"])
LI.post()
print(LI.get_bugrep())
if json['id'] == 'tr':
TR = Tumblr(json['login'], json['pass'], json['text'], json['link'],
json['datetime'], json['page_post'], json['groupe_post'],
json['title'], json['image_link'], json['hashtags'],
json['gropes_links'])
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Document</title>
</head>
<body>
<div class="container mb-5">
<h1>Your feed for {today.strftime('%d %b %Y')}</h1>
""")
w = Weather()
w.write_weather()
e = Events()
e.write_events()
r = Reddit()
with open(feed_path, "a") as file:
file.write("<h2>Reddit</h2>")
r.fetch_subs_and_write("reactjs")
r.fetch_subs_and_write("webdev")
r.fetch_subs_and_write("frontend")
r.fetch_subs_and_write("python")
r.fetch_subs_and_write("rollerblading")
with open(feed_path, "a") as file:
file.write("<h2>CNA</h2>")
cna = CNA()
def lambda_handler(event, context):
red = Reddit()
return red.scan()
asstastic
'''
SUBS = [x.strip() for x in SUBS_TEXT.strip().split('\n')]
while '' in SUBS: SUBS.remove('')
db = DB()
last_post = db.get_config('last_post')
reddit_url = 'http://www.reddit.com/r/%s/new.json' % '+'.join(SUBS)
print 'firehose from %s' % reddit_url
while True:
sleep(2)
try:
posts = Reddit.get(reddit_url)
except Exception, e:
#print 'error when querying %s: %s' % (reddit_url, str(e))
continue
for post in posts:
if last_post != None and post.id == last_post:
break
if post.selftext != None:
# TODO self-text, skip it
continue
url = post.url
shorturl = 'http://redd.it/%s' % post.id
subreddit = post.subreddit
author = post.author
title = post.title
print ','.join( [shorturl, url, subreddit, author, title] )
from Reddit import Reddit
if __name__ == '__main__':
f = open("dictionaries/insultos.txt", "r")
paraulesclau = f.readlines()
f.close()
f = open("dictionaries/topics.txt", "r")
paraulesclau += f.readlines()
f.close()
index = 0
for p in paraulesclau:
paraulesclau[index] = p[:-1].lower()
index += 1
# print(paraulesclau)
#inicialitzem reddit
Reddit('comments', paraulesclau)
class Gonewild(object):
def __init__(self):
# Single file that all output is written to, to track usage
self.exit_if_already_started()
self.db = DB() # Database instance
log_level = self.db.get_config('log_level', default='user')
if log_level == 'none':
self.root_log = open(devnull, 'w')
else:
self.root_log = open(path.join(ImageUtils.get_root(), 'history.log'), 'a')
self.logger = self.root_log # Logger used by helper classes
self.reddit = Reddit()
self.excluded_subs = self.db.get_excluded_subreddits()
def debug(self, text):
tstamp = strftime('[%Y-%m-%dT%H:%M:%SZ]', gmtime())
text = '%s Gonewild: %s' % (tstamp, text)
self.root_log.write('%s\n' % text)
if self.logger != self.root_log:
self.logger.write('%s\n' % text)
stderr.write('%s\n' % text)
def user_already_added(self, user):
return self.db.user_already_added(user)
def user_has_gone_wild(self, user):
# Look at last 100 submissions
try:
children = self.reddit.get_user('%s/submitted' % user, max_pages=1)
except Exception:
# User is 404
return False
for child in children:
if type(child) == Post:
if 'gonewild' in child.subreddit.lower() or \
'gw' in child.subreddit.lower() or \
'asstastic' in child.subreddit.lower() or \
'girlsgone' in child.subreddit.lower() or \
'gone' in child.subreddit.lower():
return True
return False
def add_excluded_subreddit(self, subreddit):
return self.db.add_excluded_subreddit(subreddit)
def setup_loggers_for_user(self, user):
# Create directories if needed
user_dir = path.join(ImageUtils.get_root(), 'content', user)
ImageUtils.create_subdirectories(user_dir)
# Setup logger
log_level = self.db.get_config('log_level', default='user')
if log_level == 'none': self.logger = open(devnull, 'w')
elif log_level == 'user': self.logger = open(path.join(user_dir, 'history.log'), 'a')
elif log_level == 'global': self.logger = self.root_log
self.db.logger = self.logger
ImageUtils.logger = self.logger
self.reddit.logger = self.logger
def restore_loggers(self):
log_level = self.db.get_config('log_level', default='user')
if log_level == 'user':
self.logger.close()
self.logger = self.root_log
self.db.logger = self.logger
ImageUtils.logger = self.logger
self.reddit.logger = self.logger
def is_excluded_child(self, child):
if child.subreddit.lower() in [x.lower() for x in self.excluded_subs]:
self.debug('''%s: poll_user: Ignoring post/comment in excluded subreddit ("%s")
Permalink: %s
Ignored: %s''' % (child.author, child.subreddit, child.permalink(), str(child)))
return True
return False
def get_and_process_urls_from_child(self, child):
urls = self.get_urls(child)
try:
if type(child) == Post:
self.db.add_post(child)
elif type(child) == Comment:
self.db.add_comment(child)
except Exception, e:
if 'already exists' not in str(e):
self.debug('%s: poll_user: %s' % (child.author, str(e)))
return # If we can't add the post/comment to DB, skip it
if len(urls) > 0:
self.debug('%s: poll_user: found %d url(s) in child %s' % (child.author, len(urls), child.permalink()))
for url_index, url in enumerate(urls):
self.process_url(url, url_index, child)
def __init__(self, username, password, clientId, clientSecret):
self.reddit = Reddit(username, password, clientId, clientSecret)
'''
Created on Oct 14, 2017
@author: Tyler-OC
'''
from Discord_API import Discord
from Database import Database
from Reddit import Reddit
import time
discord = Discord()
Data = Database()
Red = Reddit()
count = 0
while True:
Page = Red.getPage("https://www.reddit.com/r/buildapcsales/")
Products = Red.getProducts(Page)
for product in Products:
if "[monitor]" in product.lower():
if Data.CheckProduct(product):
print("Product in database!")
else:
Data.newProduct(product, Products[product])
Message = "There is a new Product to Check out! \n" + product + "\n" + Products[product]
discord.sendMessage(Message)
from SQLHandler import SQLHandler
from Reddit import Reddit
from Analyzer import Analyzer
import time
def thenOrThan(words):
return "then" if "then" in words else "than"
count = 0
while True:
sql = SQLHandler()
reddit = Reddit()
analyzer = Analyzer()
count += 1
print "Beginning Iteration " + str(count)
print "Gathering Comments"
comments = reddit.getNewComments()
views = sql.getViews()
print "Processing Data"
for comment in comments:
flag = True
words = [x.lower().strip() for x in comment[0].split()]
thenorthan = thenOrThan(words)
for i in views:
if comment[1] == i[0]:
print '[!] inserting "blank" admin'
total_scores = db.select_one('sum(score)', 'admins', 'username != ""')
db.update('admins', 'score = ?', 'username = ""', [total_removed - total_scores])
print '[!] updated "blank" admin with score: %d' % total_scores
db.commit()
#######################
# Login to reddit
print '[ ] logging in to reddit'
username = db.get_config('reddit_user')
password = db.get_config('reddit_pw')
if username == None or password == None:
print '[!] "reddit_user" or "reddit_pw" were not found in "config" table in database'
Reddit.login(username, password)
print '[ ] executing AmArch.py functionality...'
def log(txt): print txt
AmArch.execute(db, log)
# MODERATED SUBS (real-time)
print '[ ] finding modded subs...'
current = Reddit.get_modded_subreddits()
print '[ ] found %d subs modded by /u/%s' % (len(current), username)
# remove subs that should be ignored
to_ignore = db.get_config('ignore_subreddits')
if to_ignore != None:
for ignore in to_ignore.split(','):
#!/usr/bin/python
"""
"Backfill" existing database data.
Gets titles, permalinks, dates, etc from reddit.
Overwrites existing 'bad' data with accurate data.
"""
from DB import DB
from Reddit import Reddit
from sys import exit, stdout
from urllib2 import HTTPError
from os import remove as osremove
from ImageUtils import ImageUtils
db = DB()
reddit = Reddit()
def backfill_users():
q_users = '''
select id,username
from users
where deleted = 0
'''
cur = db.conn.cursor()
execur = cur.execute(q_users)
ids_and_users = execur.fetchall() # Get list of users + ids
index = 0
for (userid, username) in ids_and_users:
index += 1
print '(%d/%d) updating %s...' % (index, len(ids_and_users), username),
pass
elif (sys.argv[flag] == "--image" or sys.argv[flag] == "-i"):
pass
elif (sys.argv[flag] == "--nsfw" or sys.argv[flag] == "-NSFW"):
pass
elif (sys.argv[flag] == "--" or sys.argv[flag][0] != "-"):
break
else:
print("invalid flag", sys.argv[flag])
usage()
r = Reddit(User)
bot = 'bot1'
d = Display(praw.Reddit(bot, user_agent=bot))
if (argc > shift + 1
and (sys.argv[shift + 1] == "list" or sys.argv[shift + 1] == "l")):
pass
elif (argc > shift + 1
and (sys.argv[shift + 1] == "print" or sys.argv[shift + 1] == "p")):
pass
elif (argc > shift + 1
and (sys.argv[shift + 1] == "collect" or sys.argv[shift + 1] == "c")):
payload = r.updown(Sub)
def main():
# Just a quick note. I prefer camelCase over underscores so that is what I used for the variable names I changed.
authFile = "auth.txt"
creds = import_credentials(authFile)
if creds is False or len(creds) < 3:
print("The OAuth file is not valid, or it does not exist.")
print("Please provide valid OAuth credentials in ",
authFile,
".",
sep="")
sys.exit(
) # sys.exit() is I believe the more pythonic way of doing this.
# sys.exit() is meant more for programs where as exit(0) is meant for the interactive console. I think.
else:
reddit = Reddit(*creds)
# By creating a Reddit object we don't need to worry about passing credentials when we want to use the,
inital_user = input("Enter username (enter 'q' to quit): ")
if inital_user == "q": # Why don't you just do this?
sys.exit()
print("Getting original user's top 5 subreddits...")
# Original user's top 5 subreddits
top_subreddits = reddit.ranked_subreddits(
reddit.full_comment_and_submission_history(inital_user), max_count=10)
user_scores = {}
# Set this to the maximum number of latest users you want from each subreddit
MAX_USERS_PER_SUBREDDIT = 100
# Last 100 users in each of the top 5 subreddits
for sub_count, sub_reddit in enumerate(
top_subreddits
): # enumerate gets you a iterable and an index variable
# In this case sub_count was just the index+1
print("Getting last 100 users in " + sub_reddit[0] + " (" +
str(sub_count + 1) + " of " + str(len(top_subreddits)) + ")...")
users_list = reddit.last_subreddit_users(
sub_reddit[0], max_count=MAX_USERS_PER_SUBREDDIT)
# Top 5 subreddits for each of the users in the current subreddit
user_count = 1
for user in users_list:
print("Getting top subreddits for user " + str(user_count) +
" of " + str(len(users_list)) + "...",
end="\r",
flush=True)
user_top_subreddits = reddit.ranked_subreddits(
reddit.full_comment_and_submission_history(user), max_count=5)
score = 1
user_count += 1
for comp_sub in user_top_subreddits:
if comp_sub in top_subreddits:
score += 1
if score > 1:
user_scores.update({user: score})
print("\nMatched users so far: " + str(user_scores))
matched_users = sorted(user_scores.items(),
key=operator.itemgetter(1),
reverse=True)
print("\nUsers most like " + inital_user + ":")
for match in matched_users:
print("User: "******" | Score: " + str(match[1]))
""" "Backfill" existing database data. Gets titles, permalinks, dates, etc from reddit. Overwrites existing 'bad' data with accurate data. """ from DB import DB from Reddit import Reddit from sys import exit, stdout from urllib2 import HTTPError from os import remove as osremove from ImageUtils import ImageUtils db = DB() reddit = Reddit() def backfill_users(): q_users = ''' select id,username from users where deleted = 0 ''' cur = db.conn.cursor() execur = cur.execute(q_users) ids_and_users = execur.fetchall() # Get list of users + ids index = 0 for (userid, username) in ids_and_users: index += 1 print '(%d/%d) updating %s...' % (index, len(ids_and_users), username), stdout.flush()
from SQLHandler import SQLHandler
from Reddit import Reddit
from Analyzer import Analyzer
import time
def thenOrThan(words):
return "then" if "then" in words else "than"
count = 0
while True:
sql = SQLHandler()
reddit = Reddit()
analyzer = Analyzer()
count += 1
print "Beginning Iteration " + str(count)
print "Gathering Comments"
comments = reddit.getNewComments()
views = sql.getViews()
print "Processing Data"
for comment in comments:
flag = True
words = [x.lower().strip() for x in comment[0].split()]
thenorthan = thenOrThan(words)
def setUp(self):
self.reddit = Reddit()
return super().setUp()