Exemple #1
0
	def get_blacklisted_users():
		'''
			Returns:
				List of blacklisted users from wiki
			Raises:
				Exception if anything goes wrong, page times out, etc
		'''
		Reddit.wait()
		r = Reddit.httpy.get('http://www.reddit.com/r/AmateurArchives/wiki/banned.json')
		json = loads(r)
		wiki = json['data']['content_md']
		lines = wiki.split('\r\n')
		blacklisted = []
		for line in lines:
			if not '|' in line:
				continue
			fields = line.split('|')
			if len(fields) != 5:
				continue
			if fields[1] in ['username', ':--']:
				continue
			for user in fields[1].replace('/u/', '').split('/'):
				user = user.strip()
				if user == '': continue
				blacklisted.append(user)
		return blacklisted
Exemple #2
0
def get_top_submissions():
    titles = "<ul>"
    reddit = Reddit()
    reddit.open_reddit_read()
    submissions = reddit.top_submissions('u_hansknecht', 5)
    for entry in submissions:
        titles += "<li>{0}</li>".format(entry.title)
    titles += "</ul>"
    return titles
Exemple #3
0
	def is_valid_request(child, db, log):
		'''
			Ensures request is from an account older than MINIMUM_REQUESTER_AGE days,
			and the accounts last request was over MINIMUM_REQUEST_DAYS days ago.
			If not, removes the request and comments with the reason for removal

			Returns:
				True if post is valid request,
				False if request is not valid and was removed.
		'''
		if type(child) != Post: return True

		request_is_valid = False

		# Check if last request was < MINIMUM_REQUEST_DAYS days ago
		now = timegm(gmtime())
		for (date, permalink) in db.select('date, permalink', 'amarch_requests', 'username = ?', [child.author]):
			if date + (3600 * 24 * AmArch.MINIMUM_REQUEST_DAYS) > now:
				# Last request was < MINIMUM_REQUEST_DAYS days ago, check if the request was 'removed'
				post = Reddit.get(permalink)
				if post.banned_by == None:
					# Last request was < MINIMUM_REQUEST_DAYS days ago, wasn't removed
					child.remove(mark_as_spam=False)
					log('AmArch.is_valid_request: Request < %d days old: %s' % (AmArch.MINIMUM_REQUEST_DAYS, child.permalink()))
					body  = '## Rule: [Requests must be at least %d days apart](/r/AmateurArchives/about/sidebar)\n\n' % AmArch.MINIMUM_REQUEST_DAYS
					body += 'The [**last request**](%s) from your account was submitted %s' % (permalink, Reddit.utc_timestamp_to_hr(post.created))
					response = child.reply(body)
					response.distinguish()
					child.flair('last req < %dd' % AmArch.MINIMUM_REQUEST_DAYS)
					return False
				else:
					# XXX OPTIMIZATION
					# Last request was > MINIMUM_REQUEST_DAYS days ago but was removed
					# Therefore: User account must be > MINIMUM_REQUESTER_AGE days old
					request_is_valid = True

		if not request_is_valid:
			# Check if user is < MINIMUM_REQUESTER_AGE days old
			user = Reddit.get_user_info(child.author)
			if user.created > now - (3600 * 24 * AmArch.MINIMUM_REQUESTER_AGE):
				child.remove(mark_as_spam=False)
				log('AmArch.is_valid_request: Requester /u/%s < %d days old: %s' % (child.author, AmArch.MINIMUM_REQUESTER_AGE, child.permalink()))
				body  = '## Rule: [Requests must be from accounts more than %d days old](/r/AmateurArchives/about/sidebar)\n\n' % AmArch.MINIMUM_REQUESTER_AGE
				body += 'The account (/u/%s) was created %s.' % (child.author, Reddit.utc_timestamp_to_hr(user.created))
				response = child.reply(body)
				response.distinguish()
				child.flair('user < %dd' % AmArch.MINIMUM_REQUESTER_AGE)
				return False

		# Request is valid. Add it to the database for checking in the future
		log('AmArch.is_valid_request: Allowing request from /u/%s' % child.author)
		if db.count('amarch_requests', 'username = ?', [child.author]) == 0:
			db.insert('amarch_requests', (child.author, child.created, child.permalink()))
		else:
			db.update('amarch_requests', 'date = ?, permalink = ?', 'username = ?', [child.created, child.permalink(), child.author])
		return True
Exemple #4
0
def backfill_posts(legacy=True):

    (username, password) = db.get_credentials('reddit')
    reddit.login(username, password)

    where = ''
    if legacy:
        where = 'where legacy = 1'
    cur = db.conn.cursor()
    query = '''
		select id, userid, title, selftext, url, subreddit, over_18, created, legacy, permalink, ups, downs
			from posts
			%s
			order by id
	''' % where
    total = 0
    ids_to_fetch = []
    # Store existing values in dict
    for (postid, userid, title, selftext, url, subreddit, over_18, created,
         legacy, permalink, ups, downs) in cur.execute(query):
        ids_to_fetch.append(str(postid))
        if len(ids_to_fetch) >= 99:
            total += len(ids_to_fetch)
            ids_to_fetch.append('1234')
            url = 'http://www.reddit.com/by_id/t3_%s.json' % ',t3_'.join(
                ids_to_fetch)
            try:
                posts = reddit.get(url)
            except HTTPError, e:
                print 'HTTPError: %s' % str(e)
                posts = []
            for post in posts:
                oldpost = {}
                oldpost['title'] = post.title
                oldpost['url'] = post.url
                oldpost['selftext'] = post.selftext
                oldpost['subreddit'] = post.subreddit
                oldpost['created'] = int(post.created)
                oldpost['permalink'] = post.permalink()
                oldpost['over_18'] = int(post.over_18)
                oldpost['legacy'] = 0
                oldpost['id'] = post.id.rjust(6, '0')
                oldpost['ups'] = post.ups
                oldpost['downs'] = post.downs
                Reddit.debug('updating post %s by %s' % (post.id, post.author))
                update_post(oldpost)
            db.conn.commit()
            ids_to_fetch = list()
            print 'running total: %d' % total
Exemple #5
0
    def __init__(self):
        # Single file that all output is written to, to track usage
        self.exit_if_already_started()
        self.db = DB()  # Database instance

        log_level = self.db.get_config('log_level', default='user')
        if log_level == 'none':
            self.root_log = open(devnull, 'w')
        else:
            self.root_log = open(
                path.join(ImageUtils.get_root(), 'history.log'), 'a')
        self.logger = self.root_log  # Logger used by helper classes

        self.reddit = Reddit()
        self.excluded_subs = self.db.get_excluded_subreddits()
Exemple #6
0
def backfill_users():
	q_users = '''
		select id,username
			from users
			where deleted = 0
	'''
	cur = db.conn.cursor()
	execur = cur.execute(q_users)
	ids_and_users = execur.fetchall() # Get list of users + ids
	index = 0
	for (userid, username) in ids_and_users:
		index += 1
		print('(%d/%d) updating %s...' % (index, len(ids_and_users), username)),
		stdout.flush()
		try:
			ui = Reddit.get_user_info(username)
		except Exception as e:
			print(str(e))
			continue
		q_user = '''
			update users 
				set
					created  = %d,
					username = "******"
				where id = %d
		''' % (ui.created, ui.name, userid)
		cur.execute(q_user)
		print('done')
	
	cur.close()
Exemple #7
0
def backfill_users():
    q_users = '''
		select id,username
			from users
			where deleted = 0
	'''
    cur = db.conn.cursor()
    execur = cur.execute(q_users)
    ids_and_users = execur.fetchall()  # Get list of users + ids
    index = 0
    for (userid, username) in ids_and_users:
        index += 1
        print '(%d/%d) updating %s...' % (index, len(ids_and_users), username),
        stdout.flush()
        try:
            ui = Reddit.get_user_info(username)
        except Exception, e:
            print str(e)
            continue
        q_user = '''
			update users 
				set
					created  = %d,
					username = "******"
				where id = %d
		''' % (ui.created, ui.name, userid)
        cur.execute(q_user)
        print 'done'
Exemple #8
0
def backfill_posts(legacy=True):

	(username, password) = db.get_credentials('reddit')
	reddit.login(username, password)

	where = ''
	if legacy:
		where = 'where legacy = 1'
	cur = db.conn.cursor()
	query = '''
		select id, userid, title, selftext, url, subreddit, over_18, created, legacy, permalink, ups, downs
			from posts
			%s
			order by id
	''' % where
	total = 0
	ids_to_fetch = []
	# Store existing values in dict
	for (postid, userid, title, selftext, url, subreddit, over_18, created, legacy, permalink, ups, downs) in cur.execute(query):
		ids_to_fetch.append(str(postid))
		if len(ids_to_fetch) >= 99:
			total += len(ids_to_fetch)
			ids_to_fetch.append('1234')
			url = 'http://www.reddit.com/by_id/t3_%s.json' % ',t3_'.join(ids_to_fetch)
			try:
				posts = reddit.get(url)
			except HTTPError, e:
				print 'HTTPError: %s' % str(e)
				posts = []
			for post in posts:
				oldpost = {}
				oldpost['title']     = post.title
				oldpost['url']       = post.url
				oldpost['selftext']  = post.selftext
				oldpost['subreddit'] = post.subreddit
				oldpost['created']   = int(post.created)
				oldpost['permalink'] = post.permalink()
				oldpost['over_18']   = int(post.over_18)
				oldpost['legacy']    = 0
				oldpost['id']        = post.id.rjust(6, '0')
				oldpost['ups']       = post.ups
				oldpost['downs']     = post.downs
				Reddit.debug('updating post %s by %s' % (post.id, post.author))
				update_post(oldpost)
			db.conn.commit()
			ids_to_fetch = list()
			print 'running total: %d' % total
Exemple #9
0
	def __init__(self):
		# Single file that all output is written to, to track usage
		self.exit_if_already_started()
		self.root_log = open(path.join(ImageUtils.get_root(), 'history.log'), 'a')
		self.logger   = self.root_log # Logger used by helper classes
		self.db       = DB() # Database instance
		self.reddit   = Reddit()
		self.excluded_subs = self.db.get_excluded_subreddits()
Exemple #10
0
 def __init__(self):
     # Single file that all output is written to, to track usage
     self.exit_if_already_started()
     self.root_log = open(path.join(ImageUtils.get_root(), 'history.log'),
                          'a')
     self.logger = self.root_log  # Logger used by helper classes
     self.db = DB()  # Database instance
     self.reddit = Reddit()
     try:
         (username, password) = self.db.get_credentials('reddit')
         try:
             self.reddit.login(username, password)
         except Exception, e:
             self.debug('__init__: failed to login to reddit: %s' % str(e))
     except Exception, e:
         self.debug('__init__: failed to get reddit credentials: %s' %
                    str(e))
Exemple #11
0
class Gonewild(object):

	def __init__(self):
		# Single file that all output is written to, to track usage
		self.exit_if_already_started()
		self.root_log = open(path.join(ImageUtils.get_root(), 'history.log'), 'a')
		self.logger   = self.root_log # Logger used by helper classes
		self.db       = DB() # Database instance
		self.reddit   = Reddit()
		try:
			(username, password) = self.db.get_credentials('reddit')
			try:
				self.reddit.login(username, password)
			except Exception, e:
				self.debug('__init__: failed to login to reddit: %s' % str(e))
		except Exception, e:
			self.debug('__init__: failed to get reddit credentials: %s' % str(e))
Exemple #12
0
	def get_blacklisted_urls():
		'''
			Returns:
				List of blacklisted urls from wiki
			Raises:
				Exception if anything goes wrong, page times out, etc
		'''
		Reddit.wait()
		r = Reddit.httpy.get('http://www.reddit.com/r/AmateurArchives/wiki/illicit.json')
		json = loads(r)
		wiki = json['data']['content_md']
		illicit = []
		for url in Reddit.httpy.between(r, '](http://', ')'):
			if not 'imgur' in url: continue
			url = 'http://%s' % url
			illicit.append(url)
		return illicit
Exemple #13
0
def backfill_comments():
	(username, password) = db.get_credentials('reddit')
	reddit.login(username, password)

	cur = db.conn.cursor()
	query = '''
		select
				id,
				userid,
				postid,
				subreddit,
				text,
				created,
				legacy,
				permalink,
				ups,
				downs
		from comments
		where legacy = 1
		order by id
	'''
	execur = cur.execute(query)
	results = execur.fetchall()

	for (commentid,
	     userid,
	     postid,
	     subreddit,
	     text,
	     created,
	     legacy,
	     permalink,
	     ups,
	     downs) in results:
		# Get comment from reddit
		post = Reddit.get('http://www.reddit.com/comments/%s/_/%s' % (postid, commentid))
		if len(post.comments) > 0:
			comment = post.comments[0]
			# Update db
			query = '''
				update comments
					set
						postid    = ?,
						subreddit = ?,
						text      = ?,
						created   = ?,
						permalink = ?,
						legacy    = 0,
						ups       = ?,
						downs     = ?
					where
						id = ?
			'''
			cur.execute(query, (postid, subreddit, text, created, permalink, legacy, ups, downs, commentid) )
			db.commit()
	cur.close()
Exemple #14
0
	def __init__(self):
		# Single file that all output is written to, to track usage
		self.exit_if_already_started()
		self.db = DB() # Database instance

		log_level = self.db.get_config('log_level', default='user')
		if log_level == 'none':
			self.root_log = open(devnull, 'w')
		else:
			self.root_log = open(path.join(ImageUtils.get_root(), 'history.log'), 'a')
		self.logger   = self.root_log # Logger used by helper classes

		self.reddit   = Reddit()
		self.excluded_subs = self.db.get_excluded_subreddits()
Exemple #15
0
def backfill_comments():
    (username, password) = db.get_credentials('reddit')
    reddit.login(username, password)

    cur = db.conn.cursor()
    query = '''
		select
				id,
				userid,
				postid,
				subreddit,
				text,
				created,
				legacy,
				permalink,
				ups,
				downs
		from comments
		where legacy = 1
		order by id
	'''
    execur = cur.execute(query)
    results = execur.fetchall()

    for (commentid, userid, postid, subreddit, text, created, legacy,
         permalink, ups, downs) in results:
        # Get comment from reddit
        post = Reddit.get('http://www.reddit.com/comments/%s/_/%s' %
                          (postid, commentid))
        if len(post.comments) > 0:
            comment = post.comments[0]
            # Update db
            query = '''
				update comments
					set
						postid    = ?,
						subreddit = ?,
						text      = ?,
						created   = ?,
						permalink = ?,
						legacy    = 0,
						ups       = ?,
						downs     = ?
					where
						id = ?
			'''
            cur.execute(query, (postid, subreddit, text, created, permalink,
                                legacy, ups, downs, commentid))
            db.commit()
    cur.close()
class Expunger:
    deleteMsg = "Deleted"

    shouldDelete = False
    verboseMode = False

    def __init__(self, username, password, clientId, clientSecret):
        self.reddit = Reddit(username, password, clientId, clientSecret)

    def run(self):
        allComments = self.reddit.getComments()

        if self.reddit.login() is True:
            for commentId in allComments:
                deleteMsg = self.deleteMsg + " " + self.randomString(8)
                editSuccess = self.reddit.editComment(commentId, deleteMsg)

                print(commentId + " - Edit " +
                      ("Successful" if editSuccess else "Failed"))

                if self.verboseMode:
                    print("New comment body: \"" + deleteMsg + "\"")

            print("Operation completed")
        else:
            print("Login failed - check your details")

    def setDeleteMsg(self, msg):
        self.deleteMsg = msg

    def setVerboseMode(self, value):
        self.verboseMode = value

    def randomString(self, length):
        return "".join(
            random.choice(string.ascii_lowercase + string.ascii_uppercase +
                          string.digits) for _ in range(length))
class Test_Reddit_test(unittest.TestCase):
    def setUp(self):
        self.reddit = Reddit()
        return super().setUp()

    def test_connect_as_read(self):
        self.reddit.open_reddit_read()
        self.assertTrue(self.reddit.is_read_only())

    def test_connect_as_write(self):
        self.reddit.open_reddit_write()
        self.assertFalse(self.reddit.is_read_only())

    def test_returns_generator(self):
        self.reddit.open_reddit_read()
        top_submissions = 5
        sut = self.reddit.top_submissions('u_hansknecht', top_submissions)
        count = 0
        for submission in sut:
            count += 1
            print(submission.title)
        self.assertEqual(count, top_submissions,
                         "The top ten submissions were not collected")
Exemple #18
0
 def __init__(self):
     self.data = Data()
     self.teams = Teams()
     self.scoreboard = Scoreboard()
     self.reddit = Reddit('nba')
#!/usr/bin/env python
# -*- coding: utf-8 -*-

from HackerNews import HackerNews
from Reddit import Reddit
from RSS import RSS
from Tweet import Tweet

# CONSTANTS
RETRIES = 10  # Number of retries on URL failure
WAIT_TIME = 20 * 60  # Time to wait between posting (in seconds)

tweet = Tweet()

if (tweet.check_last_tweet()):
    news = HackerNews(WAIT_TIME)
    news.tweet_stories(5)

    reddit = Reddit(WAIT_TIME)
    reddit.tweet_stories(3)

    rss = RSS(WAIT_TIME)
    rss.tweet_stories(2)
Exemple #20
0
class Gonewild(object):
    def __init__(self):
        # Single file that all output is written to, to track usage
        self.exit_if_already_started()
        self.db = DB()  # Database instance

        log_level = self.db.get_config('log_level', default='user')
        if log_level == 'none':
            self.root_log = open(devnull, 'w')
        else:
            self.root_log = open(
                path.join(ImageUtils.get_root(), 'history.log'), 'a')
        self.logger = self.root_log  # Logger used by helper classes

        self.reddit = Reddit()
        self.excluded_subs = self.db.get_excluded_subreddits()

    def debug(self, text):
        tstamp = strftime('[%Y-%m-%dT%H:%M:%SZ]', gmtime())
        text = '%s Gonewild: %s' % (tstamp, text)
        self.root_log.write('%s\n' % text)
        if self.logger != self.root_log:
            self.logger.write('%s\n' % text)
        stderr.write('%s\n' % text)

    def user_already_added(self, user):
        return self.db.user_already_added(user)

    def user_has_gone_wild(self, user):
        # Look at last 100 submissions
        try:
            children = self.reddit.get_user('%s/submitted' % user, max_pages=1)
        except Exception:
            # User is 404
            return False
        for child in children:
            if type(child) == Post:
                if 'gonewild'   in child.subreddit.lower() or \
                   'gw'         in child.subreddit.lower() or \
                   'asstastic'  in child.subreddit.lower() or \
                   'girlsgone'  in child.subreddit.lower() or \
                   'gone'       in child.subreddit.lower():
                    return True
        return False

    def add_excluded_subreddit(self, subreddit):
        return self.db.add_excluded_subreddit(subreddit)

    def setup_loggers_for_user(self, user):
        # Create directories if needed
        user_dir = path.join(ImageUtils.get_root(), 'content', user)
        ImageUtils.create_subdirectories(user_dir)
        # Setup logger
        log_level = self.db.get_config('log_level', default='user')
        if log_level == 'none': self.logger = open(devnull, 'w')
        elif log_level == 'user':
            self.logger = open(path.join(user_dir, 'history.log'), 'a')
        elif log_level == 'global':
            self.logger = self.root_log
        self.db.logger = self.logger
        ImageUtils.logger = self.logger
        self.reddit.logger = self.logger

    def restore_loggers(self):
        log_level = self.db.get_config('log_level', default='user')
        if log_level == 'user':
            self.logger.close()
            self.logger = self.root_log
            self.db.logger = self.logger
            ImageUtils.logger = self.logger
            self.reddit.logger = self.logger

    def is_excluded_child(self, child):
        if child.subreddit.lower() in [x.lower() for x in self.excluded_subs]:
            self.debug(
                '''%s: poll_user: Ignoring post/comment in excluded subreddit ("%s")
  Permalink: %s
    Ignored: %s''' %
                (child.author, child.subreddit, child.permalink(), str(child)))
            return True
        return False

    def get_and_process_urls_from_child(self, child):
        urls = self.get_urls(child)
        try:
            if type(child) == Post:
                self.db.add_post(child)
            elif type(child) == Comment:
                self.db.add_comment(child)
        except Exception, e:
            if 'already exists' not in str(e):
                self.debug('%s: poll_user: %s' % (child.author, str(e)))
            return  # If we can't add the post/comment to DB, skip it
        if len(urls) > 0:
            self.debug('%s: poll_user: found %d url(s) in child %s' %
                       (child.author, len(urls), child.permalink()))
            for url_index, url in enumerate(urls):
                self.process_url(url, url_index, child)
Exemple #21
0
	def gonewild_check(child, json, db, log):
		'''
			Check if image(s) in child were posted to gonewild
			and the child author is not the original gonewild author.
			Remove child and explain in comment.
			Args:
				child: reddit child (post)
				json: results from rarchives
			Returns:
				True if post originated on gonewild, False otherwise.
		'''
		# Disable this on /r/AmateurArchives
		if child.subreddit.lower() == 'amateurarchives': return False

		# XXX Only enforce posts by gonewild users (not comments)
		for post in json['posts']:
			if post['subreddit'].lower() == 'gonewild' and \
			   post['author'].lower() != child.author.lower() and \
				 post['author'] != '[deleted]':
				# Child author is not the gonewild user, image *is* though
				# Remove and comment the source
				child.remove(mark_as_spam=False)
				log('Rarchives.gonewild_check: Removed %s - matches /u/%s @ http://reddit.com%s' % (child.permalink(), post['author'], post['permalink']))
				title = post['title'].replace('[', '\\[').replace('(', '\\(').replace(']', '\\]').replace(')', '\\)').replace('*', '')
				body = '## This post was removed because it is a repost of a /r/gonewild submission:\n\n'
				body += '* **/u/%s** submitted [*%s*](%s) %s' % (post['author'], title, post['permalink'], Reddit.utc_timestamp_to_hr(post['created']))
				try:
					response = child.reply(body)
					response.distinguish()
					child.flair('Gonewild repost: /u/%s' % post['author'])
				except Exception, e:
					log('Rarchives.gonewild_check: Error while replying to %s : %s' % (child.permalink(), str(e)))
				# Update 'log_amarch' db table
				db.insert('log_amarch', ('removed', child.permalink(), timegm(gmtime()), 'gonewild repost of /u/%s: %s' % (post['author'], post['permalink'])))
				return True
Exemple #22
0
    SK = Skyrock(json['login'], json['pass'], json['text'], json['link'],
                 json['datetime'], json['page_post'], json['groupe_post'],
                 json['title'], json['image_link'], json['hashtags'],
                 json['gropes_links'], json['priority'])
    SK.post()
    print(SK.get_bugrep())
if json['id'] == 'wh':
    WH = WeHear(json['login'], json['pass'], json['text'], json['link'],
                json['datetime'], json['page_post'], json['groupe_post'],
                json['title'], json['image_link'], json['hashtags'],
                json['gropes_links'], json['priority'])
    WH.post()
    print(WH.get_bugrep())
if json['id'] == 'rd':
    RD = Reddit(json['login'], json['pass'], json['text'], json['link'],
                json['datetime'], json['page_post'], json['groupe_post'],
                json['title'], json['image_link'], json['hashtags'],
                json['gropes_links'], json['priority'])
    RD.post()
    print(RD.get_bugrep())
if json['id'] == 'li':
    LI = LinkedIn(json['login'], json['pass'], json['text'], json['link'],
                  json['datetime'], json['page_post'], json['groupe_post'],
                  json['title'], json['image_link'], json['hashtags'],
                  json['gropes_links'], json["priority"])
    LI.post()
    print(LI.get_bugrep())
if json['id'] == 'tr':
    TR = Tumblr(json['login'], json['pass'], json['text'], json['link'],
                json['datetime'], json['page_post'], json['groupe_post'],
                json['title'], json['image_link'], json['hashtags'],
                json['gropes_links'])
Exemple #23
0
                <meta charset="UTF-8">
                <meta name="viewport" content="width=device-width, initial-scale=1.0">
                <title>Document</title>
            </head>
        <body>
            <div class="container mb-5">
            <h1>Your feed for {today.strftime('%d %b %Y')}</h1>
    """)

w = Weather()
w.write_weather()

e = Events()
e.write_events()

r = Reddit()

with open(feed_path, "a") as file:
    file.write("<h2>Reddit</h2>")

r.fetch_subs_and_write("reactjs")
r.fetch_subs_and_write("webdev")
r.fetch_subs_and_write("frontend")
r.fetch_subs_and_write("python")
r.fetch_subs_and_write("rollerblading")

with open(feed_path, "a") as file:
    file.write("<h2>CNA</h2>")

cna = CNA()
Exemple #24
0
def lambda_handler(event, context):
    red = Reddit()
    return red.scan()
Exemple #25
0
asstastic
'''

SUBS = [x.strip() for x in SUBS_TEXT.strip().split('\n')]
while '' in SUBS: SUBS.remove('')

db = DB()

last_post = db.get_config('last_post')

reddit_url = 'http://www.reddit.com/r/%s/new.json' % '+'.join(SUBS)
print 'firehose from %s' % reddit_url
while True:
	sleep(2)
	try:
		posts = Reddit.get(reddit_url)
	except Exception, e:
		#print 'error when querying %s: %s' % (reddit_url, str(e))
		continue
	for post in posts:
		if last_post != None and post.id == last_post:
			break
		if post.selftext != None:
			# TODO self-text, skip it
			continue
		url = post.url
		shorturl  = 'http://redd.it/%s' % post.id
		subreddit = post.subreddit
		author    = post.author
		title     = post.title
		print ','.join( [shorturl, url, subreddit, author, title] )
Exemple #26
0
from Reddit import Reddit

if __name__ == '__main__':

    f = open("dictionaries/insultos.txt", "r")

    paraulesclau = f.readlines()

    f.close()

    f = open("dictionaries/topics.txt", "r")

    paraulesclau += f.readlines()

    f.close()

    index = 0
    for p in paraulesclau:
        paraulesclau[index] = p[:-1].lower()
        index += 1

    # print(paraulesclau)

    #inicialitzem reddit
    Reddit('comments', paraulesclau)
Exemple #27
0
class Gonewild(object):

	def __init__(self):
		# Single file that all output is written to, to track usage
		self.exit_if_already_started()
		self.db = DB() # Database instance

		log_level = self.db.get_config('log_level', default='user')
		if log_level == 'none':
			self.root_log = open(devnull, 'w')
		else:
			self.root_log = open(path.join(ImageUtils.get_root(), 'history.log'), 'a')
		self.logger   = self.root_log # Logger used by helper classes

		self.reddit   = Reddit()
		self.excluded_subs = self.db.get_excluded_subreddits()
		
	def debug(self, text):
		tstamp = strftime('[%Y-%m-%dT%H:%M:%SZ]', gmtime())
		text = '%s Gonewild: %s' % (tstamp, text)
		self.root_log.write('%s\n' % text)
		if self.logger != self.root_log:
			self.logger.write('%s\n' % text)
		stderr.write('%s\n' % text)

	def user_already_added(self, user):
		return self.db.user_already_added(user)

	def user_has_gone_wild(self, user):
		# Look at last 100 submissions
		try:
			children = self.reddit.get_user('%s/submitted' % user, max_pages=1)
		except Exception:
			# User is 404
			return False
		for child in children:
			if type(child) == Post:
				if 'gonewild'   in child.subreddit.lower() or \
				   'gw'         in child.subreddit.lower() or \
				   'asstastic'  in child.subreddit.lower() or \
				   'girlsgone'  in child.subreddit.lower() or \
				   'gone'       in child.subreddit.lower():
					return True
		return False

	def add_excluded_subreddit(self, subreddit):
		return self.db.add_excluded_subreddit(subreddit)

	def setup_loggers_for_user(self, user):
		# Create directories if needed
		user_dir = path.join(ImageUtils.get_root(), 'content', user)
		ImageUtils.create_subdirectories(user_dir)
		# Setup logger
		log_level = self.db.get_config('log_level', default='user')
		if   log_level == 'none':   self.logger = open(devnull, 'w')
		elif log_level == 'user':   self.logger = open(path.join(user_dir, 'history.log'), 'a')
		elif log_level == 'global': self.logger = self.root_log
		self.db.logger     = self.logger
		ImageUtils.logger  = self.logger
		self.reddit.logger = self.logger

	def restore_loggers(self):
		log_level = self.db.get_config('log_level', default='user')
		if log_level == 'user':
			self.logger.close()
			self.logger = self.root_log
			self.db.logger     = self.logger
			ImageUtils.logger  = self.logger
			self.reddit.logger = self.logger

	def is_excluded_child(self, child):
		if child.subreddit.lower() in [x.lower() for x in self.excluded_subs]:
			self.debug('''%s: poll_user: Ignoring post/comment in excluded subreddit ("%s")
  Permalink: %s
    Ignored: %s''' % (child.author, child.subreddit, child.permalink(), str(child)))
			return True
		return False

	def get_and_process_urls_from_child(self, child):
		urls = self.get_urls(child)
		try:
			if type(child) == Post:
				self.db.add_post(child)
			elif type(child) == Comment:
				self.db.add_comment(child)
		except Exception, e:
			if 'already exists' not in str(e):
				self.debug('%s: poll_user: %s' % (child.author, str(e)))
			return # If we can't add the post/comment to DB, skip it
		if len(urls) > 0:
			self.debug('%s: poll_user: found %d url(s) in child %s' % (child.author, len(urls), child.permalink()))
			for url_index, url in enumerate(urls):
				self.process_url(url, url_index, child)
 def __init__(self, username, password, clientId, clientSecret):
     self.reddit = Reddit(username, password, clientId, clientSecret)
Exemple #29
0
'''
Created on Oct 14, 2017

@author: Tyler-OC
'''
from Discord_API import Discord
from Database import Database
from Reddit import Reddit
import time

discord = Discord()
Data = Database()
Red = Reddit()


count = 0
while True:
    Page = Red.getPage("https://www.reddit.com/r/buildapcsales/")
    Products = Red.getProducts(Page)
    
    
    
    for product in Products:
        if "[monitor]" in product.lower():
            if Data.CheckProduct(product):
                print("Product in database!")
            else:
                Data.newProduct(product, Products[product])
                Message = "There is a new Product to Check out! \n" + product + "\n" + Products[product]
                discord.sendMessage(Message)
    
from SQLHandler import SQLHandler
from Reddit import Reddit
from Analyzer import Analyzer
import time

def thenOrThan(words):
    return "then" if "then" in words else "than"

count = 0

while True:
    sql = SQLHandler()
    reddit = Reddit()
    analyzer = Analyzer()
    
    count += 1
    
    print "Beginning Iteration " + str(count)
    print "Gathering Comments"
    
    comments = reddit.getNewComments()
    views = sql.getViews()
    
    print "Processing Data"
    for comment in comments:
        flag = True
        words = [x.lower().strip() for x in comment[0].split()]
        thenorthan = thenOrThan(words)
        
        for i in views:
            if comment[1] == i[0]:
Exemple #31
0
	print '[!] inserting "blank" admin'
total_scores = db.select_one('sum(score)', 'admins', 'username != ""')
db.update('admins', 'score = ?', 'username = ""', [total_removed - total_scores])
print '[!] updated "blank" admin with score: %d' % total_scores
db.commit()


#######################

# Login to reddit
print '[ ] logging in to reddit'
username = db.get_config('reddit_user')
password = db.get_config('reddit_pw')
if username == None or password == None:
	print '[!] "reddit_user" or "reddit_pw" were not found in "config" table in database'
Reddit.login(username, password)

print '[ ] executing AmArch.py functionality...'
def log(txt): print txt
AmArch.execute(db, log)


# MODERATED SUBS (real-time)
print '[ ] finding modded subs...'
current = Reddit.get_modded_subreddits()
print '[ ] found %d subs modded by /u/%s' % (len(current), username)

# remove subs that should be ignored
to_ignore = db.get_config('ignore_subreddits')
if to_ignore != None:
	for ignore in to_ignore.split(','):
Exemple #32
0
#!/usr/bin/python
"""
	"Backfill" existing database data.
	Gets titles, permalinks, dates, etc from reddit.
	Overwrites existing 'bad' data with accurate data.
"""

from DB import DB
from Reddit import Reddit
from sys import exit, stdout
from urllib2 import HTTPError
from os import remove as osremove
from ImageUtils import ImageUtils

db = DB()
reddit = Reddit()


def backfill_users():
    q_users = '''
		select id,username
			from users
			where deleted = 0
	'''
    cur = db.conn.cursor()
    execur = cur.execute(q_users)
    ids_and_users = execur.fetchall()  # Get list of users + ids
    index = 0
    for (userid, username) in ids_and_users:
        index += 1
        print '(%d/%d) updating %s...' % (index, len(ids_and_users), username),
Exemple #33
0
        pass

    elif (sys.argv[flag] == "--image" or sys.argv[flag] == "-i"):
        pass

    elif (sys.argv[flag] == "--nsfw" or sys.argv[flag] == "-NSFW"):
        pass

    elif (sys.argv[flag] == "--" or sys.argv[flag][0] != "-"):
        break

    else:
        print("invalid flag", sys.argv[flag])
        usage()

r = Reddit(User)

bot = 'bot1'
d = Display(praw.Reddit(bot, user_agent=bot))

if (argc > shift + 1
        and (sys.argv[shift + 1] == "list" or sys.argv[shift + 1] == "l")):
    pass

elif (argc > shift + 1
      and (sys.argv[shift + 1] == "print" or sys.argv[shift + 1] == "p")):
    pass

elif (argc > shift + 1
      and (sys.argv[shift + 1] == "collect" or sys.argv[shift + 1] == "c")):
    payload = r.updown(Sub)
Exemple #34
0
def main():
    # Just a quick note. I prefer camelCase over underscores so that is what I used for the variable names I changed.
    authFile = "auth.txt"

    creds = import_credentials(authFile)
    if creds is False or len(creds) < 3:
        print("The OAuth file is not valid, or it does not exist.")
        print("Please provide valid OAuth credentials in ",
              authFile,
              ".",
              sep="")
        sys.exit(
        )  # sys.exit() is I believe the more pythonic way of doing this.
        # sys.exit() is meant more for programs where as exit(0) is meant for the interactive console. I think.
    else:
        reddit = Reddit(*creds)
        # By creating a Reddit object we don't need to worry about passing credentials when we want to use the,

    inital_user = input("Enter username (enter 'q' to quit): ")
    if inital_user == "q":  # Why don't you just do this?
        sys.exit()

    print("Getting original user's top 5 subreddits...")
    # Original user's top 5 subreddits
    top_subreddits = reddit.ranked_subreddits(
        reddit.full_comment_and_submission_history(inital_user), max_count=10)

    user_scores = {}
    # Set this to the maximum number of latest users you want from each subreddit
    MAX_USERS_PER_SUBREDDIT = 100
    # Last 100 users in each of the top 5 subreddits
    for sub_count, sub_reddit in enumerate(
            top_subreddits
    ):  # enumerate gets you a iterable and an index variable
        # In this case sub_count was just the index+1
        print("Getting last 100 users in " + sub_reddit[0] + " (" +
              str(sub_count + 1) + " of " + str(len(top_subreddits)) + ")...")
        users_list = reddit.last_subreddit_users(
            sub_reddit[0], max_count=MAX_USERS_PER_SUBREDDIT)

        # Top 5 subreddits for each of the users in the current subreddit
        user_count = 1
        for user in users_list:
            print("Getting top subreddits for user " + str(user_count) +
                  " of " + str(len(users_list)) + "...",
                  end="\r",
                  flush=True)
            user_top_subreddits = reddit.ranked_subreddits(
                reddit.full_comment_and_submission_history(user), max_count=5)
            score = 1
            user_count += 1
            for comp_sub in user_top_subreddits:
                if comp_sub in top_subreddits:
                    score += 1
            if score > 1:
                user_scores.update({user: score})

        print("\nMatched users so far: " + str(user_scores))

    matched_users = sorted(user_scores.items(),
                           key=operator.itemgetter(1),
                           reverse=True)

    print("\nUsers most like " + inital_user + ":")
    for match in matched_users:
        print("User: "******" | Score: " + str(match[1]))
Exemple #35
0
"""
	"Backfill" existing database data.
	Gets titles, permalinks, dates, etc from reddit.
	Overwrites existing 'bad' data with accurate data.
"""

from DB      import DB
from Reddit  import Reddit
from sys     import exit, stdout
from urllib2 import HTTPError
from os      import remove as osremove
from ImageUtils import ImageUtils

db = DB()
reddit = Reddit()

def backfill_users():
	q_users = '''
		select id,username
			from users
			where deleted = 0
	'''
	cur = db.conn.cursor()
	execur = cur.execute(q_users)
	ids_and_users = execur.fetchall() # Get list of users + ids
	index = 0
	for (userid, username) in ids_and_users:
		index += 1
		print '(%d/%d) updating %s...' % (index, len(ids_and_users), username),
		stdout.flush()
Exemple #36
0
from SQLHandler import SQLHandler
from Reddit import Reddit
from Analyzer import Analyzer
import time


def thenOrThan(words):
    return "then" if "then" in words else "than"


count = 0

while True:
    sql = SQLHandler()
    reddit = Reddit()
    analyzer = Analyzer()

    count += 1

    print "Beginning Iteration " + str(count)
    print "Gathering Comments"

    comments = reddit.getNewComments()
    views = sql.getViews()

    print "Processing Data"
    for comment in comments:
        flag = True
        words = [x.lower().strip() for x in comment[0].split()]
        thenorthan = thenOrThan(words)
 def setUp(self):
     self.reddit = Reddit()
     return super().setUp()