def main():
reddit = ReddiWrap(user_agent='made_this_up_quick')
reddit.login('brainsareneat', 'isminus1')
outfile = "graph.pkl"
g = Graph(reddit, outfile)
while True:
g.update()
def main():
reddit = ReddiWrap(user_agent='made_this_up_quick')
reddit.login('brainsareneat', 'isminus1')
outfile = "graph.pkl"
g = Graph(reddit, outfile)
while True:
g.update()
def main():
#get startup info
while True:
try:
Guser = raw_input("Gmail username :"******"Gmail password :"******"connecting to smtp.gmail.com")
myCellEmail = raw_input("SMS email address?")
mail = smtplib.SMTP('smtp.gmail.com', 587)
mail.ehlo()
mail.starttls()
mail.ehlo()
mail.login(Guser,Gpass)
mail.quit()
break
except smtplib.SMTPConnectError:
print("cannot connect to smtp.gmail.com:587")
except smtplib.SMTPAuthenticationError:
print("cannot authenticate. doublecheck the Username/password")
subreddit = raw_input("monitor reddit.com/r/")
#Logging posts on the subreddit
recordedPosts=[]
reddit = ReddiWrap()
startPosts = reddit.get("/r/%s/new" % subreddit )
for post in startPosts:
recordedPosts.append(post.id)
#the main check loop, refreshes once per second
while(True):
newPosts=reddit.get("/r/%s" % subreddit+"/new/?count=0")
mailToSend = []
for post in newPosts:
if (not post.id in recordedPosts):
mailToSend.append(post)
recordedPosts.append(post.id)
if (len(mailToSend)>0):
mail = smtplib.SMTP('smtp.gmail.com', 587)
mail.ehlo()
mail.starttls()
mail.ehlo()
mail.login(Guser,Gpass)
for newpost in mailToSend:
seconds = (datetime.datetime.now()-datetime.datetime.fromtimestamp(newpost.created-7*60*60)).seconds
print("(Posted %s:%s minutes ago): \"%s\"" %( seconds/60, (seconds%60/10==0)*"0"+str(seconds%60) , newpost.title) )
mail.sendmail("RedditMonitor indev", [myCellEmail], newpost.title+"\r\n"+newpost.selftext*post.is_self+newpost.url*(not newpost.is_self))
mail.quit()
sleep(1)
import time
import urllib2
tobaccoName = ""
class MyHTMLParser(HTMLParser):
def handle_starttag(self, tag, attrs):
global tobaccoName
if (tag == "meta"):
if 'itemprop' in dict(attrs):
if dict(attrs)['itemprop'] == "itemreviewed":
tobaccoName = dict(attrs)['content']
reddit = ReddiWrap(user_agent='ReddiWrap')
USERNAME = '******'
PASSWORD = '******'
MOD_SUB = 'PipeTobacco' # A subreddit moderated by USERNAME
# Load cookies from local file and verify cookies are valid
reddit.load_cookies('cookies.txt')
# If we had no cookies, or cookies were invalid,
# or the user we are logging into wasn't in the cookie file:
if not reddit.logged_in or reddit.user.lower() != USERNAME.lower():
print('logging into %s' % USERNAME)
login = reddit.login(user=USERNAME, password=PASSWORD)
if login != 0:
# 1 means invalid password, 2 means rate limited, -1 means unexpected error
# To print titles of top 10 in r/aww
import urllib
from ReddiWrap import ReddiWrap
from datetime import datetime
def getUTCTime(timestamp_string):
return datetime.utcfromtimestamp(timestamp_string)
def getTime(timestamp_string):
return datetime.fromtimestamp(timestamp_string)
# Create new ReddiWrap instance
reddit = ReddiWrap()
aww = reddit.get('/r/aww')
strFormat = "%Y-%m-%d"
topten = aww[:10]
print "Title", "|", "Time", "|", "Votes + -"
for p in topten:
#image = urllib.urlopen("http://site.com/image.jpg").read()
#print p.url, p.title
print "=", p.title, "="
print "Votes | +:", p.upvotes, "-:", p.downvotes, "fuzzy:", p.score
print "Time[Local]:", getTime(p.created) ,"Time[UTC]:", getUTCTime(p.created_utc)
print
"""
Nice T**s Collector [NSFW]
Crawler used to collect all pics reddit user 'Only_Says_Nice_Tits' commented on.
Developed on Python 2.7.3
External Libraries:
ReddiWrap: Reddit.com API wrapper
Web: Web requests utility library, used by ReddiWrap
(c) Antonio Herraiz August/2013
"""
from ReddiWrap import ReddiWrap
reddit = ReddiWrap(user_agent='ReddiWrap')
import time # For sleep(), to avoid API rate limit
count = 0
pics = []
comments = reddit.get_user_comments('Only_Says_Nice_Tits')
if comments != None:
while True:
for comment in comments:
# reddit.last_url will be like: http://reddit.com/r/funny/comments/1jkgf3/cbfmmzu.json
post = reddit.get('/r/%s/comments/%s/%s' % (comment.subreddit, comment.link_id[3:], comment.id))
url = post[0].url
if 'imgur' in url and 'i.imgur' not in url:
# TODO: http://imgur.com/ipv9GiY ==> http://i.imgur.com/ipv9GiY.xxx
print('Transforming imgur URL')
pics.append(url)
print('Pic URL: %s' % (url))
cursor.execute("INSERT INTO `Posts` (`Title`,`UserID`,`SubredditID`) VALUES (%s,%s,%s)",(Title,UserID,SubredditID))
def searchForPostIDInDatabase(post,cursor):
Title = post.title.encode('ascii', 'ignore')
cursor.execute("SELECT PostID FROM Posts WHERE Title = (%s) LIMIT 1",(Title))
data = cursor.fetchall()
#print data
while data:
postID = [int(i[0]) for i in data]
return int(postID[0])
else:
return False
#Copy 'n paste from reddiwrap examples...
reddit = ReddiWrap(user_agent='ReddiWrap')
USERNAME = ''
PASSWORD = ''
MOD_SUB = '' # A subreddit moderated by USERNAME
# Load cookies from local file and verify cookies are valid
reddit.load_cookies('cookies.txt')
# If we had no cookies, or cookies were invalid,
# or the user we are logging into wasn't in the cookie file:
if not reddit.logged_in or reddit.user.lower() != USERNAME.lower():
print('logging into %s' % USERNAME)
login = reddit.login(user=USERNAME, password=PASSWORD)
if login != 0:
# 1 means invalid password, 2 means rate limited, -1 means unexpected error
import time
import urllib2
tobaccoName = ""
class MyHTMLParser(HTMLParser):
def handle_starttag(self, tag, attrs):
global tobaccoName
if(tag == "meta"):
if 'itemprop' in dict(attrs):
if dict(attrs)['itemprop'] == "itemreviewed":
tobaccoName = dict(attrs)['content']
reddit = ReddiWrap(user_agent='ReddiWrap')
USERNAME = '******'
PASSWORD = '******'
MOD_SUB = 'PipeTobacco' # A subreddit moderated by USERNAME
# Load cookies from local file and verify cookies are valid
reddit.load_cookies('cookies.txt')
# If we had no cookies, or cookies were invalid,
# or the user we are logging into wasn't in the cookie file:
if not reddit.logged_in or reddit.user.lower() != USERNAME.lower():
print('logging into %s' % USERNAME)
login = reddit.login(user=USERNAME, password=PASSWORD)
if login != 0:
# 1 means invalid password, 2 means rate limited, -1 means unexpected error
# To print titles of top 10 in r/aww
import urllib
from ReddiWrap import ReddiWrap
# Create new ReddiWrap instance
reddit = ReddiWrap()
front = reddit.get('/r/all')
topten = front[:10]
for p in topten:
print p.title
import time
import datetime
from ReddiWrap import ReddiWrap
import numpy as np
reddit = ReddiWrap() # Create new instance of ReddiWrap
login_result = reddit.login('brainsareneat', 'isminus1')
all_posts = []
new = reddit.get('/r/all/new')
while True:
time.sleep(1.5)
for post in new:
all_posts.append(post)
posts= np.array(all_posts)
np.savez('records.npz', posts = posts)
new = reddit.get_next()
print 'ok...'
#!/usr/bin/python
from ReddiWrap import ReddiWrap
import myLib
import time
import sys
reddit = ReddiWrap(user_agent='ReddiWrap')
USERNAME = '******'
PASSWORD = '******'
SUBREDDIT_NAMES = ['POLITICS', 'FUNNY', 'PICS', 'todayilearned']
while True:
for MOD_SUB in SUBREDDIT_NAMES:
print "######### " + MOD_SUB + " ###########"
# Load cookies from local file and verify cookies are valid
reddit.load_cookies('cookies.txt')
# If we had no cookies, or cookies were invalid,
# or the user we are logging into wasn't in the cookie file:
if not reddit.logged_in or reddit.user.lower() != USERNAME.lower():
print('logging into %s' % USERNAME)
login = reddit.login(user=USERNAME, password=PASSWORD)
if login != 0:
# 1 means invalid password, 2 means rate limited, -1 means unexpected error
print('unable to log in: %d' % login)
print('remember to change USERNAME and PASSWORD')
exit(1)
# Save cookies so we won't have to log in again later
reddit.save_cookies('cookies.txt')
#!/usr/bin/env python
from ReddiWrap import ReddiWrap
reddit = ReddiWrap(user_agent='ReddiWrap')
USERNAME = ''
PASSWORD = ''
reddit.load_cookies("cookies.txt")
if not reddit.logged_in or reddit.user.lower() != USERNAME.lower():
login = reddit.login(user=USERNAME, password=PASSWORD)
if login != 0:
# 1 means invalid password, 2 means rate limited, -1 means unexpected error
print('unable to log in: %d' % login)
exit(1)
# Save cookies so we won't have to log in again later
reddit.save_cookies('cookies.txt')
info = reddit.user_info()
output = "L:%(link)d C:%(comment)d " % {"link": info.link_karma, "comment": info.comment_karma}
if info.has_mail:
output = output + u"\u2709"
print(output)
import time
import datetime
from ReddiWrap import ReddiWrap
import numpy as np
reddit = ReddiWrap() # Create new instance of ReddiWrap
login_result = reddit.login('ahoytestaccount', 'isminus1')
try:
postinfos = [item in np.load('text.npy')]
indexed = [item in np.load('indexed.npy')]
except:
indexed = []
postinfos = []
while True:
loaded = False
while not loaded:
#try:
postholder = np.load('records.npz')
posts = postholder['posts']
loaded = True
#except:
time.sleep(3)
print "failed"
first_time = posts[0].created
for post in [posts[-1 -num] for num in range(len(posts))]:
print (first_time - post.created) / (60 * 60 * 24)
if first_time - post.created > 60 * 60 * 24:
if post.id not in indexed:
time.sleep(2)
#!/usr/bin/python
from ReddiWrap import ReddiWrap
reddit = ReddiWrap(user_agent='ReddiWrap')
USERNAME = '******'
PASSWORD = '******'
MOD_SUB = 'scrapers' # A subreddit moderated by USERNAME
# Load cookies from local file and verify cookies are valid
reddit.load_cookies('cookies.txt')
# If we had no cookies, or cookies were invalid,
# or the user we are logging into wasn't in the cookie file:
if not reddit.logged_in or reddit.user.lower() != USERNAME.lower():
print('logging into %s' % USERNAME)
login = reddit.login(user=USERNAME, password=PASSWORD)
if login != 0:
# 1 means invalid password, 2 means rate limited, -1 means unexpected error
print('unable to log in: %d' % login)
print('remember to change USERNAME and PASSWORD')
exit(1)
# Save cookies so we won't have to log in again later
reddit.save_cookies('cookies.txt')
print('logged in as %s' % reddit.user)
uinfo = reddit.user_info()
print('\nlink karma: %d' % uinfo.link_karma)
print('comment karma: %d' % uinfo.comment_karma)
#!/usr/bin/python
from ReddiWrap import ReddiWrap
reddit = ReddiWrap(user_agent='ReddiWrap')
USERNAME = '******'
PASSWORD = '******'
MOD_SUB = 'POLITICS' # A subreddit moderated by USERNAME
# Load cookies from local file and verify cookies are valid
reddit.load_cookies('cookies.txt')
# If we had no cookies, or cookies were invalid,
# or the user we are logging into wasn't in the cookie file:
if not reddit.logged_in or reddit.user.lower() != USERNAME.lower():
print('logging into %s' % USERNAME)
login = reddit.login(user=USERNAME, password=PASSWORD)
if login != 0:
# 1 means invalid password, 2 means rate limited, -1 means unexpected error
print('unable to log in: %d' % login)
print('remember to change USERNAME and PASSWORD')
exit(1)
# Save cookies so we won't have to log in again later
reddit.save_cookies('cookies.txt')
print('logged in as %s' % reddit.user)
uinfo = reddit.user_info()
print('\nlink karma: %d' % uinfo.link_karma)
print('comment karma: %d' % uinfo.comment_karma)
import time
import datetime
from ReddiWrap import ReddiWrap
import numpy as np
reddit = ReddiWrap() # Create new instance of ReddiWrap
login_result = reddit.login('brainsareneat', 'isminus1')
if login_result != 0: # Anything other than '0' means an error occurred
print 'unable to log in.'
exit(1)
olddict = {}
newdict = {}
clock = datetime.datetime
a = clock.now()
f = open("%s_%s_%srecord.csv" % (a.month, a.day, a.hour), "a")
f.write("id, author, subreddit, title, created, link, url")
b = clock.now()
while b - a < datetime.timedelta(hours=6):
print b - a
time.sleep(1)
new = reddit.get('/r/all/new')
seen = False
for post in new:
entry = {"subreddit":post.subreddit.encode('ascii', 'ignore'), "link":post.permalink.encode('ascii', 'ignore'), "title":post.title.encode('ascii', 'ignore'),\
"created":post.created, "id":post.id, "url":post.url.encode('ascii', 'ignore'), "author":post.author.encode('ascii', 'ignore')}
#! /usr/bin/env python
from datetime import timedelta, date
import sys, os, json, time, datetime
sys.path.append(os.path.abspath('reddiwrap'))
from ReddiWrap import ReddiWrap
import xml.etree.ElementTree as ET
def get_nested_comments(comment):
comment_str = ""
for ch_com in comment.children:
comment_str += " " + comment.body + get_nested_comments(ch_com)
return comment_str
reddit = ReddiWrap()
USERNAME = '******'
PASSWORD = '******'
from local_settings import *
for a in range(1, len(sys.argv)):
SUB = sys.argv[a]
#Log in
reddit.load_cookies('cookies.txt')
if not reddit.logged_in or reddit.user.lower() != USERNAME.lower():
print('logging into %s' % USERNAME)
login = reddit.login(user=USERNAME, password=PASSWORD)
import re
import time
from django.utils.encoding import smart_str
from datetime import datetime
from pyh import *
from ReddiWrap import ReddiWrap
reddit = ReddiWrap()
# regular expressions to match up soundcloud and youtube
youtubeMatch = re.compile('^(http[s]?):\/\/(?:www\.)?youtube\.com\/watch\?(?=.*v=\w+)(?:\S+)?$')
soundcloudMatch = re.compile('^(http[s]?):\/\/(?:www\.)?soundcloud\.com\/')
eightTracksMatch = re.compile('^(http[s]?):\/\/(www\.)?8tracks\.com\/')
jpopSuki = re.compile('^(http[s]?):\/\/(www\.)?jpopsuki\.tv\/video\/')
# list of subreddits without the /r/, can be changed/removed at will
subreddits = ('chillstep', 'dnb', 'dubstep', 'electronicmusic', 'liquiddnb',
'progtrance', 'realdubstep', 'swinghouse', 'Techno', 'trance',
'tranceandbass', 'uplifting', 'jpop', 'kpop')
now = datetime.now()
#####################################
# setup phy page
page = PyH("Music - " + now.strftime("%m.%d.%Y"))
page.addCSS("css/bootstrap.css")
page.addCSS("css/docs.css")
page.addJS("js/bootstrap.js")
#TOP NAVBAR
navbar = page << div(cl="navbar navbar-inverse navbar-fixed-top")
navbarInner = navbar << div(cl="navbar-inner")
#!/usr/bin/python
from ReddiWrap import ReddiWrap
reddit = ReddiWrap(user_agent='ReddiWrap')
USERNAME = '******'
PASSWORD = '******'
MOD_SUB = 'scrapers' # A subreddit moderated by USERNAME
# Load cookies from local file and verify cookies are valid
reddit.load_cookies('cookies.txt')
# If we had no cookies, or cookies were invalid,
# or the user we are logging into wasn't in the cookie file:
if not reddit.logged_in or reddit.user.lower() != USERNAME.lower():
print('logging into %s' % USERNAME)
login = reddit.login(user=USERNAME, password=PASSWORD)
if login != 0:
# 1 means invalid password, 2 means rate limited, -1 means unexpected error
print('unable to log in: %d' % login)
print('remember to change USERNAME and PASSWORD')
exit(1)
# Save cookies so we won't have to log in again later
reddit.save_cookies('cookies.txt')
print('logged in as %s' % reddit.user)
uinfo = reddit.user_info()
print('\nlink karma: %d' % uinfo.link_karma)
print('comment karma: %d' % uinfo.comment_karma)
