def main():
token = readability.xauth(
CONSUMER_KEY, CONSUMER_SECRET, USERNAME, PASSWORD)
rdd = readability.oauth(
CONSUMER_KEY, CONSUMER_SECRET, token=token)
user = rdd.get_me()
logger.info("Updating readability library")
library_urls = [u.article.url for u in user.bookmarks()]
logger.info("Found %d articles in library", len(library_urls))
# Fetch URLs
urls = get_article_urls_from_twitter_favourites(TWITTER_USERNAME)
urls += get_top_hacker_news_articles(5)
urls += get_economist_articles(5)
urls += get_atlantic_articles(2) # Only 3 as it's too noisy
logger.info("Found %d articles to add", len(urls))
num_dupes = num_new = num_errors = 0
for url in urls:
if url in library_urls:
num_dupes += 1
else:
logger.info("Adding %s", url)
try:
rdd.add_bookmark(url)
except ResponseError:
num_errors += 1
except Exception, e:
logger.error("Unexpected exception: %s", e)
num_errors += 1
else:
num_new += 1
def main():
token = readability.xauth(CONSUMER_KEY, CONSUMER_SECRET, USERNAME,
PASSWORD)
rdd = readability.oauth(CONSUMER_KEY, CONSUMER_SECRET, token=token)
user = rdd.get_me()
logger.info("Updating readability library")
library_urls = [u.article.url for u in user.bookmarks()]
logger.info("Found %d articles in library", len(library_urls))
# Fetch URLs
urls = get_article_urls_from_twitter_favourites(TWITTER_USERNAME)
urls += get_top_hacker_news_articles(5)
urls += get_economist_articles(5)
urls += get_atlantic_articles(2) # Only 3 as it's too noisy
logger.info("Found %d articles to add", len(urls))
num_dupes = num_new = num_errors = 0
for url in urls:
if url in library_urls:
num_dupes += 1
else:
logger.info("Adding %s", url)
try:
rdd.add_bookmark(url)
except ResponseError:
num_errors += 1
except Exception, e:
logger.error("Unexpected exception: %s", e)
num_errors += 1
else:
num_new += 1
def run(self, token, user_id):
rdd = readability.oauth(app.config['READABILITY_CONSUMER_KEY'],
app.config['READABILITY_CONSUMER_SECRET'], token=token)
for bookmark in rdd.get_bookmarks():
url = bookmark.article.url
worker.run('retrieve_page', url, user_id=user_id, source='readability', thematic="readability") # FIXME: thematic sould be specified
def setup_rdd():
"""Boostraps Readability instance from environment."""
try:
c_key, c_secret = get_consumer_keys()
except ValueError:
print >> sys.stderr, 'READABILITY_ACCESS_TOKEN and READABILITY_ACCESS_SECRET must be set.'
sys.exit(1)
token = get_access_token()
return readability.oauth(c_key, c_secret, token=token)
def home():
insta_api = InstagramAPI(access_token=local_settings.INSTAGRAM_TOKEN)
feed = insta_api.user_recent_media(count=15)
medias = feed[0]
rdd_api = readability.oauth(local_settings.READABILITY_CLIENT_KEY, local_settings.READABILITY_CLIENT_SECRET, token=local_settings.READABILITY_USER_TOKEN)
bookmarks = rdd_api.get_bookmarks(limit=10)
# ['author', 'content', 'content_size', 'date_published', 'domain', 'excerpt', 'id', 'next_page_href', 'processed', 'short_url', 'title', 'url', 'word_count']
return render_template("index.html", medias=medias, bookmarks=bookmarks)
def home():
insta_api = InstagramAPI(access_token=local_settings.INSTAGRAM_TOKEN)
feed = insta_api.user_recent_media(count=15)
medias = feed[0]
rdd_api = readability.oauth(local_settings.READABILITY_CLIENT_KEY,
local_settings.READABILITY_CLIENT_SECRET,
token=local_settings.READABILITY_USER_TOKEN)
bookmarks = rdd_api.get_bookmarks(limit=10)
# ['author', 'content', 'content_size', 'date_published', 'domain', 'excerpt', 'id', 'next_page_href', 'processed', 'short_url', 'title', 'url', 'word_count']
return render_template("index.html", medias=medias, bookmarks=bookmarks)
def get_readability_api(api_key=None, api_secret=None, username=None, password=None):
""" Authorise in Readability API"""
if not api_key:
api_key = os.environ.get('READABILITY_API_KEY', '')
if not api_secret:
api_secret = os.environ.get('READABILITY_API_SECRET', '')
if not username:
username = os.environ.get('READABILITY_USERNAME', '')
if not password:
password = os.environ.get('READABILITY_PASSWORD', '')
token = readability.xauth(api_key, api_secret, username, password)
rdd = readability.oauth(api_key, api_secret, token=token)
return rdd
def rdd_delete_bookmarks():
"""
清空 readability
"""
import readability
token = readability.xauth(configs.RDD_APIKEY, configs.RDD_SECRET, configs.RDD_USERNAME, configs.RDD_PASSWORD)
rdd = readability.oauth(configs.RDD_APIKEY, configs.RDD_SECRET, token=token)
list = rdd.get_bookmarks()
count = 0
for bm in list:
resource = 'bookmarks/%s' % bm.id
rdd._delete_resource(resource)
count += 1
print count
print 'delete count: ', count
def _import(list):
"""
导入 readablility
"""
import readability
token = readability.xauth(configs.RDD_APIKEY, configs.RDD_SECRET, configs.RDD_USERNAME, configs.RDD_PASSWORD)
rdd = readability.oauth(configs.RDD_APIKEY, configs.RDD_SECRET, token=token)
count = 0
for url, state in list:
try:
rdd.add_bookmark(url, archive=int(state))
count += 1
print count
except Exception, e:
if e.message:
print url, state
print e.message
def add_urls(urls, oauth_token, oauth_secret):
rdd = readability.oauth(
settings.API_KEY,
settings.API_SECRET,
token=( oauth_token, oauth_secret))
for url in urls:
try:
logging.error('Adding url %s' % url.get('url'))
rdd.add_bookmark(url = url.get('url'), archive = url.get('achive', False))
except Exception as e:
logger.error(e.__str__())
time.sleep(settings.ARTICLE_THROTTLE)
logging.error('Removing lock for %s' % oauth_token)
unlock(oauth_token)
def _get_user(self):
token = self.get_token()
self.api = readability.oauth(API_KEY, API_SECRET, token=token)
return self.api.get_me()
def main():
try:
# This script is meant only to run on a TTY
assert sys.stdout.isatty() and sys.stdin.isatty()
# Parse args
parser = argparse.ArgumentParser(description='Move Readability queue to Instapaper.')
parser.add_argument('-v', '--verbose', action='store_true', default=False)
parser.add_argument('-a', '--all', action='store_true', default=False,
help="Transfer all queued Readability items to Instapaper, whether or not they're already present.")
parser.add_argument('-i', '--instapaper-csv', type=argparse.FileType('r'),
help="Use supplied CSV (e.g. from instapaper.com) to filter what will be transferred.")
parser.add_argument('-k', '--readability-key', action='store', default=os.environ.get('READABILITY_KEY'),
help="The Readability API key to use.")
parser.add_argument('-s', '--readability-secret', action='store', default=os.environ.get('READABILITY_SECRET'),
help="The Readability API secret to use.")
args = parser.parse_args()
if not args.readability_key:
parser.error("A Readability API key is required. Specify with -k or the READABILITY_KEY envvar.")
if not args.readability_secret:
parser.error("A Readability API secret is required. Specify with -s or the READABILITY_SECRET envvar.")
# Warn about not using an Instapaper CSV
if not args.all and not args.instapaper_csv:
print >>sys.stderr, colored.yellow('Warning')+':', "No Instapaper CSV supplied.", \
"Won't filter already-added URLs. This can be hazardous to your rate limit.\n"
# Load the set of URLs already added to Instapaper
already_added = set()
if args.instapaper_csv:
for row in unicode_dict_csv_read(args.instapaper_csv):
already_added.add(row['URL'].strip().lower())
args.instapaper_csv.close()
# Log into Instapaper
sys.stdout.write("Instapaper email: ")
sys.stdout.flush()
insta_uname = raw_input().strip()
insta_pwd = getpass.getpass()
print (CURSOR_UP_ONE_AND_ERASE_LINE) * 2 # clear prompts
sys.stdout.write('Logging into Instapaper...')
sys.stdout.flush()
insta = instapaperlib.Instapaper(insta_uname, insta_pwd)
r, msg = insta.auth()
if r == 200:
print ' [ '+colored.green('OK')+' ]'
else:
print ' [ '+colored.red('FAILED')+' ]'
print >>sys.stderr, msg
sys.exit(1)
# Auth with Readability
sys.stdout.write("\nReadability username: ")
sys.stdout.flush()
read_uname = raw_input().strip()
read_pwd = getpass.getpass()
print (CURSOR_UP_ONE_AND_ERASE_LINE) * 3 # clear prompts
sys.stdout.write('Logging into Readability...')
sys.stdout.flush()
try:
read_token = readability.xauth(args.readability_key, args.readability_secret, read_uname, read_pwd)
rdd = readability.oauth(args.readability_key, args.readability_secret, token=read_token)
except readability.api.AuthenticationError, e:
print ' [ '+colored.red('FAILED')+' ]'
print >>sys.stderr, 'Invalid username or password.'
sys.exit(2)
else:
print ("No developer keys existing in apikeys.json. Please contact Readability to get your own developer keys.")
sys.exit(0)
# load settings
settingsFile = "data/settings.json"
settings = parser.getSettings(settingsFile)
if settings == False or not settings['token1'] or not settings['token2']:
print ("No Readability credentials existing. Please run ./setup.py")
sys.exit(0)
# do a connection test with the tokens
print ("Connecting to Readability to verify your token...")
token = (settings["token1"], settings["token2"])
connected = False
try:
rdd = readability.oauth(keys["consumerKey"], keys["consumerSecret"], token=token)
me = rdd.get_me()
connected = True
except:
pass
if not connected:
print ("Your Readability credentials seem wrong. Please run ./setup.py")
sys.exit(0)
print("Verification successful!")
# returns the html or false of a page
url = 'http://www.daemonology.net/hn-daily/'
fakeConnection = False # True for using data/fakehtml.txt instead of connecting
conn = None
if os.path.exists('secretdb.dat'):
if DEBUG:
print "Secret DB exists.. reading token from there"
ot_file = open('secretdb.dat', 'rb')
oauth_tokens = marshal.load(ot_file)
if DEBUG_SECRETS:
print "Read OAuth Tokens from DB", oauth_tokens
ot_file.close()
xauth_token = oauth_tokens[USERNAME]
print "Connecting to Readability..."
conn = readability.oauth(CONSUMER_KEY, CONSUMER_SECRET, token=xauth_token)
if DEBUG:
if conn is None:
print "Unable to connect to readability with local tokens"
else:
print "Connection to Readability created...", conn
if conn is None:
if DEBUG:
print "Requesting new tokens form Readability"
xauth_token = readability.xauth(CONSUMER_KEY, CONSUMER_SECRET, USERNAME,
PASSWORD)
if DEBUG_SECRETS:
print "Got XAuth Token", xauth_token
#!/usr/bin/python
import readability
from datetime import date, timedelta
from sqlalchemy import *
''' readability config '''
token = readability.xauth('read_username', 'read_apikey', 'read_email', 'read_pass')
rdd = readability.oauth('read_username', 'read_apikey', token=token)
''' sqlalchemy setup '''
engine = create_engine('mysql://*****:*****@db_host/db')
connection = engine.connect()
''' Fetch new bookmarks since yesterday '''
yesterday = date.today() - timedelta(1)
for b in rdd.get_bookmarks(added_since=yesterday.strftime('%m-%d-%y')):
result = connection.execute("INSERT INTO bookmarks VALUES(NULL, " + str(b.id) + ", '" + b.article.title.encode("utf-8") + "', '" + b.article.url.encode("utf-8") + "')" )
def readability_login(user, password):
xauth_token = (readability.xauth(READABILITY_CONSUMER_KEY, READABILITY_CONSUMER_SECRET, user, password))
return readability.oauth(READABILITY_CONSUMER_KEY, READABILITY_CONSUMER_SECRET, token=xauth_token)
def __init__(self, key, secret, username, pwd):
self.token = readability.xauth(key, secret, username,pwd)
self.rdd = readability.oauth(key, secret, token=self.token)
soup = Soup(requests.get(source_url).content)
urls = []
for td in soup("td", attrs={"class": "title"}):
anchor = td.find("a")
if not anchor:
continue
urls.append(anchor["href"])
if len(urls) == n:
break
return urls
from config import *
token = readability.xauth(CONSUMER_KEY, CONSUMER_SECRET, USERNAME, PASSWORD)
rdd = readability.oauth(CONSUMER_KEY, CONSUMER_SECRET, token=token)
user = rdd.get_me()
logger.info("Updating readability library")
logger.info("Fetching library")
library_urls = [u.article.url for u in user.bookmarks()]
logger.info("Found %d articles in library", len(library_urls))
# Fetch URLs
urls = get_article_urls_from_twitter_favourites(TWITTER_USERNAME)
urls += get_top_hacker_news_articles()
num_dupes = num_new = num_errors = 0
for url in urls:
#!/usr/bin/python
from PIL import Image
import sys, cStringIO, urllib, re
import MySQLdb, readability
# --/ readability authorization
token = readability.xauth('getassembly', 'CutXN5cHHAS8NxnPt3eq9au6hHfvUqcB', 'getassembly', 'assembly')
rdd = readability.oauth('getassembly', 'CutXN5cHHAS8NxnPt3eq9au6hHfvUqcB', token=token)
# --/ bookmark url
url = sys.argv[1]
print url
# --/ add to readability
b = rdd.add_bookmark(url)
a = rdd.get_article(b.article.id)
# --/ article content
t = a.title
c = a.content
print t
print c
# --/ extract video
m = re.compile(r'<iframe src="http://www.youtube.com/embed/(.*?)".*?</iframe>').search(c)
if m is None:
vid = ''
else:
vid = m.group(1)
