def fetch_pinboard():
# create pinboard connection (using username/password)
d = pinboard.open(token=pinboard_token)
dates = d.dates()
nb_dates = len(dates)
posts = []
text = "<h2>Pinboard 10 random picked links:</h2><br/>"
while(len(posts) < 10):
nb_random_date = int(floor(random.random() * nb_dates))
random_date = dates[nb_random_date:nb_random_date+1][0]["date"]
all_posts = d.posts(date=random_date)
random_int = int(floor(random.random() * len(all_posts)))
post = all_posts[random_int]
posts.append(post)
t = time.mktime(post['time_parsed'])
text += '<p style="font-size:80%; color: #666;margin-bottom: 0px;">' + time.strftime("%Y-%m-%d", time.localtime(t)) + "</p>"
text += '<p style="margin-bottom: 0px;">'+post["description"]+"</p>"
text += '<a href="' + post["href"] +'">' + post["href"] + "</a>" + "<br/><hr/><br/>"
text += "<br/>"
gm = gmail.GMail(gmail_username,gmail_password)
msg = gmail.Message('Today 10 random links', to=to_who,html=text)
gm.send(msg)
def main(args):
config = json.load(open(args.config))
p = pinboard.open(config['pinboard']['user'],
config['pinboard']['password'])
# Get today's posts to read
bookmarks = [
b for b in p.posts(date=datetime.datetime.now())
if b.get('toread', 'no') == 'yes'
]
for b in bookmarks:
h = httplib2.Http()
resp, content = h.request(b['href'])
doc = decruft.Document(content)
# Send that mail
send_to_kindle(config, b, doc)
# Mark it sent
keys = ['description', 'extended', 'tags']
updated = {}
for key in keys:
updated[key] = b[key]
updated['replace'] = 'yes'
updated['toread'] = 'no'
updated['url'] = b['href']
p.add(**updated)
def get_tags():
p = pinboard.open("mattfinlayson", "m0foLamb")
tags = p.tags()
tags_set = set()
for tag in tags:
tags_set.add(tag['name'].decode("utf-8"))
return tags_set
def downloadNewLinksFromPinboard(destinationPath, options):
log.debug("Starting up.")
last_update_file = options['pinboard']['lastUpdateFile']
already_downloaded_tag = options['pinboard']['alreadyDownloadedTag']
failed_download_tag = options['pinboard']['failedDownloadedTag']
enable_downloads = options['pinboard']['enableDownloads']
last_update_time = ""
if os.path.exists(last_update_file):
with open(last_update_file, 'r') as f:
for line in f:
last_update_time = line
pinboard_conn = pinboard.open(token=options['pinboard']['apiToken'])
pinboard_last_update_time = pinboard_conn.last_update()
log.debug("Last updated at %s. Pinboard updated at %s", last_update_time, pinboard_last_update_time)
if last_update_time == pinboard_last_update_time:
log.info("No changes. Exiting.")
return
successfulDownloads = [ ]
failedDownloads = [ ]
log.debug("Looking for new bookmarks...")
posts = pinboard_conn.posts(fromdt=last_update_time)
for p in posts:
url = p['href']
name = p['description']
tags = p['tags']
update_time = p['time']
urlMatch = False
for pat in options['pinboard']['urlPatterns']:
if re.search(pat, url):
urlMatch = True
if urlMatch and already_downloaded_tag not in tags and failed_download_tag not in tags:
log.debug(u"Found %s (%s)", name, url)
try:
name = download(url, destinationPath, options)
except Exception as e:
tags.append(failed_download_tag)
log.exception("Download failed. Adding failed tag.")
failedDownloads.append(name)
else:
tags.append(already_downloaded_tag)
log.debug("Download complete. Adding completed tag.")
successfulDownloads.append(name)
if enable_downloads:
pinboard_conn.add(url, name, replace='yes', tags=tags)
if enable_downloads:
with open(last_update_file, 'w') as f:
f.write(pinboard_last_update_time)
if len(successfulDownloads) > 0 or len(failedDownloads) > 0:
sucStr = string.join(['"'+name[0:20]+'"' for name in successfulDownloads], ", ")
failStr = string.join(['"'+name[0:20]+'"' for name in failedDownloads], ", ")
message = "%d succeeded, %d failed. OK: %s, Failed: %s" % (len(successfulDownloads), len(failedDownloads), sucStr, failStr)
Notification.send("PinTheVideo done", message, options)
def initialize_pinboard(gen):
p = pinboard.open(token=gen.settings['PINBOARD_TOKEN'])
args = dict()
if 'PINBOARD_COUNT' in gen.settings.keys():
args['count'] = gen.settings['PINBOARD_COUNT']
if 'PINBOARD_TAG' in gen.settings.keys():
args['tag'] = gen.settings['PINBOARD_TAG']
posts = p.posts(**args)
gen.context['pinboard_activity'] = posts
def initialize_pinboard(gen):
p = pinboard.open(token=gen.settings["PINBOARD_TOKEN"])
args = dict()
if "PINBOARD_COUNT" in gen.settings.keys():
args["count"] = gen.settings["PINBOARD_COUNT"]
if "PINBOARD_TAG" in gen.settings.keys():
args["tag"] = gen.settings["PINBOARD_TAG"]
posts = p.posts(**args)
gen.context["pinboard_activity"] = posts
def get_pinboard(self, usern, passw):
"""
Sign in to the pinboard service
"""
usern = '' + usern
passw = '' + passw
try:
pinboard_data = pinboard.open(username=usern, password=passw)
except urllib2.HTTPError, error:
print error
return {"Pass": False}
def process_mailbox(mailbox, _pinboard_username, _pinboard_password):
# find all 'unseen' messages
rv, data = mailbox.search(None, 'UnSeen')
if rv == 'OK':
pinboard_conn = pinboard.open(_pinboard_username, _pinboard_password)
for num in data[0].split():
rv2, data2 = mailbox.fetch(num, '(RFC822)')
if rv2 != 'OK':
logging.error("ERROR getting message {0}".format(num))
return
msg = email.message_from_string(data2[0][1])
dh = decode_header(msg.get('subject'))
default_charset = 'ASCII'
subject = ''.join([unicode(t[0], t[1] or default_charset) for t in dh])
charmap = {
0x201c: u'"',
0x201d: u'"',
0x2018: u"'",
0x2019: u"'",
0x2014: u'-',
}
subject = subject.translate(charmap)
subject = subject.replace('[peterl]', '').replace('\n', ' ').replace('\r', '')
subject = subject.strip()
logging.info(u'Message {0}: {1}'.format(num, subject))
logging.info(u'Raw Date: {0}'.format(msg['Date']))
body = get_first_text_block(msg)
urls = re.findall(
'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',
body)
logging.info(u'URL: {0}'.format(urls[0]))
pinboard_conn.add(urls[0], subject)
# mark as seen
# mailbox.store(data[0].replace(' ', ','), '+FLAGS', '\Seen')
# mailbox.store(data[0].replace(' ', ','), '+FLAGS', '\UnSeen')
else:
logging.info(u'rv: {0}'.format(rv))
return
def get_client(self):
"""Create a pinboard client.
"""
# Get a pinboard client
if self.options.token:
self.log.debug('logging in with token')
else:
self.log.debug('logging in with username and password')
client = pinboard.open(
self.options.user,
self.options.password,
self.options.token,
)
return client
def _get_client(username, password, token):
"""Create a pinboard client with the provided credentials.
"""
# Get a pinboard client
if token:
LOG.debug('logging in with token')
else:
LOG.debug('logging in with username and password')
client = pinboard.open(
username,
password,
token,
)
return client
def startup():
global pinboard_object
msg("Pinboard linker script version " + version + ".")
login = get_login()
try:
if len(login) == 1:
pinboard_object = pinboard.connect(token=login[0])
else:
pinboard_object = pinboard.open(login[0], login[1])
except HTTPError, e:
if e.code == 401:
err("Oops! Did you enter the correct login information?")
else:
err("Error! Unknown exception: " + str(e))
def test_delete_tag(self):
"""Test tag deletion"""
p = pinboard.open(token=conf.token)
test_url = 'http://github.com'
test_tag = '__testing__'
# Clean pre-conditions
p.delete(test_url)
# Test pre-conditions (no test tag)
tags = p.tags()
self.assertNotIn(test_tag, get_tag_names(tags))
# Adding a test bookmark
p.add(url=test_url,
description='GitHub',
extended='It\'s a GitHub!',
tags=(test_tag),
toread=False,
replace="yes")
api_wait()
# Tags contains new tag
tags = p.tags()
self.assertTrue(type(tags), dict)
self.assertIn(test_tag, get_tag_names(tags))
# Deleting test tag
p.delete_tag(test_tag)
api_wait()
# There are no posts with test tag
posts = p.posts(tag=test_tag)
self.assertFalse(posts)
# And no test tag any more
tags = p.tags()
self.assertNotIn(test_tag, get_tag_names(tags))
# Clean Up
p.delete(test_url)
def test_delete_tag(self):
"""Test tag deletion"""
p = pinboard.open(token=conf.token)
test_url = 'http://github.com'
test_tag = '__testing__'
# Clean pre-conditions
p.delete(test_url)
# Test pre-conditions (no test tag)
tags = p.tags()
self.assertNotIn(test_tag, get_tag_names(tags))
# Adding a test bookmark
p.add(url=test_url,
description='GitHub',
extended='It\'s a GitHub!',
tags=(test_tag),
toread=False,
replace="yes")
api_wait()
# Tags contains new tag
tags = p.tags()
self.assertTrue(type(tags), dict)
self.assertIn(test_tag, get_tag_names(tags))
# Deleting test tag
p.delete_tag(test_tag)
api_wait()
# There are no posts with test tag
posts = p.posts(tag=test_tag)
self.assertFalse(posts)
# And no test tag any more
tags = p.tags()
self.assertNotIn(test_tag, get_tag_names(tags))
# Clean Up
p.delete(test_url)
def run_main(username, password):
all_links = {}
feedobj = feedparser.parse(base_url + feed_url)
for item in feedobj['items']:
all_links[item['links'][0]['href']] = item['title']
item_text = item['content'][0]['value']
soup = BeautifulSoup(item_text)
for link in soup.findAll('a'):
all_links[(link.get('href'))] = link.getText()
pinboardobj = pinboard.open(username, password)
for link in all_links.keys():
time.sleep(rate_limit_seconds)
print('Adding link %s (%s)' % (link, all_links[link]))
if not link.startswith(mailto_url):
pinboardobj.add(url = link,
description = all_links[link],
extended = link,
tags = ("schof.org"))
def main(args):
config = json.load(open(args.config))
p = pinboard.open(config['pinboard']['user'], config['pinboard']['password'])
# Get today's posts to read
bookmarks = [b for b in p.posts(date=datetime.datetime.now()) if b.get('toread', 'no') == 'yes']
for b in bookmarks:
h = httplib2.Http()
resp, content = h.request(b['href'])
doc = decruft.Document(content)
# Send that mail
send_to_kindle(config, b, doc)
# Mark it sent
keys = ['description', 'extended', 'tags']
updated = {}
for key in keys:
updated[key] = b[key]
updated['replace'] = 'yes'
updated['toread'] = 'no'
updated['url'] = b['href']
p.add(**updated)
from flask import render_template,flash
from flask import jsonify
import pinboard
import json
import operator
DATABASE = "/tmp/app.db"
DEBUG = True
SECRET_KEY = "0219"
app = Flask(__name__,static_folder="../client/templates")
app.config.from_object(__name__)
p = pinboard.open("haradashinya","harashin0219")
@app.route("/tags",methods=["GET"])
def tags():
tags = [item for item in p.tags() if item["count"] > 0][0:20]
sorted_tags = sorted(tags,key = lambda x: x["count"] ,reverse=True)
return jsonify({"tags": sorted_tags})
@app.route("/search/<keyword>",methods=["POST","GET"])
def search(keyword):
posts = p.posts(tag = keyword)
res = []
def test_token(self):
print '\nTest Token Auth and Common Cases'
p = pinboard.open(token=conf.token)
self.common_case(p)
def test_token(self):
p = pinboard.open(token=conf.token)
self.common_case(p)
def test_canonical(self):
p = pinboard.open(conf.username, conf.password)
self.common_case(p)
return True
return False
# Set stdout to Unicode
sys.stdout = codecs.getwriter('utf8')(sys.stdout)
pinuser = # Add User Name
pinpasswd = # Add Password
# Create a day object for today
today = date.today()
# Connect to pinboard api
try:
p = pinboard.open(pinuser, pinpasswd)
except (RuntimeError, TypeError, NameError):
print 'Could not retrieve Pinboard links from the API'
# Get a list of dictionaries from pinboard api
todays_posts = p.posts(date=today)
# Print out only the key/value pairs we're interested in
for x in todays_posts:
desc = '* [' + x['description'] + ']'
url = '(' + x['href']+ ')'
if contains(x['tags'], lambda y: y == u'\xa1'):
print desc + url + " @2ndLook"
def __init__(self, username, pwd):
self.all_tags = []
self.pinboard_account = pinboard.open(username, pwd)
self.get_tags()
def downloadNewLinksFromPinboard(destinationPath, options):
log.debug("Starting up.")
last_update_file = options['pinboard']['lastUpdateFile']
already_downloaded_tag = options['pinboard']['alreadyDownloadedTag']
failed_download_tag = options['pinboard']['failedDownloadedTag']
enable_downloads = options['pinboard']['enableDownloads']
last_update_time = ""
if os.path.exists(last_update_file):
with open(last_update_file, 'r') as f:
for line in f:
last_update_time = line
pinboard_conn = pinboard.open(token=options['pinboard']['apiToken'])
pinboard_last_update_time = pinboard_conn.last_update()
log.debug("Last updated at %s. Pinboard updated at %s", last_update_time,
pinboard_last_update_time)
if last_update_time == pinboard_last_update_time:
log.info("No changes. Exiting.")
return
successfulDownloads = []
failedDownloads = []
log.debug("Looking for new bookmarks...")
posts = pinboard_conn.posts(fromdt=last_update_time)
for p in posts:
url = p['href']
name = p['description']
tags = p['tags']
update_time = p['time']
urlMatch = False
for pat in options['pinboard']['urlPatterns']:
if re.search(pat, url):
urlMatch = True
if urlMatch and already_downloaded_tag not in tags and failed_download_tag not in tags:
log.debug(u"Found %s (%s)", name, url)
try:
name = download(url, destinationPath, options)
except Exception as e:
tags.append(failed_download_tag)
log.exception("Download failed. Adding failed tag.")
failedDownloads.append(name)
else:
tags.append(already_downloaded_tag)
log.debug("Download complete. Adding completed tag.")
successfulDownloads.append(name)
if enable_downloads:
pinboard_conn.add(url, name, replace='yes', tags=tags)
if enable_downloads:
with open(last_update_file, 'w') as f:
f.write(pinboard_last_update_time)
if len(successfulDownloads) > 0 or len(failedDownloads) > 0:
sucStr = string.join(
['"' + name[0:20] + '"' for name in successfulDownloads], ", ")
failStr = string.join(
['"' + name[0:20] + '"' for name in failedDownloads], ", ")
message = "%d succeeded, %d failed. OK: %s, Failed: %s" % (len(
successfulDownloads), len(failedDownloads), sucStr, failStr)
Notification.send("PinTheVideo done", message, options)
def get_links(tags):
p = pinboard.open("mattfinlayson", "m0foLamb")
output = []
for tag in tags:
output.append(p.posts(tag=tag, count=3))
return output
def test_canonical(self):
p = pinboard.open(conf.username, conf.password)
self.common_case(p)
#!/usr/bin/env python
import pinboard
import sys
import requests
from bs4 import BeautifulSoup
p = pinboard.open(sys.argv[1], sys.argv[2])
posts = p.posts()
for post in posts:
# {u'extended': u'', u'hash': u'1ab85941f250820b7cb25dccae71c3f0', u'description': u'', u'tags': [u'gis', u'maps', u'geo', u'python'], u'time_parsed': time.struct_time(tm_year=2014, tm_mon=1, tm_mday=29, tm_hour=20, tm_min=4, tm_sec=30, tm_wday=2, tm_yday=29, tm_isdst=-1), u'href': u'http://mapnik.org/', u'time': u'2014-01-29T20:04:30Z'}
# print post
changed = 0;
print "Processing post: " + post['href']
if post['description'] == '':
try:
h = requests.get(post['href'])
except:
print " Unexpected error: ", sys.exc_info()[0]
continue
if not h.status_code == requests.codes.ok:
print " Got stauts " + str(h.status_code) + ", skipping"
continue
def test_canonical(self):
print '\nTest Username/Pwd Auth and Common Cases'
p = pinboard.open(conf.username, conf.password)
self.common_case(p)
maintype = response.headers['Content-Type'].split(';')[0].lower()
# We don't want image links
if maintype not in ('image/png', 'image/jpeg', 'image/gif'):
# Need to handle gzip content if we want to grab the page title. Lots of sites send gzip now.
if response.info().get('Content-Encoding') == 'gzip':
buf = StringIO(response.read())
f = gzip.GzipFile(fileobj=buf)
data = f.read()
# I guess I just prefer a real link to shitty t.co links
fullURL = response.url
if fullURL is not None:
print fullURL
# This could be done with lxml but BeautifulSoup is easy
soup = BeautifulSoup(data)
# Get the title for the page
myTitle = soup.html.head.title
pyTitle = myTitle.string
print pyTitle
# Assemble the bookmark notes. Create Twitter link for RSS viewing
bookmarkExtended = '<p>'+user+'</p>\n<p>' + tweetMsg + '</p>\n\n' + '<a href="'+sourceURL+'">Twitter Source</a>'
try:
p = pinboard.open(username = pyAccount, token = pyToken)
postResult = p.add(url=fullURL, description=pyTitle, extended=bookmarkExtended, tags= (rssTag))
except (RuntimeError, TypeError, NameError):
print RuntimeError
print NameError
print TypeError
except urllib2.HTTPError, err:
# If it's an http error like 404, just skip the link. We don't have time for this junk.
continue
def test_token(self):
p = pinboard.open(token=conf.token)
self.common_case(p)
def test_canonical(self):
print '\nTest Username/Pwd Auth and Common Cases'
p = pinboard.open(conf.username, conf.password)
self.common_case(p)
if found_notes.totalNotes == len(all_evernote_bookmarks):
keep_looking = False
else:
note_offset = len(all_evernote_bookmarks)
time.sleep(1)
print "Success. Retrieved %d Bookmarks." % len(all_evernote_bookmarks)
all_evernote_bookmarks_map = dict()
all_evernote_uris = list()
for note in all_evernote_bookmarks:
if note.attributes.sourceURL != None:
all_evernote_bookmarks_map[note.attributes.sourceURL] = note
all_evernote_uris.append(note.attributes.sourceURL)
all_evernote_uris.reverse()
# Now we do the same dance with Pinboard
p = pinboard.open(token=pinboard_token)
print "connecting to Pinboard…"
p = pinboard.open(pinboard_username, pinboard_pass)
print "Retrieving all Pinboard bookmarks…"
all_pinboard_posts = p.posts()
print "Success. Retrieved %d Bookmarks." % len(all_pinboard_posts)
all_pinboard_uris = list()
all_pinboard_posts_map = dict()
for post in all_pinboard_posts:
if post["href"] != None:
all_pinboard_uris.append(post["href"])
all_pinboard_posts_map[post["href"]] = post
all_pinboard_uris.reverse()
# now we compare and prep the sync
def test_token(self):
print '\nTest Token Auth and Common Cases'
p = pinboard.open(token=conf.token)
self.common_case(p)
def get_pinboard_links(self):
try:
p = pinboard.open(token=self.pinboard_api_token)
except urllib2.HTTPError, error:
raise PinboardToMTError("Can't connect to Pinboard: %s" % error)
import pinboard
import PinboardCredentials
import BufferCredentials
from datetime import date, timedelta
p = pinboard.open(token=PinboardCredentials.PINBOARD_API_TOKEN)
# posts = p.posts(fromdt=p["last_updated"]) # only get content since last run
pastdays = date.today() - timedelta(3)
posts = p.posts(fromdt=pastdays)
updates = []
for post in posts:
updates.append(post["description"] + ' ' + post["href"])
from pprint import pprint as pp
from colorama import Fore
from buffpy.models.update import Update
from buffpy.managers.profiles import Profiles
from buffpy.managers.updates import Updates
from buffpy.api import API
token = BufferCredentials.BUFFER_APP_TOKEN
api = API(client_id=BufferCredentials.BUFFER_CLIENT_ID,
client_secret=BufferCredentials.BUFFER_CLIENT_SECRET,
access_token=token)
profile = Profiles(api=api).filter(service='twitter')[0]
for update in updates:
profile.updates.new(update.encode('ascii', 'ignore'))
def __init__(self, username, pwd):
self.all_tags = []
self.pinboard_account = pinboard.open(username, pwd)
self.get_all_tags()
