def _import_tags(username, password):
""" Import tags from delicious. Returns a list of dict objects
representing tags """
dapi = DeliciousAPI(username, password)
tags = dapi.tags_get()
return tags['tags']
def _import_bookmarks(username, password, token='.deliciousbackup'):
""" Import bookmarks from delicious. Returns a list of dict
objects representing posts """
token_path = os.path.join(os.path.abspath(os.path.expanduser('~')),
token)
if os.path.exists(token_path):
with open(token_path) as f:
last_run = datetime.fromtimestamp(float(f.read()))
else:
last_run = time.time()
with open(token_path, 'w') as f:
f.write(str(last_run))
last_run = datetime.fromtimestamp(last_run)
dapi = DeliciousAPI(username, password)
api_date = dapi.posts_update()
api_date = api_date['update']['time']
last_update = datetime(api_date.tm_year, api_date.tm_mon,
api_date.tm_mday, hour=api_date.tm_hour,
minute=api_date.tm_min,
second=api_date.tm_sec)
if last_update > last_run:
#last_run.microsecond=0
posts = dapi.posts_all(fromdt=last_run.isoformat())
else:
posts = dapi.posts_all()
return posts['posts']
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('db', help="The ybookmarks.sqlite file in firefox "
"profile directory")
args = parser.parse_args()
conn = sqlite3.connect(args.db)
curs = conn.cursor()
curs.execute("select rowid, name, url, shared, description, added_date from bookmarks")
bookmarks = dict((id, Bookmark(name=name, url=url,
add_date=added_date/1000000,
private=(shared != "true"),
description=description))
for id, name, url, shared, description, added_date
in curs)
curs.execute("select rowid, name from tags")
tags = dict((id, name) for id, name in curs)
bookmarks_tags = defaultdict(list)
curs.execute("select bookmark_id, tag_id from bookmarks_tags")
for bookmark_id, tag_id in curs:
bookmarks_tags[bookmark_id].append(tag_id)
for bookmark_id, bookmark in bookmarks.items():
bookmark.tags = [tags[tid] for tid in bookmarks_tags[bookmark_id]]
api = DeliciousAPI(raw_input("username: "******"password: "******"Posting %s" % bm.name
api.posts_add(url=bm.url, description=bm.name,
extended=bm.description, tags=','.join(bm.tags),
dt=bm.add_date.strftime('%Y-%m-%dT%H:%M:%SZ'),
replace=True,
shared=not bm.private)
time.sleep(1)
def setUpApi(self, codec='latin-1'):
"Setup API with custom encoding."
global pwd, usr
if not (usr or pwd):
self.api = None
else:
self.creds = usr, pwd
self.api = DeliciousAPI(usr, pwd, codec)
# should raise or return:
update = self.api.posts_update()
if not update:
sys.exit("Cannot stat server, unknown error")
class ApiSystemTest(unittest.TestCase):
"""Base class for API system tests.
Derive API system tests from this baseclass.
"""
def setUpApi(self, codec='latin-1'):
"Setup API with custom encoding."
global pwd, usr
if not (usr or pwd):
self.api = None;
else:
self.creds = usr, pwd
self.api = DeliciousAPI(usr, pwd, codec)
# should raise or return:
update = self.api.posts_update()
if not update:
sys.exit("Cannot stat server, unknown error")
def setUp(self):
"Default setUp before each ApiSystemTest testcase"
return self.setUpApi()
def assertContains(self, container, object):
self.failUnless(object in container, "%s not in %s" %(object, container))
class ApiSystemTest(unittest.TestCase):
"""Base class for API system tests.
Derive API system tests from this baseclass.
"""
def setUpApi(self, codec='latin-1'):
"Setup API with custom encoding."
global pwd, usr
if not (usr or pwd):
self.api = None
else:
self.creds = usr, pwd
self.api = DeliciousAPI(usr, pwd, codec)
# should raise or return:
update = self.api.posts_update()
if not update:
sys.exit("Cannot stat server, unknown error")
def setUp(self):
"Default setUp before each ApiSystemTest testcase"
return self.setUpApi()
def assertContains(self, container, object):
self.failUnless(object in container,
"%s not in %s" % (object, container))
class ApiSystemTest(unittest.TestCase): """Base class for API system tests. Derive API system tests from this baseclass. """ def setUpApi(self, codec='latin-1'): "Setup API with custom encoding." if usr == '' or pwd == '': self.api = None; else: self.api = DeliciousAPI(usr, pwd, codec) def setUp(self): "Default setUp before each ApiSystemTest testcase" return self.setUpApi() def testInstance(self): "Test for valid DeliciousAPI instance" self.assert_(isinstance(self.api, DeliciousAPI)) # precautions so we don't query to quickly: def _get_post(self, url): p = self.api.posts_get(url=url) time.sleep(5) tries = 3 while tries: p = self.api.posts_get(url=url) if p['posts'] == []: tries = tries - 1 else: return p return p def _get_nonpost(self, url): p = self.api.posts_get(url=url) time.sleep(5) tries = 3 while tries: p = self.api.posts_get(url=url) if p['posts'] == []: return p else: tries = tries - 1 return p
def __init__(self, username, password):
"""
Initializes the DeliciousSyncDB object.
This functions accepts two arguments
username and password. Both are required.
"""
self.api = DeliciousAPI(username, password)
def execute(self):
dapi = DeliciousAPI(settings.DELICIOUS_USER,settings.DELICIOUS_PASSWORD)
for post in dapi.posts_recent()['posts']:
try:
bm = Bookmark.objects.get(source=CONTENT_SOURCE_DELICIOUS,
source_id=post['hash'])
except ObjectDoesNotExist:
bookmark = Bookmark(source=CONTENT_SOURCE_DELICIOUS)
bookmark.title = post['description']
bookmark.url = post['href']
bookmark.tags = post['tag']
bookmark.description = post['extended']
bookmark.source_id = post['hash']
#bookmark.meta_value = post['meta']
bookmark.date_created = parser.parse(post['time'])
bookmark.save()
else:
break
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('db',
help="The ybookmarks.sqlite file in firefox "
"profile directory")
args = parser.parse_args()
conn = sqlite3.connect(args.db)
curs = conn.cursor()
curs.execute(
"select rowid, name, url, shared, description, added_date from bookmarks"
)
bookmarks = dict(
(id,
Bookmark(name=name,
url=url,
add_date=added_date / 1000000,
private=(shared != "true"),
description=description))
for id, name, url, shared, description, added_date in curs)
curs.execute("select rowid, name from tags")
tags = dict((id, name) for id, name in curs)
bookmarks_tags = defaultdict(list)
curs.execute("select bookmark_id, tag_id from bookmarks_tags")
for bookmark_id, tag_id in curs:
bookmarks_tags[bookmark_id].append(tag_id)
for bookmark_id, bookmark in bookmarks.items():
bookmark.tags = [tags[tid] for tid in bookmarks_tags[bookmark_id]]
api = DeliciousAPI(raw_input("username: "******"password: "******"Posting %s" % bm.name
api.posts_add(url=bm.url,
description=bm.name,
extended=bm.description,
tags=','.join(bm.tags),
dt=bm.add_date.strftime('%Y-%m-%dT%H:%M:%SZ'),
replace=True,
shared=not bm.private)
time.sleep(1)
class DeliciousHelper(object):
"""
Helper class for the delicious api
"""
def __init__(self, username, password):
self._username = username
self._password = password
self._api = DeliciousAPI(self._username, self._password)
def get_last_update(self):
dt = datetime.fromtimestamp(mktime(self._api.posts_update()['update']['time']))
return dt
def get_recent_links(self, dt):
l = []
for link in self._api.posts_recent(count=30)['posts']:
pdt = datetime.strptime(link['time'], '%Y-%m-%dT%H:%M:%SZ')
if pdt > dt:
l.append(link)
return l
def setUpApi(self, codec='latin-1'):
"Setup API with custom encoding."
global pwd, usr
if not (usr or pwd):
self.api = None;
else:
self.creds = usr, pwd
self.api = DeliciousAPI(usr, pwd, codec)
# should raise or return:
update = self.api.posts_update()
if not update:
sys.exit("Cannot stat server, unknown error")
def cook_select_resource (request, key):
"""Take the submitted resource and tag it in delicious using the users prerferences ready for the Baking step"""
# This is in the framework template return HttpResponse(render_to_string(request, template_name, data),
# Process request["url"]
# res_url = request.REQUEST["resource_url"]
res_url = request.REQUEST.get("resource_url")
# res_txt = request.REQUEST["resource_text"]
res_txt = request.REQUEST.get("resource_text")
if res_txt is None:
res_txt = "Untitled resource";
if res_url is None:
print "No URL given"
else:
# a = pydelicious.apiNew('ibbomashup', '**************')
a = DeliciousAPI('ibbomashup', '**************')
a.tags_get()
print "Select the resource "+res_url+" notes text is "+res_txt
a.posts_add(res_url, res_txt, tags="test", extended="text");
# a.posts_add("http://my.com/", "title", extended="description", tags="my tags")
return HttpResponse("fred.html")
def update_links(self):
api = DeliciousAPI(self.username, self.password)
# get the latest update according to delicious
latest_update_utc = api.posts_update()
latest_update_utc = utc_to_epoch(latest_update_utc['update']['time'])
if self.last_known_update_utc < latest_update_utc:
posts = api.posts_all(fromdt=self._last_update_timestring())
for p in posts['posts']:
link = LinkPost()
link.blog_id = self.blog_id
link.title = p['description']
link.link = p['href']
link.description_raw = p['extended']
link.publish(publish_time = self._parse_datetime(p['time']))
link.tags = p['tag']
link.save()
self.last_known_update_utc = self._parse_time(posts['update']) + 1
self.save()
def main():
username = str(raw_input('Username:'******'Pwd:')
a = DeliciousAPI(username, pwd)
# Get list of tagged urls
tagposts = get_tagposts('programming')
sleep(2)
# Put tagged urls into list
list1 = []
for a in range(len(tagposts)):
count = 0
for b in tagposts[a]:
count += 1
if count == 4:
list1.append(tagposts[a][b])
# Get all instances of urls
dic2 = {}
for url in list1:
post = get_urlposts(url)
# Put tags into histogram
for a in range(len(post)):
count = 0
for b in post[a]:
count += 1
if count == 3:
if (post[a][b] not in dic2):
dic2[post[a][b]] = 1
else:
dic2[post[a][b]] = dic2[post[a][b]] + 1
# Invert the histogram and print most often first
dic3 = invert_dict(dic2)
# Print results
res = dic3.keys()
res.sort()
res.reverse()
print res
for z in res:
print dic3[z], "appears", z, "times"
import getpass
from pydelicious import DeliciousAPI, DeliciousItemExistsError
sourceUser = raw_input("Username [source]: ")
sourcePasswd = getpass.getpass("Password [source]: ")
sourceDlcs = DeliciousAPI(sourceUser, sourcePasswd)
print "Getting all bookmarks from source:" + sourceUser
sourceBkmrks = sourceDlcs.posts_all()['posts']
print "Done getting bookmarks"
fromTag = ' from:' + sourceUser
destUser = raw_input("Username [destination]: ")
destPasswd = getpass.getpass("Password [destination]: ")
destDlcs = DeliciousAPI(destUser, destPasswd)
sourceSize = len(sourceBkmrks)
for i in range(46,sourceSize):
bkmrk = sourceBkmrks[i]
href = bkmrk['href']
desc = bkmrk['description']
ext = bkmrk['extended'] if 'extended' in bkmrk else ''
share = bkmrk['shared'] if 'shared' in bkmrk else 'yes'
date = bkmrk['time']
tag = bkmrk['tag'] if 'tag' in bkmrk else ''
tag += fromTag
print 'Copying (' + str(i+1) + '/' + str(sourceSize) + ') '+ desc + ' (' + href + ')'
try :
destDlcs.posts_add(href, desc, extended = ext, tags = tag, dt= date, shared = share)
except DeliciousItemExistsError:
desc + ' (' + href + ') already exists'
from pydelicious import DeliciousAPI
from getpass import getpass
# logon to delicious
account = DeliciousAPI('igniteflow', getpass('Password:'******'posts']
# inspect your data
for post in posts:
print post['description'], post['tag'], post['href']
##files = ['delicios_import.png','delicios_import.png']
##files = ['bookmarks-2013-11-05.html']
if len(files) == 0:
print "Please choose files with tags 'a' (and href attr) and try again."
print "Exit."
sys.exit()
k = 1
while k:
try:
us = raw_input('Please type your login on Delicios: ')
pa = raw_input('Please type your passw on Delicios: ')
print "\n"
user = DeliciousAPI(us, pa)
sign = user.posts_recent()
k = 0
except Exception, d:
print str(d) + "\n", "Authentication problem. Try again."
k = 1
yes = True
no = False
# | posts_add(self, url, description, extended='', tags='', dt='', replace=False, shared=True, **kwds)
# | Add a post to del.icio.us. Returns a `result` message or raises an
# | ``DeliciousError``. See ``self.request()``.
# |
# | &url (required)
# | the url of the item.
def post_to_Delicious(art_data):
a = DeliciousAPI(config.DELICIOUS_USER,config.DELICIOUS_PWD)
arturl = config.BASE_URL % art_data['bibcode'].strip()
entry_tags = ",".join(art_data['keywords'])
a.posts_add(arturl, art_data['title'], extended=config.EXTENDED_DESCRIPTION, tags=entry_tags)
def __init__(self, username, password):
self._username = username
self._password = password
self._api = DeliciousAPI(self._username, self._password)
##files = ['delicios_import.png','delicios_import.png']
##files = ['bookmarks-2013-11-05.html']
if len(files) == 0:
print "Please choose files with tags 'a' (and href attr) and try again."
print "Exit."
sys.exit()
k = 1
while k:
try:
us = raw_input('Please type your login on Delicios: ')
pa = raw_input('Please type your passw on Delicios: ')
print "\n"
user = DeliciousAPI(us,pa)
sign = user.posts_recent()
k = 0
except Exception, d:
print str(d) + "\n", "Authentication problem. Try again."
k = 1
yes = True
no = False
# | posts_add(self, url, description, extended='', tags='', dt='', replace=False, shared=True, **kwds)
# | Add a post to del.icio.us. Returns a `result` message or raises an
# | ``DeliciousError``. See ``self.request()``.
# |
# | &url (required)
def setUpApi(self, codec='latin-1'): "Setup API with custom encoding." if usr == '' or pwd == '': self.api = None; else: self.api = DeliciousAPI(usr, pwd, codec)
get('DELICIOUS_USER', default='delicious-user-here'),
get('DELICIOUS_PASS', default='some-delicious-password-here'),
get('RETRY_ATTEMPTS', default=10, convert=int),
get('DATABASE_FILE', default='aacinfo_db.sqlite'),
get('SQLALCHEMY_DATABASE_URI', default='sqlite:///realpath/db_file'),
get('SQLALCHEMY_ECHO', default=True, convert=word_for_true),
get('USERNAME', default='admin'),
get('PASSWORD', default='pass'),
get('SLACK_HOOK', default='some_slack_url_here')
))
#slack
slacker = Slack(app.config['SLACK_HOOK'])
db = SQLAlchemy(app)
a = DeliciousAPI(app.config['DELICIOUS_USER'], app.config['DELICIOUS_PASS'])
EMAIL_REGEX = '\w[\w\.-]*@\w[\w\.-]+\.\w+'
login_manager = login.LoginManager()
login_manager.init_app(app)
@login_manager.user_loader
def load_user(user_id):
return db.session.query(Users).get(user_id)
def get_mailchimp_api():
return mailchimp.Mailchimp(app.config['MAILCHIMP_APIKEY'])
def addPostToDelicious(link,title,text,author,type_name):
def main(argv):
"""This will prepare al input data and call a command function to perform
the operations. Default command is `info()`.
Configuration file is loaded and used to store username/password.
"""
global DEBUG
if not argv:
argv = sys.argv[1:]
### Parse argument vector
optparser, opts, args = parse_argv_split(__options__, argv, __usage__)
if opts['verboseness']:
v = int(opts['verboseness'])
DEBUG = v
pydelicious.DEBUG = v
# First argument is command
if len(args) > 0:
cmdid = args.pop(0)
else:
cmdid = 'info'
if not cmdid in __cmds__:
optparser.exit("Command must be one of %s" % ", ".join(__cmds__))
### Parse config file
conf = ConfigParser()
conf_file = opts['config']
# reads whatever it can or nothing:
conf.read(conf_file)
# Check for section and initialize using options if needed
if not 'dlcs' in conf.sections():
# initialize a new configuration?
if not 'username' in opts:
opts['username'] = os.getlogin()
if not 'password' in opts:
opts['password'] = getpass.getpass("Password for %s: " %
opts['username'])
conf.add_section('dlcs')
conf.set('dlcs', 'username', opts['username'])
v = raw_input("Save password to config (%s)? [Y]es/No: " % conf_file)
if v in ('y', 'Y', ''):
conf.set('dlcs', 'password', opts['password'])
conf.write(open(conf_file, 'w'))
if not 'local-files' in conf.sections():
# Other default settings:
conf.add_section('local-files')
conf.set('local-files', 'tags', expanduser("~/.dlcs-tags.xml"))
conf.set('local-files', 'posts', expanduser("~/.dlcs-posts.xml"))
conf.write(open(conf_file, 'w'))
#return "Config written. Just run dlcs again or review the default config first."
### Merge config items under 'dlcs' with opts
# conf provides defaults, command line options override
options = dict(conf.items('dlcs'))
options.update(opts)
if not 'password' in options:
options['password'] = getpass.getpass("Password for %s: " %
options['username'])
# Force output encoding
sys.stdout = codecs.getwriter(options['encoding'])(sys.stdout)
# TODO: run tests, args = [a.decode(options['encoding']) for a in args]
# DeliciousAPI instance to pass to the command functions
dlcs = DeliciousAPI(options['username'],
options['password'],
codec=options['encoding'])
# TODO: integrate debugwrapper if DEBUG:
if DEBUG > 2:
dlcs = DebugWrapper(dlcs, sys.stderr)
### Defer processing to command function
cmd = getattr(sys.modules[__name__], cmdid)
try:
return cmd(conf, dlcs, *args, **options)
except PyDeliciousException, e:
print >> sys.stderr, e
from pydelicious import PyDeliciousThrottled
from pydelicious import PyDeliciousUnauthorized
from getpass import getpass
from time import sleep
import xml.dom.minidom
from xml.dom.minidom import Node
import time
if (len(sys.argv) < 2):
print "This script will copy your Google Bookmarks to Delicious. Any bookmarks at Delicious that point to the same url as a bookmark being imported will have it's tags and date updated. You will be prompted for your delicious username and password. All utf-8 characters are converted to ascii. Sorry i18n users."
print "usage:\n" + sys.argv[0] + " <googleBookmark.xml>\n\nWhere <googleBookmark.xml> is your google bookmark feed from http://www.google.com/bookmarks/find?q=&output=rss&num=10000"
exit(0)
doc = xml.dom.minidom.parse(sys.argv[1])
username = raw_input('username:'******'Password:'******'ascii','replace')
u = node.getElementsByTagName("link")
if (len(u) > 0 and len(u[0].childNodes) > 0):
url = u[0].childNodes[0].data.encode('ascii','replace')
da = node.getElementsByTagName("pubDate")
if (len(da) > 0 and len(da[0].childNodes) > 0):
date = da[0].childNodes[0].data.encode('ascii','replace')
class DeliciousSyncDB:
def __init__(self, username, password):
"""
Initializes the DeliciousSyncDB object.
This functions accepts two arguments
username and password. Both are required.
"""
self.api = DeliciousAPI(username, password)
def _cleanTags(self, tags):
"""
Utility function that sets tags to lower case, removes
commas and double quotes and removes all duplicate tags.
Returns a unicode string.
"""
tags = tags.lower().replace('\"', '').replace(',', '').split(' ')
tags = set(tags)
tags = ' '.join(tags)
return u'%s' % tags
def _syncPost(self, post):
"""
Utility function that saves bookmarks to the
local database.
In the case a bookmark already exists it will be
updated instead.
"""
time_obj = time.strptime(post['time'], "%Y-%m-%dT%H:%M:%SZ")
save_date = datetime.datetime(*time_obj[0:7])
try:
shared = post['shared']
except KeyError:
shared = True
finally:
if shared == 'no': shared = False
tags = self._cleanTags(post['tag'])
d = {
'title': post['description'],
'url': post['href'],
'tags': tags,
'notes': post['extended'],
'post_hash': post['hash'],
'post_meta': post['meta'],
'save_date': save_date,
'shared': shared
}
b, created = Bookmark.objects.get_or_create(url = d['url'], defaults = d)
if created == False:
if not b.post_meta == unicode(d['post_meta']):
b = Bookmark(
id=b.id,
title=d['title'],
url=d['url'],
tags=d['tags'],
notes=d['notes'],
post_hash=d['post_hash'],
post_meta=d['post_meta'],
save_date=d['save_date'],
shared=d['shared']
)
b.save(syncAPI=True)
def syncRecent(self, results=15):
"""
Syncronizes recent Delicious boomarks to a
local database.
This uses the posts/all api call instead of
using the posts/recent call. Doing so provides the
meta attribute in order to update previously saved
bookmarks with modified data from Delicious.
syncRecent takes one argument for results. If not
specified the default number of results returned
is 15.
"""
posts = self.api.posts_all(results = str(results))
for post in posts['posts']:
self._syncPost(post)
def syncAll(self):
"""
Syncronizes all Delicious boomarks to a
local database.
Using the meta attribute previously saved bookmarks
will also be updated as well.
"""
posts = self.api.posts_all()
for post in posts['posts']:
self._syncPost(post)
def processQueue(self):
"""
Queue processor to process local bookmarks that are
going to be pushed to Delicious using posts/add.
Bookmarks saved locally are flagged as queued and
unflagged as they have been processed.
If a boomark is unsucessfully updated the program will
fail silently and move on to the next one until the
entire queue has been processed once through.
"""
bookmarks = Bookmark.objects.filter(queued=True)
for bookmark in bookmarks:
time.sleep(1.5)
try:
if bookmark.shared == False:
shared = 'no'
else:
shared = 'yes'
self.api.posts_add(
bookmark.url,
bookmark.title,
extended=bookmark.notes,
tags=bookmark.tags,
shared=shared,
replace='yes')
bookmark.queued = False
bookmark.save(syncAPI=True)
except:
pass
print 'Update existing bookmark: ', entry['uri']
meta = getBookmarkMeta(entry['uri'])
own = deliciousApi.posts_get(url=entry['uri'])
result = {'url': meta['url'],
'description': own['posts'][0]['description'],
'extended': own['posts'][0]['extended'] if '' != own['posts'][0]['extended'] else meta['extended'],
'tags': ' '.join(tags + meta['tags']) + ' ' + own['posts'][0]['tag']}
deliciousApi.posts_add(result['url'], result['description'], extended=result['extended'], tags=result['tags'], replace=True, shared=False)
# # #
# ## ## ## # # #
# # # # # # # # ## #
# # # # # # # # # #
# # # ###### # # # #
# # # # # # # ##
# # # # # # # #
# Init delicious api
deliciousApi = DeliciousAPI(deliciousUser, deliciousPassword)
allPosts = deliciousApi.posts_all()
# Read and process JSON Data
jsonDataFile = open(importFile)
jsonData = json.load(jsonDataFile)
jsonDataFile.close()
processEntry(jsonData)
print 'All done ... have a nice day!'
