def new_user(request):
"""Add a new user to the system manually."""
rdict = request.params
u = User()
u.username = unicode(rdict.get('username'))
u.email = unicode(rdict.get('email'))
passwd = get_random_word(8)
u.password = passwd
u.activated = True
u.is_admin = False
u.api_key = User.gen_api_key()
try:
DBSession.add(u)
DBSession.flush()
# We need to return the password since the admin added the user
# manually. This is only time we should have/give the original
# password.
ret = dict(u)
ret['random_pass'] = passwd
return _api_response(request, ret)
except IntegrityError, exc:
# We might try to add a user that already exists.
LOG.error(exc)
request.response.status_int = 400
return _api_response(request, {
'error': 'Bad Request: User exists.',
})
def process(self):
"""Given a file, process it"""
if self.file_handle.closed:
self.file_handle = open(self.file_handle.name)
parsed = etree.parse(self.file_handle)
count = 0
ids = []
for post in parsed.findall('post'):
if 'javascript:' in post.get('href'):
continue
add_date = dateparser.parse(post.get('time'))
try:
bmark = self.save_bookmark(post.get('href'),
post.get('description'),
post.get('extended'),
post.get('tag'),
dt=add_date)
count = count + 1
if bmark:
bmark.stored = bmark.stored.replace(tzinfo=None)
DBSession.flush()
except InvalidBookmark, exc:
bmark = None
if bmark:
ids.append(bmark.bid)
if count % COMMIT_SIZE == 0:
transaction.commit()
def test_update_post(self):
"""Updates allowed over the last one
If you /post/add to an existing bookmark, the new data overrides the
old data
"""
self._get_good_request()
# now build a new version of the request we can check
session = DBSession()
prms = {
'url': u'http://google.com',
'description': u'This is my updated google desc',
'extended': 'updated extended notes about it in full form',
'tags': u'python search updated',
'api_key': API_KEY
}
req_params = urllib.urlencode(prms)
self.testapp.get('/admin/delapi/posts/add?' + req_params)
session.flush()
res = Bmark.query.filter(Bmark.hash_id == GOOGLE_HASH).one()
ok_('updated' in res.description,
'Updated description took: ' + res.description)
ok_('updated' in res.extended,
'Updated extended took: ' + res.extended)
ok_('python' in res.tags, 'Found the python tag in the bmark')
ok_('search' in res.tags, 'Found the search tag in the bmark')
ok_('updated' in res.tags, 'Found the updated tag in the bmark')
def test_unique_ct(self):
"""Verify that our unique count method is working"""
ct = 5
common = 'testing.com'
users = []
for i in range(ct):
user = User()
user.username = gen_random_word(10)
DBSession.add(user)
users.append(user)
for i in range(ct - 2):
b = Bmark(
url=gen_random_word(12),
username=users[i].username
)
DBSession.add(b)
# Add in our dupes
c = Bmark(
url=common,
username=users[3].username
)
DBSession.add(c)
DBSession.flush()
d = Bmark(
url=common,
username=users[4].username
)
DBSession.add(d)
DBSession.flush()
ct = BmarkMgr.count(distinct=True)
eq_(4, ct, 'We should have a total of 4: ' + str(ct))
def test_google_import(self):
"""Test that we can upload our google file"""
self.testapp.post('/login',
params={
"login": "******",
"password": "******",
"form.submitted": "Log In",
},
status=302)
session = DBSession()
loc = os.path.dirname(__file__)
del_file = open(os.path.join(loc, 'googlebookmarks.html'))
res = DBSession.execute("SELECT api_key FROM users WHERE username = '******'").fetchone()
API_KEY = res['api_key']
post = {
'api_key': API_KEY,
}
res = self.testapp.post('/admin/import',
params=post,
upload_files=[('import_file',
'googlebookmarks.html',
del_file.read())],
)
eq_(res.status, "302 Found",
msg='Import status is 302 redirect by home, ' + res.status)
session.flush()
# test all the data we want to test after the import
_google_data_test()
def process(self):
"""Given a file, process it"""
soup = BeautifulSoup(self.file_handle)
count = 0
ids = []
for tag in soup.findAll("dt"):
if "javascript:" in str(tag):
continue
# if we have a dd as next sibling, get it's content
if tag.nextSibling and tag.nextSibling.name == "dd":
extended = tag.nextSibling.text
else:
extended = u""
link = tag.a
# Skip any bookmarks with an attribute of PRIVATE.
if link.has_key("PRIVATE"):
continue
import_add_date = float(link["add_date"])
if import_add_date > 9999999999:
# Remove microseconds from the timestamp
import_add_date = import_add_date / 1000
add_date = datetime.fromtimestamp(import_add_date)
try:
bmark = self.save_bookmark(
unicode(link["href"]),
unicode(link.text),
unicode(extended),
u" ".join(unicode(link.get("tags", "")).split(u",")),
dt=add_date,
)
count = count + 1
DBSession.flush()
except InvalidBookmark:
bmark = None
if bmark:
ids.append(bmark.bid)
if count % COMMIT_SIZE == 0:
transaction.commit()
# Commit any that are left since the last commit performed.
transaction.commit()
from bookie.bcelery import tasks
# For each bookmark in this set that we saved, sign up to
# fetch its content.
for bid in ids:
tasks.fetch_bmark_content.delay(bid)
# Start a new transaction for the next grouping.
transaction.begin()
def edit_error(request):
rdict = request.matchdict
params = request.params
post = request.POST
with ReqAuthorize(request, username=rdict['username']):
if 'new' in request.url:
try:
bmark = BmarkMgr.store(post['url'], request.user.username,
post['description'], post['extended'],
post['tags'])
# Assign a task to fetch this pages content and parse it out
# for storage and indexing.
DBSession.flush()
tasks.fetch_bmark_content.delay(bmark.bid)
except InvalidBookmark, exc:
# There was an issue using the supplied data to create a new
# bookmark. Send the data back to the user with the error
# message.
bmark = Bmark(post['url'],
request.user.username,
desc=post['description'],
ext=post['extended'],
tags=post['tags'])
return {
'new': True,
'bmark': bmark,
'message': exc.message,
'user': request.user,
}
else:
def test_google_import(self):
"""Test that we can upload our google file"""
session = DBSession()
loc = os.path.dirname(__file__)
del_file = open(os.path.join(loc, 'googlebookmarks.html'))
post = {
'api_key': 'testapi',
}
res = self.testapp.post(
'/import',
params=post,
upload_files=[('import_file', 'googlebookmarks.html',
del_file.read())],
)
eq_(res.status,
"302 Found",
msg='Import status is 302 redirect by home, ' + res.status)
session.flush()
# test all the data we want to test after the import
_google_data_test()
def _new_user(args):
"""Handle adding a new user to the system.
If you don't include the required info, it will prompt you for it
"""
if not args.username:
args.username = raw_input('username? ')
if not args.email:
args.email = raw_input('email address? ')
if not args.username or not args.email:
raise Exception('Must supply a username and email address')
import transaction
_init_sql(args)
from bookie.models import DBSession
sess = DBSession()
u = User()
u.username = unicode(args.username)
passwd = get_random_word(8)
u.password = passwd
u.email = unicode(args.email)
u.activated = True
u.is_admin = False
u.api_key = User.gen_api_key()
print dict(u)
print passwd
sess.add(u)
sess.flush()
transaction.commit()
def test_tag_with_space(self):
"""Test that we strip out spaces from tags and don't get empty tags
"""
self._get_good_request()
# now build a new version of the request we can check
session = DBSession()
prms = {
'url': u'http://google.com',
'description': u'This is my updated google desc',
'extended': 'updated extended notes about it in full form',
'tags': u'python search updated ',
'api_key': u'testapi',
}
req_params = urllib.urlencode(prms)
self.testapp.get('/delapi/posts/add?' + req_params)
session.flush()
res = Bmark.query.filter(Bmark.url == u'http://google.com').one()
ok_(
len(res.tags) == 3,
'Should only have 3 tags: ' + str([str(t) for t in res.tags]))
for tag in res.tags:
ok_(tag[0] != " ", "Tag should not start with a space")
ok_(tag[-1] != " ", "Tag should not end with a space")
def test_per_user(self):
"""We should only get a pair of results for this single user"""
ct = 5
common = 'testing.com'
user = User()
user.username = gen_random_word(10)
DBSession.add(user)
usercommon = User()
usercommon.username = common
DBSession.add(usercommon)
for i in range(ct - 2):
b = Bmark(url=gen_random_word(12), username=user.username)
DBSession.add(b)
# add in our dupes
c = Bmark(
url=gen_random_word(10),
username=usercommon.username,
)
DBSession.add(c)
DBSession.flush()
d = Bmark(
url=gen_random_word(10),
username=usercommon.username,
)
DBSession.add(d)
DBSession.flush()
ct = BmarkMgr.count(username=usercommon.username)
eq_(2, ct, 'We should have a total of 2: ' + str(ct))
def db_init_bookmark():
"""install the initial bookmark in a new install"""
require('hosts', provided_by=[sample])
require('ini', provided_by=[sample])
parse_ini(env["ini_file"])
from datetime import datetime
import transaction
from bookie.models import initialize_sql
from sqlalchemy import create_engine
engine = create_engine(env.ini.get('app:bookie', 'sqlalchemy.url'))
initialize_sql(engine)
from bookie.models import DBSession
from bookie.models import Bmark
bmark_us = Bmark(u'http://bmark.us',
u'admin',
desc=u"Bookie Website",
ext= u"Bookie Documentation Home",
tags = u"bookmarks")
bmark_us.stored = datetime.now()
bmark_us.updated = datetime.now()
DBSession.add(bmark_us)
DBSession.flush()
transaction.commit()
def test_one_import(self):
"""You should be able to only get one import running at a time"""
self._login_admin()
# Prep the db with 2 other imports ahead of this user's.
# We have to commit these since the request takes place in a new
# session/transaction.
DBSession.add(ImportQueue(username=u'testing',
file_path=u'testing.txt'))
DBSession.add(ImportQueue(username=u'testing2',
file_path=u'testing2.txt'))
DBSession.flush()
transaction.commit()
res = self._upload()
res.follow()
# now let's hit the import page, we shouldn't get a form, but instead a
# message about our import
res = self.app.get('/admin/import')
self.assertTrue('<form' not in res.body, "We shouldn't have a form")
self.assertTrue(
'waiting in the queue' in res.body,
"We want to display a waiting message.")
self.assertTrue(
'2 other imports' in res.body,
"We want to display a count message." + res.body)
def test_tag_with_space(self):
"""Test that we strip out spaces from tags and don't get empty tags
"""
self._get_good_request()
# now build a new version of the request we can check
session = DBSession()
prms = {
'url': u'http://google.com',
'description': u'This is my updated google desc',
'extended': 'updated extended notes about it in full form',
'tags': u'python search updated ',
'api_key': API_KEY
}
req_params = urllib.urlencode(prms)
self.testapp.get('/admin/delapi/posts/add?' + req_params)
session.flush()
res = Bmark.query.filter(Bmark.hash_id == GOOGLE_HASH).one()
ok_(len(res.tags) == 3,
'Should only have 3 tags: ' + str([str(t) for t in res.tags]))
for tag in res.tags:
ok_(tag[0] != " ", "Tag should not start with a space")
ok_(tag[-1] != " ", "Tag should not end with a space")
def upgrade(migrate_engine):
"""By default start all installs out with a bookmark to bmark.us
It's cheesy, but helps with testing to have some base data and is good for
pubbing the links
"""
from datetime import datetime
import transaction
from bookie.models import initialize_sql
initialize_sql(migrate_engine)
from bookie.models import DBSession
from bookie.models import Bmark
bmark_us = Bmark('http://bmark.us',
desc="Bookie Website",
ext="Bookie Documentation Home",
tags="bookmarks")
bmark_us.stored = datetime.now()
bmark_us.updated = datetime.now()
DBSession.add(bmark_us)
DBSession.flush()
transaction.commit()
def process(self):
"""Given a file, process it"""
if self.file_handle.closed:
self.file_handle = open(self.file_handle.name)
parsed = etree.parse(self.file_handle)
count = 0
ids = []
for post in parsed.findall('post'):
if 'javascript:' in post.get('href'):
continue
add_date = dateparser.parse(post.get('time'))
try:
bmark = self.save_bookmark(
post.get('href'),
post.get('description'),
post.get('extended'),
post.get('tag'),
dt=add_date)
count = count + 1
if bmark:
bmark.stored = bmark.stored.replace(tzinfo=None)
DBSession.flush()
except InvalidBookmark, exc:
bmark = None
if bmark:
ids.append(bmark.bid)
if count % COMMIT_SIZE == 0:
transaction.commit()
def upgrade(migrate_engine):
"""By default start all installs out with a bookmark to bmark.us
It's cheesy, but helps with testing to have some base data and is good for
pubbing the links
"""
from datetime import datetime
import transaction
from bookie.models import initialize_sql
initialize_sql(migrate_engine)
from bookie.models import DBSession
from bookie.models import Bmark
bmark_us = Bmark('http://bmark.us',
desc="Bookie Website",
ext= "Bookie Documentation Home",
tags = "bookmarks")
bmark_us.stored = datetime.now()
bmark_us.updated = datetime.now()
DBSession.add(bmark_us)
DBSession.flush()
transaction.commit()
def new_user(username, email):
"""Add new user function, pass username, email
:param username: string of new user
:param email: string of new email
"""
require('hosts', provided_by=[sample])
require('ini', provided_by=[sample])
parse_ini(env["ini_file"])
import transaction
from bookie.models import initialize_sql
initialize_sql(dict(env.ini.items('app:main')))
from bookie.models import DBSession
from bookie.models.auth import get_random_word, User
sess = DBSession()
u = User()
u.username = unicode(username)
passwd = get_random_word(8)
u.password = passwd
u.email = unicode(email)
u.activated = True
u.is_admin = False
u.api_key = User.gen_api_key()
print dict(u)
print passwd
sess.add(u)
sess.flush()
transaction.commit()
def test_update_post(self):
"""Updates allowed over the last one
If you /post/add to an existing bookmark, the new data overrides the
old data
"""
self._get_good_request()
# now build a new version of the request we can check
session = DBSession()
prms = {
"url": u"http://googles.com",
"description": u"This is my updated google desc",
"extended": "updated extended notes about it in full form",
"tags": u"python search updated",
"api_key": API_KEY,
}
req_params = urllib.urlencode(prms)
self.testapp.get("/admin/delapi/posts/add?" + req_params)
session.flush()
res = Bmark.query.filter(Bmark.hash_id == GOOGLE_HASH).one()
ok_("updated" in res.description, "Updated description took: " + res.description)
ok_("updated" in res.extended, "Updated extended took: " + res.extended)
ok_("python" in res.tags, "Found the python tag in the bmark")
ok_("search" in res.tags, "Found the search tag in the bmark")
ok_("updated" in res.tags, "Found the updated tag in the bmark")
def test_unique_ct(self):
"""Verify that our unique count method is working"""
ct = 5
common = 'testing.com'
users = []
for i in range(ct):
user = User()
user.username = gen_random_word(10)
DBSession.add(user)
users.append(user)
for i in range(ct - 2):
b = Bmark(url=gen_random_word(12), username=users[i].username)
DBSession.add(b)
# Add in our dupes
c = Bmark(url=common, username=users[3].username)
DBSession.add(c)
DBSession.flush()
d = Bmark(url=common, username=users[4].username)
DBSession.add(d)
DBSession.flush()
ct = BmarkMgr.count(distinct=True)
eq_(4, ct, 'We should have a total of 4: ' + str(ct))
def test_one_import(self):
"""You should be able to only get one import running at a time"""
self._login_admin()
# Prep the db with 2 other imports ahead of this user's.
# We have to commit these since the request takes place in a new
# session/transaction.
DBSession.add(
ImportQueue(username=u'testing', file_path=u'testing.txt'))
DBSession.add(
ImportQueue(username=u'testing2', file_path=u'testing2.txt'))
DBSession.flush()
transaction.commit()
res = self._upload()
res.follow()
# now let's hit the import page, we shouldn't get a form, but instead a
# message about our import
res = self.app.get('/admin/import')
self.assertTrue('<form' not in res.body, "We shouldn't have a form")
self.assertTrue('waiting in the queue' in res.body,
"We want to display a waiting message.")
self.assertTrue('2 other imports' in res.body,
"We want to display a count message." + res.body)
def new_user(request):
"""Add a new user to the system manually."""
rdict = request.params
u = User()
u.username = unicode(rdict.get('username'))
u.email = unicode(rdict.get('email'))
passwd = get_random_word(8)
u.password = passwd
u.activated = True
u.is_admin = False
u.api_key = User.gen_api_key()
try:
DBSession.add(u)
DBSession.flush()
# We need to return the password since the admin added the user
# manually. This is only time we should have/give the original
# password.
ret = dict(u)
ret['random_pass'] = passwd
return ret
except IntegrityError, exc:
# We might try to add a user that already exists.
LOG.error(exc)
request.response.status_int = 400
return {
'error': 'Bad Request: User exists.',
}
def test_update_post(self):
"""Updates allowed over the last one
If you /post/add to an existing bookmark, the new data overrides the
old data
"""
self._get_good_request()
# now build a new version of the request we can check
session = DBSession()
prms = {
'url': u'http://google.com',
'description': u'This is my updated google desc',
'extended': 'updated extended notes about it in full form',
'tags': u'python search updated',
'api_key': u'testapi',
}
req_params = urllib.urlencode(prms)
self.testapp.get('/delapi/posts/add?' + req_params)
session.flush()
res = Bmark.query.filter(Bmark.url == u'http://google.com').one()
ok_('updated' in res.description,
'Updated description took: ' + res.description)
ok_('updated' in res.extended,
'Updated extended took: ' + res.extended)
ok_('python' in res.tags, 'Found the python tag in the bmark')
ok_('search' in res.tags, 'Found the search tag in the bmark')
ok_('updated' in res.tags, 'Found the updated tag in the bmark')
def tearDown(self):
"""Regular tear down method"""
session = DBSession()
Bmark.query.delete()
Tag.query.delete()
session.execute(bmarks_tags.delete())
session.flush()
transaction.commit()
def tearDown(self):
"""Regular tear down method"""
session = DBSession()
Bmark.query.delete()
Tag.query.delete()
session.execute(bmarks_tags.delete())
session.flush()
transaction.commit()
def tearDown(self):
"""We need to empty the bmarks table on each run"""
testing.tearDown()
session = DBSession()
Bmark.query.delete()
Tag.query.delete()
session.execute(bmarks_tags.delete())
session.flush()
transaction.commit()
def import_bmarks(self):
"""Allow users to upload a bookmark export file for processing"""
username = self.matchdict.get('username')
# if auth fails, it'll raise an HTTPForbidden exception
with ReqAuthorize(self.request):
data = {}
post = self.POST
# We can't let them submit multiple times, check if this user has
# an import in process.
if ImportQueueMgr.get(username=username, status=NEW):
# They have an import, get the information about it and shoot
# to the template.
return {
'existing': True,
'import_stats': ImportQueueMgr.get_details(
username=username)
}
if post:
# we have some posted values
files = post.get('import_file', None)
if files is not None:
storage_dir_tpl = self.settings.get('import_files',
'/tmp/bookie')
storage_dir = storage_dir_tpl.format(
here=self.settings.get('app_root'))
out_fname = store_import_file(storage_dir, username, files)
# Mark the system that there's a pending import that needs
# to be completed
q = ImportQueue(username, unicode(out_fname))
DBSession.add(q)
DBSession.flush()
# Schedule a task to start this import job.
tasks.importer_process.delay(q.id)
return HTTPFound(
location=self.request.route_url('user_import',
username=username))
else:
msg = self.request.session.pop_flash()
if msg:
data['error'] = msg
else:
data['error'] = None
return data
else:
# we need to see if they've got
# just display the form
return {
'existing': False
}
def tearDown(self):
"""Tear down each test"""
testing.tearDown()
session = DBSession()
Bmark.query.delete()
Tag.query.delete()
session.execute(bmarks_tags.delete())
session.flush()
transaction.commit()
def tearDown(self):
"""We need to empty the bmarks table on each run"""
testing.tearDown()
session = DBSession()
Bmark.query.delete()
Tag.query.delete()
session.execute(bmarks_tags.delete())
session.flush()
transaction.commit()
def tearDown(self):
"""Tear down each test"""
testing.tearDown()
session = DBSession()
Bmark.query.delete()
Tag.query.delete()
session.execute(bmarks_tags.delete())
session.flush()
transaction.commit()
def _get_good_request(self, content=False, second_bmark=False):
"""Return the basics for a good add bookmark request"""
session = DBSession()
# the main bookmark, added second to prove popular will sort correctly
prms = {
'url': u'http://google.com',
'description': u'This is my google desc',
'extended': u'And some extended notes about it in full form',
'tags': u'python search',
'api_key': API_KEY,
'username': u'admin',
'inserted_by': u'chrome_ext',
}
# if we want to test the readable fulltext side we want to make sure we
# pass content into the new bookmark
if content:
prms['content'] = u"<p>There's some content in here dude</p>"
# rself.assertEqualparams = urllib.urlencode(prms)
res = self.testapp.post(
'/api/v1/admin/bmark?',
content_type='application/json',
params=json.dumps(prms),
)
if second_bmark:
prms = {
'url': u'http://bmark.us',
'description': u'Bookie',
'extended': u'Exteded notes',
'tags': u'bookmarks',
'api_key': API_KEY,
'username': u'admin',
'inserted_by': u'chrome_ext',
}
# if we want to test the readable fulltext side we want to make
# sure we pass content into the new bookmark
prms['content'] = u"<h1>Second bookmark man</h1>"
# rself.assertEqualparams = urllib.urlencode(prms)
res = self.testapp.post(
'/api/v1/admin/bmark?',
content_type='application/json',
params=json.dumps(prms)
)
session.flush()
transaction.commit()
# Run the celery task for indexing this bookmark.
tasks.reindex_fulltext_allbookmarks(sync=True)
return res
def make_bookmark(user=None):
"""Generate a fake bookmark for testing use."""
bmark = Bmark(random_url(), username="******", desc=random_string(), ext=random_string(), tags=u"bookmarks")
if user:
bmark.username = user.username
bmark.user = user
DBSession.add(bmark)
DBSession.flush()
return bmark
def make_user(username=None):
"""Generate a fake user to test against."""
user = User()
if not username:
username = random_string(10)
user.username = username
DBSession.add(user)
DBSession.flush()
return user
def make_user(username=None):
"""Generate a fake user to test against."""
user = User()
if not username:
username = random_string(10)
user.username = username
DBSession.add(user)
DBSession.flush()
return user
def empty_db():
"""On teardown, remove all the db stuff"""
Bmark.query.delete()
Readable.query.delete()
# we can't remove the toread tag we have from our commands
Tag.query.filter(Tag.name != 'toread').delete()
Hashed.query.delete()
DBSession.execute(bmarks_tags.delete())
DBSession.flush()
transaction.commit()
def _get_good_request(self, content=False, second_bmark=False):
"""Return the basics for a good add bookmark request"""
session = DBSession()
# the main bookmark, added second to prove popular will sort correctly
prms = {
'url': u'http://google.com',
'description': u'This is my google desc',
'extended': u'And some extended notes about it in full form',
'tags': u'python search',
'api_key': API_KEY,
'username': '******',
'inserted_by': 'chrome_ext',
}
# if we want to test the readable fulltext side we want to make sure we
# pass content into the new bookmark
if content:
prms['content'] = "<p>There's some content in here dude</p>"
# req_params = urllib.urlencode(prms)
res = self.testapp.post(
'/api/v1/admin/bmark?',
content_type='application/json',
params=json.dumps(prms),
)
if second_bmark:
prms = {
'url': u'http://bmark.us',
'description': u'Bookie',
'extended': u'Exteded notes',
'tags': u'bookmarks',
'api_key': API_KEY,
'username': '******',
'inserted_by': 'chrome_ext',
}
# if we want to test the readable fulltext side we want to make
# sure we pass content into the new bookmark
prms['content'] = "<h1>Second bookmark man</h1>"
# req_params = urllib.urlencode(prms)
res = self.testapp.post('/api/v1/admin/bmark?',
content_type='application/json',
params=json.dumps(prms))
session.flush()
transaction.commit()
# Run the celery task for indexing this bookmark.
tasks.reindex_fulltext_allbookmarks(sync=True)
return res
def tearDown(self):
"""We need to empty the bmarks table on each run"""
testing.tearDown()
if BOOKIE_TEST_INI == 'test.ini':
SqliteBmarkFT.query.delete()
Bmark.query.delete()
Tag.query.delete()
Hashed.query.delete()
DBSession.execute(bmarks_tags.delete())
DBSession.flush()
transaction.commit()
def empty_db():
"""On teardown, remove all the db stuff"""
if BOOKIE_TEST_INI == 'test.ini':
SqliteBmarkFT.query.delete()
Bmark.query.delete()
# we can't remove the toread tag we have from our commands
Tag.query.filter(Tag.name != 'toread').delete()
Hashed.query.delete()
DBSession.execute(bmarks_tags.delete())
DBSession.flush()
transaction.commit()
def edit_error(request):
rdict = request.matchdict
params = request.params
post = request.POST
with ReqAuthorize(request, username=rdict['username']):
if 'new' in request.url:
try:
try:
bmark = BmarkMgr.get_by_url(post['url'])
except:
bmark = None
if bmark:
return {
'new': False,
'bmark': bmark,
'message': "URL already Exists",
'user': request.user,
}
bmark = BmarkMgr.store(
post['url'],
request.user.username,
post['description'],
post['extended'],
post['tags'])
# Assign a task to fetch this pages content and parse it out
# for storage and indexing.
DBSession.flush()
tasks.fetch_bmark_content.delay(bmark.bid)
except InvalidBookmark, exc:
# There was an issue using the supplied data to create a new
# bookmark. Send the data back to the user with the error
# message.
bmark = Bmark(
post['url'],
request.user.username,
desc=post['description'],
ext=post['extended'],
tags=post['tags'])
return {
'new': True,
'bmark': bmark,
'message': exc.message,
'user': request.user,
}
else:
def tearDown(self):
"""We need to empty the bmarks table on each run"""
testing.tearDown()
# DBSession.execute("TRUNCATE bmarks;")
# DBSession.execute("TRUNCATE fulltext;")
# DBSession.execute("TRUNCATE tags;")
# DBSession.execute("TRUNCATE bmarks_tags;")
SqliteModel.query.delete()
Bmark.query.delete()
Tag.query.delete()
DBSession.execute(bmarks_tags.delete())
DBSession.flush()
transaction.commit()
def tearDown(self):
"""We need to empty the bmarks table on each run"""
testing.tearDown()
# DBSession.execute("TRUNCATE bmarks;")
# DBSession.execute("TRUNCATE fulltext;")
# DBSession.execute("TRUNCATE tags;")
# DBSession.execute("TRUNCATE bmarks_tags;")
SqliteModel.query.delete()
Bmark.query.delete()
Tag.query.delete()
DBSession.execute(bmarks_tags.delete())
DBSession.flush()
transaction.commit()
def process(self):
"""Given a file, process it"""
if self.file_handle.closed:
self.file_handle = open(self.file_handle.name)
self.file_handle.seek(0)
parsed = etree.parse(self.file_handle)
count = 0
ids = []
for post in parsed.findall("post"):
if "javascript:" in post.get("href"):
continue
add_date = dateparser.parse(post.get("time"))
try:
bmark = self.save_bookmark(
unicode(post.get("href")),
unicode(post.get("description")),
unicode(post.get("extended")),
unicode(post.get("tag")),
dt=add_date,
)
count = count + 1
if bmark:
bmark.stored = bmark.stored.replace(tzinfo=None)
DBSession.flush()
except InvalidBookmark:
bmark = None
if bmark:
ids.append(bmark.bid)
if count % COMMIT_SIZE == 0:
transaction.commit()
# Commit any that are left since the last commit performed.
transaction.commit()
from bookie.bcelery import tasks
# For each bookmark in this set that we saved, sign up to
# fetch its content.
for bid in ids:
tasks.fetch_bmark_content.delay(bid)
# Start a new transaction for the next grouping.
transaction.begin()
def make_bookmark(user=None):
"""Generate a fake bookmark for testing use."""
bmark = Bmark(random_url(),
username="******",
desc=random_string(),
ext=random_string(),
tags=u"bookmarks")
if user:
bmark.username = user.username
bmark.user = user
DBSession.add(bmark)
DBSession.flush()
return bmark
def test_activation_delete(self):
"""Make sure removing an activation does not remove a user."""
tst = User()
tst.username = gen_random_word(10)
tst.activation = Activation('signup')
DBSession.add(tst)
DBSession.flush()
DBSession.delete(tst.activation)
users = UserMgr.get_list()
# We still have the admin user as well so the count is two.
eq_(2, len(users),
'We should have a total of 2 users still: ' + str(len(users)))
def empty_db():
"""On teardown, remove all the db stuff"""
DBSession.execute(bmarks_tags.delete())
Readable.query.delete()
Bmark.query.delete()
Tag.query.delete()
# we can't remove the toread tag we have from our commands
Hashed.query.delete()
ImportQueue.query.delete()
DBSession.flush()
transaction.commit()
# Clear the fulltext index as well.
_reset_index()
def _get_good_request(self):
"""Return the basics for a good add bookmark request"""
session = DBSession()
prms = {
'url': u'http://google.com',
'description': u'This is my google desc',
'extended': u'And some extended notes about it in full form',
'tags': u'python search',
'api_key': u'testapi',
}
req_params = urllib.urlencode(prms)
res = self.testapp.get('/delapi/posts/add?' + req_params)
session.flush()
transaction.commit()
return res
def _get_good_request(self):
"""Return the basics for a good add bookmark request"""
session = DBSession()
prms = {
'url': u'http://google.com',
'description': u'This is my google desc',
'extended': u'And some extended notes about it in full form',
'tags': u'python search',
'api_key': u'testapi',
}
req_params = urllib.urlencode(prms)
res = self.testapp.get('/delapi/posts/add?' + req_params)
session.flush()
transaction.commit()
return res
def test_skip_running(self):
"""Verify that if running, it won't get returned again"""
self._login_admin()
res = self._upload()
eq_(res.status, "302 Found",
msg='Import status is 302 redirect by home, ' + res.status)
# now verify that we've got our record
imp = ImportQueueMgr.get_ready()
imp = imp[0]
imp.status=2
DBSession.flush()
imp = ImportQueueMgr.get_ready()
ok_(not imp, 'We should get no results back')
def test_datestimes_set(self):
"""Test that we get the new datetime fields as we work"""
now = datetime.now()
self._get_good_request()
res = Bmark.query.filter(Bmark.url == u'http://google.com').one()
ok_(res.stored >= now,
"Stored time is now or close to now {0}:{1}".format(res.stored, now))
res.url = u"Somethingnew.com"
session = DBSession()
session.flush()
# now hopefully have an updated value
ok_(res.updated >= now,
"Stored time is now or close to now {0}:{1}".format(res.updated, now))
def _get_good_request(self):
"""Return the basics for a good add bookmark request"""
session = DBSession()
prms = {
'url': u'http://google.com',
'description': u'This is my google desc',
'extended': u'And some extended notes about it in full form',
'tags': u'python search',
'api_key': API_KEY,
'content': 'bmark content is the best kind of content man',
}
req_params = urllib.urlencode(prms)
res = self.testapp.post('/api/v1/admin/bmark', params=req_params)
session.flush()
transaction.commit()
return res
def test_skip_running(self):
"""Verify that if running, it won't get returned again"""
self._login_admin()
res = self._upload()
eq_(res.status,
"302 Found",
msg='Import status is 302 redirect by home, ' + res.status)
# now verify that we've got our record
imp = ImportQueueMgr.get_ready()
imp = imp[0]
imp.status = 2
DBSession.flush()
imp = ImportQueueMgr.get_ready()
ok_(not imp, 'We should get no results back')
def test_activation_cascade(self):
"""Removing a user cascades the activations as well."""
tst = User()
tst.username = gen_random_word(10)
tst.activation = Activation('signup')
DBSession.add(tst)
DBSession.flush()
DBSession.delete(tst)
users = UserMgr.get_list()
# We still have the admin user as well so the count is one.
eq_(1, len(users),
'We should have a total of 1 user still: ' + str(len(users)))
activations = DBSession.query(Activation).all()
eq_(0, len(activations), 'There should be no activations left')
def process(self):
"""Given a file, process it"""
soup = BeautifulSoup(self.file_handle)
count = 0
ids = []
for tag in soup.findAll('dt'):
if 'javascript:' in str(tag):
continue
# if we have a dd as next sibling, get it's content
if tag.nextSibling and tag.nextSibling.name == 'dd':
extended = tag.nextSibling.text
else:
extended = ""
link = tag.a
import_add_date = float(link['add_date'])
if import_add_date > 9999999999:
# Remove microseconds from the timestamp
import_add_date = import_add_date / 1000
add_date = datetime.fromtimestamp(import_add_date)
try:
bmark = self.save_bookmark(link['href'],
link.text,
extended,
" ".join(
link.get('tags',
'').split(',')),
dt=add_date)
count = count + 1
DBSession.flush()
except InvalidBookmark, exc:
bmark = None
if bmark:
ids.append(bmark.bid)
if count % COMMIT_SIZE == 0:
transaction.commit()
def test_datestimes_set(self):
"""Test that we get the new datetime fields as we work"""
now = datetime.now()
self._get_good_request()
res = Bmark.query.filter(Bmark.url == u'http://google.com').one()
ok_(
res.stored >= now,
"Stored time is now or close to now {0}:{1}".format(
res.stored, now))
res.url = u"Somethingnew.com"
session = DBSession()
session.flush()
# now hopefully have an updated value
ok_(
res.updated >= now,
"Stored time is now or close to now {0}:{1}".format(
res.updated, now))
def empty_db():
"""On teardown, remove all the db stuff"""
DBSession.execute(bmarks_tags.delete())
Readable.query.delete()
Bmark.query.delete()
StatBookmark.query.delete()
Tag.query.delete()
# we can't remove the toread tag we have from our commands
Hashed.query.delete()
ImportQueue.query.delete()
# Delete the users not admin in the system.
Activation.query.delete()
User.query.filter(User.username != 'admin').delete()
AppLog.query.delete()
DBSession.flush()
transaction.commit()
# Clear the fulltext index as well.
_reset_index()
def _reset_password(args):
"""Reset a user's password"""
if not args.username:
args.username = raw_input('username? ')
if not args.password:
args.password = raw_input('password? ')
if not args.username or not args.password:
raise Exception('Must supply a username and password')
import transaction
_init_sql(args)
from bookie.models import DBSession
sess = DBSession()
u = UserMgr.get(username=unicode(args.username))
u.password = args.password
sess.flush()
transaction.commit()
def _get_good_request(self, new_tags=None):
"""Return the basics for a good add bookmark request"""
session = DBSession()
prms = {
'url': u'http://google.com',
'description': u'This is my google desc SEE',
'extended': u'And some extended notes about it in full form',
'tags': u'python search',
'api_key': API_KEY,
}
if new_tags:
prms['tags'] = new_tags
req_params = urllib.urlencode(prms)
res = self.testapp.post('/api/v1/admin/bmark', params=req_params)
session.flush()
transaction.commit()
tasks.reindex_fulltext_allbookmarks(sync=True)
return res