def test_buildCategoryList(self):
cat_nodes = weblib_cgi._buildCategoryList(store.getWeblib(), '')
self.assertTrue(not cat_nodes[0][0].highlight)
self.assertEqual(self._str_cat_nodes(cat_nodes), [
u'Kremlin',
u'.Русский',
u'.Français',
u'.日本語',
u'.English',
])
# highlight a top level cat
cat_nodes = weblib_cgi._buildCategoryList(store.getWeblib(), 'Kremlin')
self.assertTrue(cat_nodes[0][0].highlight)
self.assertEqual(self._str_cat_nodes(cat_nodes), [
u'Kremlin',
u'.Русский',
u'.Français',
u'.日本語',
u'.English',
])
# highlight a subcat
cat_nodes = weblib_cgi._buildCategoryList(store.getWeblib(), 'English')
self.assertEqual(self._str_cat_nodes(cat_nodes), [
u'Kremlin',
u'.Русский',
u'.Français',
u'.日本語',
u'[',
u'.English',
u']',
])
def test_POST_category_collapse(self):
wlib = store.getWeblib()
self.assertTrue('c' not in wlib.tags.getById(124).flags)
# turn it on
self.checkPathForPattern("/weblib/@124/form?method=POST&category_collapse=on", [
'200 OK',
'setCategoryCollapse @124 True',
])
self.assertTrue('c' in wlib.tags.getById(124).flags)
# turn it off
self.checkPathForPattern("/weblib/@124/form?method=POST&category_collapse=", [
'200 OK',
'setCategoryCollapse @124 False',
])
self.assertTrue('c' not in wlib.tags.getById(124).flags)
# turn it off again
self.checkPathForPattern("/weblib/@124/form?method=POST&category_collapse=", [
'200 OK',
'setCategoryCollapse @124 False',
])
self.assertTrue('c' not in wlib.tags.getById(124).flags)
def queryTag(wfile, req, nameOrId):
wlib = store.getWeblib()
tag = weblib.parseTag(wlib, nameOrId)
tagName = tag and tag.name or ''
# Note: URL is expected to have valid tag parameter. If it turns up
# nothing, one possibility is user has entered an invalid URL
# manually. In that case the query below should turn out empty
# result. We choose not go for the alternative or redirecting user
# to / or inbox because it seems even more confusing.
# category pane
categoryList = _buildCategoryList(wlib, tagName)
# webitem pane
if tag:
webItems = _query_by_tag(wlib, tag)
else:
# TODO: HACK!!!
# Create a fake tag to fill something in the result
# Seems WeblibRenderer() is OK with this
fakeTagNode = WebItemTagNode(nameOrId)
webItems = [fakeTagNode]
renderer = WeblibRenderer(wfile)
renderer.setLayoutParam(None)
renderer.output(
wlib.tags,
tag,
wlib.getDefaultTag(),
categoryList,
webItems)
def test_POST(self):
test_data = 'a\r\n b'
self.checkPathForPattern('/weblib/tag_categorize?category_description=' + urllib.quote(test_data) + '&method=POST', [
'HTTP/1.0 302 Found',
'location: /weblib',
])
self.assertEqual(store.getWeblib().category.getDescription(), test_data)
def test_PUT_char_workout(self):
# test_PUT_input_escape() is a quick basic test
# this one is going to give character escaping a good work out
url = '/weblib/_?' + urllib.urlencode({
'method': 'PUT',
'title': u'€!"#$%&\'()*+,-. /0123456789: ;<=>?@[\\]^_`{|}~'.encode('utf8'),
'description': u'description:€!"#$%&\'()*+,-. /0123456789: ;<=>?@[\\]^_`{|}~\r\n[For testing]'.encode('utf8'),
'url': u'url:€!"#$%&\'()*+,-. /0123456789: ;<=>?@[\\]^_`{|}~'.encode('utf8'),
'tags': u'€!"$% &\'()*-./; =?[\\]^ _`{|}~'.encode('utf8'),
'create_tags': '1'
})
self.checkPathForPattern(url,[
'HTTP/1.0 302 Found',
'location: /updateParent',
])
# one item has added
wlib = store.getWeblib()
tag = wlib.tags.getByName(u'€!"$% &\'()*-./; =?[\\]^ _`{|}~')
self.assert_(tag)
lastId = wlib.webpages._lastId # undocumented
page = wlib.webpages.getById(lastId)
self.assert_(page)
self.assertEqual(page.name, u'€!"#$%&\'()*+,-. /0123456789: ;<=>?@[\\]^_`{|}~')
self.assertEqual(page.description, u'description:€!"#$%&\'()*+,-. /0123456789: ;<=>?@[\\]^_`{|}~\r\n[For testing]')
self.assertEqual(page.url, u'url:€!"#$%&\'()*+,-. /0123456789: ;<=>?@[\\]^_`{|}~')
self.assertEqual(page.tags, [tag])
def test_POST_add_new_tag(self):
wlib = store.getWeblib()
# before
self.assertEqual(len(wlib.webpages.getById(2).tags), 2) # Kremlin, Русский
self.assertEqual(len(wlib.webpages.getById(3).tags), 2) # Kremlin, Français
url = ''.join(['/weblib/multiform',
'?id_list=2%2C3', # 2 - Russian, 3 - French
'&%40122=on&%40122changed=1', # add Français
'&%40121=on&%40121changed=', # Русский unchanged
'&add_tags=aNewTag',
'&method=POST',
'&create_tags=1',
])
self.checkPathForPattern(url, [
'HTTP/1.0 302 Found',
'location: /updateParent',
])
# after
newTag = wlib.tags.getByName('aNewTag')
self.assertTrue(newTag)
self.assertTrue(newTag.id > 124) # new tag should have a higher id
# after
item = wlib.webpages.getById(2)
tagIds = sorted([t.id for t in item.tags])
self.assertEqual(tagIds, [121,122,124,newTag.id]) # Русский, Français, Kremlin, aNewTag
item = wlib.webpages.getById(3)
tagIds = sorted([t.id for t in item.tags])
self.assertEqual(tagIds, [122,124,newTag.id]) # Français, Kremlin, aNewTag
def test_POST_rename(self):
wlib = store.getWeblib()
# before
page = wlib.webpages.getById(2)
tag = wlib.tags.getByName('Kremlin')
self.assertEqual(len(wlib.tags),6)
self.assertEqual(tag.name, 'Kremlin')
self.assertTrue(tag in page.tags)
self.assertTrue(not wlib.tags.getByName('Buckingham'))
self.assertTrue('Buckingham' not in wlib.category.getDescription())
self.checkPathForPattern("/weblib/@124/form?method=POST&name=Buckingham", [
'HTTP/1.0 302 Found',
'location: /updateParent',
])
# after
page = wlib.webpages.getById(2)
tag = wlib.tags.getByName('Buckingham')
self.assertEqual(len(wlib.tags),6)
self.assertEqual(tag.name, 'Buckingham')
self.assertTrue(tag in page.tags)
self.assertTrue(not wlib.tags.getByName('Kremlin'))
self.assertTrue('Buckingham' in wlib.category.getDescription())
def doLaunchURL(wfile, req):
wlib = store.getWeblib()
item = wlib.webpages.getById(req.rid)
if not item:
wfile.write('404 not found\r\n\r\n%s not found' % req.rid)
return
if util.isFileURL(item.url):
# TODO: HACK win32 only??
# TODO: It is dangerous to launch anything could be executable or script
from minds.weblib.win32 import ntfs_util
ntfs_util.launch(item.url)
wfile.write('content-type: text/html\r\n')
wfile.write('Cache-control: no-cache\r\n')
wfile.write('\r\n')
wfile.write('''<html>
<head>
<script>window.close();</script>
</head>
<body>
File launched in separate window. Please close this window.
</body>
</html>
''')
else:
response.redirect(wfile, item.url)
def main(wfile, req):
# this is called from the controller weblib
# if rid is defined, make sure it is valid
if req.rid > 0:
if not store.getWeblib().webpages.getById(req.rid):
wfile.write('404 not found\r\n\r\n')
wfile.write('rid %s not found' % req.rid)
if req.method == 'PUT':
bean = Bean(req)
doPutResource(wfile, req, bean)
elif req.method == 'POST':
wfile.write('404 Method Not Allowed\r\n\r\n')
wfile.write('Use PUT to update the item.')
elif req.method == 'DELETE':
doDeleteResource(wfile, req)
else: # otherwise it is GET
bean = Bean(req)
if req.rid == -1 and bean.oldItem:
# if bookmarklet to an existing item, redirect to the appropiate rid
url = '%s?%s' % (request.rid_url(bean.item.id), req.env.get('QUERY_STRING',''))
response.redirect(wfile, url)
else:
doGetResource(wfile, req, bean)
def _parse_PUT(self, req):
"""
Parse submission from form
method: PUT
parameters: description, title, url, tags, created, modified, lastused
(plus some more auxiliary parameters?)
"""
wlib = store.getWeblib()
if self.oldItem:
# Update an existing item
# Selectively update the field if parameter is supplied.
# That way an API call can send a subset of parameters.
self.item = self.oldItem.__copy__()
if 'title' in req.form: self.item.name = req.param('title')
if 'url' in req.form: self.item.url = req.param('url')
if 'description' in req.form: self.item.description = req.param('description')
if 'created' in req.form: self.item.created = req.param('created')
if 'modified' in req.form: self.item.modified = req.param('modified')
if 'lastused' in req.form: self.item.lastused = req.param('lastused')
if 'tags' in req.form: self._parseTags(req)
else:
# create new item
self.item = weblib.WebPage(
name = req.param('title'),
url = req.param('url'),
description = req.param('description'),
created = req.param('created'),
modified = req.param('modified'),
lastused = req.param('lastused'),
)
self._parseTags(req)
def import_bookmarks(bookmarks):
"""
Import flat collection of bookmarks.
bookmarks is a list of Bookmark.
@return (added, updated)
"""
wlib = store.getWeblib()
update_count = 0
add_count = 0
for b in bookmarks:
page = weblib.WebPage(
name = b.name,
url = b.url,
description = b.description,
created = b.created,
modified = b.modified,
)
page.tags_description = b.tags
isNew, newPage = wlib.putWebPage(page)
if isNew:
update_count += 1
else:
add_count += 1
log.info('Import completed items added=%s updated=%s' % (add_count, update_count))
return (add_count, update_count)
def queryRoot(wfile, req, sort):
wlib = store.getWeblib()
# category pane
categoryList = _buildCategoryList(wlib)
# upgrade_info
upgrade_info = upgrade_checker.pollUpgradeInfo()
# webitem pane
webItems = map(WebItemNode, query_wlib.queryRoot(wlib))
renderer = WeblibRenderer(wfile)
if sort:
renderer.cookie['weblib_sort'] = sort
renderer.setLayoutParam(None)
renderer.output(
'/weblib',
wlib.tags,
None,
wlib.getDefaultTag(),
categoryList,
upgrade_info,
sort,
webItems,
)
def build_category(folder, state, path=None):
""" walk the folder tree recursively and build the category description """
if path == None:
path = []
state.cat_buf.write(' ' * (len(path)-1)) # negative is ok
state.cat_buf.write(folder.name)
state.cat_buf.write('\n')
wlib = store.getWeblib()
path.append(folder)
tags = ','.join([f.name for f in path])
for item in folder.children:
if isinstance(item, Folder):
build_category(item, state, path)
else:
page = weblib.WebPage(
name = item.name,
url = item.url,
description = item.description,
created = item.created,
modified = item.modified,
)
page.tags_description = tags
isNew, newPage = wlib.putWebPage(page)
if isNew:
state.update_count += 1
else:
state.add_count += 1
path.pop()
def main(rfile, wfile, env):
wlib = store.getWeblib()
req = request.WeblibRequest(rfile, env)
log.debug(unicode(req))
path = req.path
if req.rid:
# rid based (note rid maybe -1)
if path and path.startswith('go;'):
doGoResource(wfile, req)
elif path and path == 'url':
doLaunchURL(wfile, req)
elif path and path.startswith('snapshot'):
weblibSnapshot.main(wfile, env, req.method, req.form, req.rid, path)
elif path == 'form':
doWeblibForm(wfile, req)
else:
# show form by default
doWeblibForm(wfile, req)
elif req.tid:
doweblibTagForm(wfile, req)
else:
if path == 'load':
doLoad(wfile, req)
elif path == 'save':
doSave(wfile, req)
else:
doQuery(wfile, req)
def test_POST_remove(self):
wlib = store.getWeblib()
# before
self.assertEqual(len(wlib.webpages.getById(2).tags), 2) # Kremlin, Русский
self.assertEqual(len(wlib.webpages.getById(3).tags), 2) # Kremlin, Français
url = ''.join(['/weblib/multiform',
'?id_list=2%2C3', # 2 - Russian, 3 - French
'&%40122=&%40122changed=1', # remove Français
'&%40121=on&%40121changed=', # Русский unchanged
'&add_tags=inbox',
'&method=POST',
])
self.checkPathForPattern(url, [
'HTTP/1.0 302 Found',
'location: /updateParent',
])
# after
item = wlib.webpages.getById(2)
tagIds = sorted([t.id for t in item.tags])
self.assertEqual(tagIds, [120,121,124]) # inbox, Русский, Kremlin
item = wlib.webpages.getById(3)
tagIds = sorted([t.id for t in item.tags])
self.assertEqual(tagIds, [120,124]) # inbox, Kremlin
def _buildTags(req):
"""
Build base_tag and form_tag. base_tag is the existing tag queried
by tid. form_tag is a new tag instance build from parsing the form
fields plus existing data.
@return (base_tag, form_tag) or None if tid doesn't match any tag.
"""
wlib = store.getWeblib()
base_tag = wlib.tags.getById(req.tid)
if not base_tag:
return None
if req.method == 'GET':
form_tag = base_tag.__copy__()
else:
name = req.param('name') or ' ' # avoid empty string
form_tag = weblib.Tag(
id = req.tid, # should be same
name = name, # from form
description = base_tag.description, # from repository
flags = base_tag.flags, # from repository
)
# HACK! Got around the empty string check in Tag.__init__
if not req.param('name'):
form_tag.name = ''
return base_tag, form_tag
def doDeleteResource(wfile, base_tag):
wlib = store.getWeblib()
log.info('Deleting Tag %s' % unicode(base_tag))
wlib.tag_merge_del(base_tag, None)
response.redirect(wfile, '/updateParent?url=/weblib')
def doDeleteResource(wfile, req):
wlib = store.getWeblib()
item = wlib.webpages.getById(req.rid)
if item:
log.info('Deleting WebPage %s' % unicode(item))
store.getStore().removeItem(item)
response.redirect(wfile, '/updateParent')
def test_PUT_existing(self):
wlib = store.getWeblib()
self.assertEqual(len(wlib.webpages),5)
item = wlib.webpages.getById(1)
self.assertEqual(item.name, 'MindRetrieve - Search Your Personal Web')
# PUT form
self.checkPathForPattern('/weblib/1?' + urllib.urlencode({
'method': 'PUT',
'url': 'http://www.mindretrieve.net/',
'title': 'Test Title',
'description': 'some description',
'created': '1902',
'modified': '1900',
'lastused': '1901',
'tags': 'Kremlin, English',
'nickname': '_nickname_',
}),[
'HTTP/1.0 302 Found',
'location: /updateParent',
])
# one item has changed
self.assertEqual(len(wlib.webpages),5)
item = wlib.webpages.getById(1)
self.assertEqual(item.name , 'Test Title')
self.assertEqual(item.description, 'some description')
self.assertEqual(item.url , 'http://www.mindretrieve.net/')
self.assertEqual(item.created , '1902')
# self.assertEqual(item.modified , '1900')
# self.assertEqual(item.lastused , '1901')
tags = ','.join(sorted(tag.name.lower() for tag in item.tags))
self.assertEqual(tags, 'english,kremlin')
self.assertEqual(item.nickname, '_nickname_')
# PUT partial parameters (only URL)
self.checkPathForPattern('/weblib/1?' + urllib.urlencode({
'method': 'PUT',
'url': 'new url',
}),[
'HTTP/1.0 302 Found',
'location: /updateParent',
])
# url has changed
item = wlib.webpages.getById(1)
self.assertEqual(item.url , 'new url')
# the rest is unchanged
self.assertEqual(item.name , 'Test Title')
self.assertEqual(item.description, 'some description')
self.assertEqual(item.created , '1902')
# self.assertEqual(item.modified , '1900')
# self.assertEqual(item.lastused , '1901')
tags = ','.join(sorted(tag.name.lower() for tag in item.tags))
self.assertEqual(tags, 'english,kremlin')
self.assertEqual(item.nickname, '_nickname_')
def doDelete(wfile, req):
wlib = store.getWeblib()
entries = _buildEntries(req)
for item in entries:
try:
log.debug('Delete web page: %s', unicode(item))
store.getStore().removeItem(item)
except:
log.exception('Unable to delete: %s', unicode(item))
response.redirect(wfile, '/updateParent')
def doPost(wfile, req):
wlib = store.getWeblib()
# TODO: parse and check for error?
text = req.param('category_description')
# setDescription() has a quick & dirty way to get rid of illegal characters
wlib.category.setDescription(text)
response.redirect(wfile, '/weblib')
def render(self, node, bean, tags):
item = bean.item
wlib = store.getWeblib()
if item.id == -1:
node.form_title.content %= 'Add Entry'
node.edit_header.omit()
else:
node.form_title.content %= 'Edit Entry'
node.add_header.omit()
form = node.form
id = item.id < 0 and '_' or str(item.id)
form.atts['action'] = request.rid_url(id)
if bean.errors:
escaped_errors = map(saxutils.escape, bean.errors)
form.error.message.raw = '<br />'.join(escaped_errors)
else:
form.error.omit()
if item:
form.name .atts['value'] = item.name
form.created .atts['value'] = item.created
form.url .atts['value'] = item.url
form.description.content = item.description
form.tags .atts['value'] = bean.item.tags_description
form.nickname .atts['value'] = item.nickname
if weblib_util.isFileURL(item.url):
scheme, netloc, url_path, _, _, _ = urlparse.urlparse(item.url)
pathname = weblib_util.nt_url2pathname(url_path)
form.url_link.atts['href'] = '/weblib/%s/url#%s' % (item.id, item.url)
form.filename.content = pathname
else:
form.url_link.atts['href'] = item.url
form.filename.omit()
# if item.modified:
# form.modified_txt.content = item.modified
# if item.fetched:
# form.snapshot_txt.content = item.fetched
tags_strings = [u' "%s"' % response.jsEscapeString(unicode(tag)) for tag in tags]
node.form.tags_array.raw = node.form.tags_array.raw % ',\n'.join(tags_strings)
new_tags = bean.newTags and u', '.join(bean.newTags) or ''
encoded_tags = response.jsEscapeString(new_tags)
node.form.new_tags_js_var.raw = node.form.new_tags_js_var.raw % encoded_tags
# weblibForm get invoked from CGI weblib.py
#if __name__ == "__main__":
# main(sys.stdin, sys.stdout, os.environ)
def doGoResource(wfile, req):
# the path are really for user's information only.
# rid alone determines where to go.
wlib = store.getWeblib()
item = wlib.webpages.getById(req.rid)
if not item:
wfile.write('404 not found\r\n\r\n%s not found' % req.rid)
return
item = wlib.visit(item)
response.redirect(wfile, item.url)
def __init__(self, tagOrName):
self.tagName = unicode(tagOrName)
if isinstance(tagOrName, weblib.Tag):
self.id = tagOrName.id
self.tag = tagOrName
else:
wlib = store.getWeblib()
self.tag = wlib.tags.getByName(tagOrName)
self.id = self.tag and self.tag.id or -1
self.level = 0
self.comma = False
self.highlight = False
def import_tree(root_folder):
state = WalkState()
build_category(root_folder, state)
# append cat_buf to category description
wlib = store.getWeblib()
new_cat = wlib.category.getDescription() + state.cat_buf.getvalue()
wlib.category.setDescription(new_cat)
log.info('Import completed items added=%s updated=%s' % (state.add_count, state.update_count))
return (state.add_count, state.update_count)
def test_POST_illegal(self):
test_data = '@bad1\r\n#bad2\r\ngood'
self.checkPathForPattern('/weblib/tag_categorize?category_description=' + urllib.quote(test_data) + '&method=POST', [
'HTTP/1.0 302 Found',
'location: /weblib',
])
# no error. But illegal characters got converted to '?'
wlib = store.getWeblib()
self.assertEqual(wlib.category.getDescription(), '.bad1\r\n.bad2\r\ngood')
self.assert_(wlib.tags.getByName('.bad1'))
self.assert_(wlib.tags.getByName('.bad2'))
self.assert_(wlib.tags.getByName('good'))
def _tag_from_cookie(req):
tid = req.cookie.get('weblib_tag')
if not tid:
return None
tid = tid.value
try:
if not tid.startswith('@'):
return None
tid = int(tid[1:])
except ValueError:
return None
wlib = store.getWeblib()
return wlib.tags.getById(tid)
def main(rfile, wfile, env):
wlib = store.getWeblib()
req = request.WeblibRequest(rfile, env)
log.debug(unicode(req))
path = req.path
if req.rid:
# rid based (note rid maybe -1)
if path and path.startswith('go;'):
doGoResource(wfile, req)
elif path and path == 'url':
doLaunchURL(wfile, req)
elif path and path.startswith('snapshot'):
weblibSnapshot.main(wfile, env, req.method, req.form, req.rid, path)
elif path == 'form':
doWeblibForm(wfile, req)
else:
# show form by default
doWeblibForm(wfile, req)
elif req.tid:
doweblibTagForm(wfile, req)
else:
if path == 'load':
doLoad(wfile, req)
elif path == 'save':
doSave(wfile, req)
else:
# query
querytxt = req.param('query')
tag = req.param('tag')
# redirect to default tag (if it is defined)
if not ('tag' in req.form or querytxt):
dt = wlib.getDefaultTag()
if dt:
url = request.tag_url([dt])
response.redirect(wfile, url)
return
# if req.param('action') == 'cancel':
# response.redirect(wfile, request.WEBLIB_URL)
if tag:
queryTag(wfile, req, tag)
elif querytxt:
queryWebLib(wfile, req, tag, querytxt)
else:
queryRoot(wfile, req)
def doPostCategoryCollapse(wfile, req):
wlib = store.getWeblib()
flag = req.param('category_collapse').lower() == 'on'
log.debug('setCategoryCollapse @%s %s' % (req.tid, flag))
wlib.setCategoryCollapse(req.tid, flag)
# response for debug only
wfile.write('content-type: text/plain\r\n')
wfile.write('cache-control: no-cache\r\n')
wfile.write('\r\n')
wfile.write('setCategoryCollapse @%s %s' % (req.tid, flag))
def queryRoot(wfile, req):
wlib = store.getWeblib()
# category pane
categoryList = _buildCategoryList(wlib)
# webitem pane
webItems = map(WebItemNode, query_wlib.queryRoot(wlib))
renderer = WeblibRenderer(wfile)
renderer.setLayoutParam(None)
renderer.output(
wlib.tags,
None,
wlib.getDefaultTag(),
categoryList,
webItems)
