def filter_tiddlers_from_bag(bag, filters):
"""
Return the list of tiddlers resulting from filtering
bag by filter. The filter is a string that will be
parsed to a list of filters.
"""
store = bag.store
# XXX isinstance considered harmful
if isinstance(filters, basestring):
filters, leftovers = parse_for_filters(filters)
if store:
return recursive_filter(filters, get_tiddlers_from_bag(bag))
else:
return recursive_filter(filters, bag.list_tiddlers())
def generate_html(self, plugin_name, plugins, base_tiddlers):
"""
recurse through the template stack and generate the HTML on the way back out.
"""
plugin_html = {}
if isinstance(plugins, dict):
for template in plugins:
recipe_data = plugins[template].split('?', 1)
recipe = _get_recipe(self.environ, recipe_data[0])
plugin_tiddlers = control.get_tiddlers_from_recipe(recipe, self.environ)
if len(recipe_data) == 2:
filters = parse_for_filters(recipe_data[1])[0]
plugin_tiddlers = recursive_filter(filters, plugin_tiddlers)
try:
plugin_plugins = self.environ['tiddlyweb.config']['tw_pages_serializers'][template]['plugins']
plugin_html[template] = self.generate_html(template, plugin_plugins, plugin_tiddlers)
except KeyError:
#there is no plugin by that name, so try a (non TiddlyWebPages) serializer instead
plugin_html[template] = self.pass_through_external_serializer(template, plugin_tiddlers)
server_prefix = self.get_server_prefix()
try:
self.template.set_template(plugin_name)
content = self.template.render(tiddlers=base_tiddlers, extra=plugin_html, prefix=server_prefix, query=self.query, root_vars=self.environ['tiddlyweb.recipe_template'])
except KeyError:
content = self.pass_through_external_serializer(plugin_name, base_tiddlers)
return content
def send_tiddlers(environ, start_response, tiddlers=None):
"""
Output the tiddlers contained in the provided
Tiddlers collection in a Negotiated representation.
Often, but not always, a wiki.
"""
last_modified = None
etag = None
download = environ['tiddlyweb.query'].get('download', [None])[0]
filters = environ['tiddlyweb.filters']
store = environ['tiddlyweb.store']
if filters:
candidate_tiddlers = Tiddlers(store=store)
try:
candidate_tiddlers.title = tiddlers.title
candidate_tiddlers.is_search = tiddlers.is_search
candidate_tiddlers.is_revisions = tiddlers.is_revisions
except AttributeError:
pass
try:
for tiddler in recursive_filter(filters, tiddlers):
recipe = tiddler.recipe
if not tiddler.store:
tiddler = store.get(tiddler)
if recipe:
tiddler.recipe = recipe
candidate_tiddlers.add(tiddler)
except FilterError, exc:
raise HTTP400('malformed filter: %s' % exc)
def match_related_articles(title, matches, tiddlers):
def empty_generator(): return ;yield 'never'
tiddlers = [tiddler for tiddler in tiddlers]
try:
source_tiddler = recursive_filter(parse_for_filters('select=title:%s' % title)[0], tiddlers).next()
except StopIteration:
#nothing to match on, so return an empty generator
return empty_generator()
sort_set = []
for tiddler in tiddlers:
count = 0
for match in matches:
try:
source = getattr(source_tiddler, match)
test = getattr(tiddler, match)
test_func = ATTRIBUTE_SELECTOR.get(match, compare_text)
count += test_func(source, test)
except AttributeError:
count += compare_fields(source_tiddler.fields, tiddler.fields, match)
if count > 0 and source_tiddler.title != tiddler.title:
sort_set.append([tiddler,count])
def sort_function(a,b): return cmp(b[1],a[1])
sort_set.sort(sort_function)
result = (tiddler_set[0] for tiddler_set in sort_set)
return result
def send_tiddlers(environ, start_response, tiddlers=None):
"""
Output the tiddlers contained in the provided
Tiddlers collection in a Negotiated representation.
Often, but not always, a wiki.
"""
last_modified = None
etag = None
download = environ['tiddlyweb.query'].get('download', [None])[0]
filters = environ['tiddlyweb.filters']
store = environ['tiddlyweb.store']
if tiddlers.store is None and not filters:
logging.warn('Incoming tiddlers no store set %s', inspect.stack()[1])
if filters:
candidate_tiddlers = Tiddlers(store=store)
try:
candidate_tiddlers.title = tiddlers.title
candidate_tiddlers.link = tiddlers.link
candidate_tiddlers.is_search = tiddlers.is_search
candidate_tiddlers.is_revisions = tiddlers.is_revisions
except AttributeError:
pass
try:
for tiddler in recursive_filter(filters, tiddlers):
candidate_tiddlers.add(tiddler)
except FilterError, exc:
raise HTTP400('malformed filter: %s' % exc)
def test_parsing():
"""
Incomplete testing of parsing the filter string
as part of the query string parsing, leaving the rest
of the query string intact.
"""
string = 'slag=absolute;foo=;select=tag:systemConfig;select=tag:blog;fat=1;sort=-modified;limit=0,10;select=title:monkey'
filters, leftovers = parse_for_filters(string)
assert len(filters) == 5
assert leftovers == 'slag=absolute;foo=;fat=1'
text_filters = []
for filter, text, environ in filters:
text_filters.append(text)
assert len(text_filters) == 5
assert text_filters[0][1] == 'tag:systemConfig'
assert text_filters[1][1] == 'tag:blog'
assert text_filters[2][1] == '-modified'
assert text_filters[3][1] == '0,10'
tiddlers = [Tiddler('a'), Tiddler('monkey')]
tiddlers[1].tags = ['systemConfig', 'blog']
tiddlers = list(recursive_filter(filters, tiddlers))
assert len(tiddlers) == 1
assert tiddlers[0].title == 'monkey'
def _filter_readable(environ, entities, filters):
"""
Traverse entities to get those that are readable
and those that pass the filter.
XXX: There is a bug here, depending on how
filters are to be interpreted: If limit is used
it is being calculated before the readability
of the entities is checked.
"""
store = environ['tiddlyweb.store']
def _load(entities):
for entity in entities:
if hasattr(entity, 'store') and entity.store:
yield entity
else:
entity = store.get(entity)
yield entity
kept_entities = Container()
try:
for entity in recursive_filter(filters, _load(entities)):
try:
entity.policy.allows(environ['tiddlyweb.usersign'], 'read')
kept_entities.add(entity)
except(UserRequiredError, ForbiddenError):
pass
except AttributeError as exc:
raise FilterError('malformed filter: %s' % exc)
return kept_entities
def _filter_readable(environ, entities, filters):
"""
Traverse entities to get those that are readable
and those that pass the filter.
XXX: There is a bug here, depending on how
filters are to be interpreted: If limit is used
it is being calculated before the readability
of the entities is checked.
"""
store = environ['tiddlyweb.store']
def _load(entities):
for entity in entities:
if hasattr(entity, 'store') and entity.store:
yield entity
else:
entity = store.get(entity)
yield entity
kept_entities = Container()
try:
for entity in recursive_filter(filters, _load(entities)):
try:
entity.policy.allows(environ['tiddlyweb.usersign'], 'read')
kept_entities.add(entity)
except (UserRequiredError, ForbiddenError):
pass
except AttributeError as exc:
raise FilterError('malformed filter: %s' % exc)
return kept_entities
def send_tiddlers(environ, start_response, tiddlers=None):
"""
Output the tiddlers contained in the provided
Tiddlers collection in a Negotiated representation.
Often, but not always, a wiki.
"""
last_modified = None
etag = None
download = environ['tiddlyweb.query'].get('download', [None])[0]
filters = environ['tiddlyweb.filters']
store = environ['tiddlyweb.store']
if filters:
candidate_tiddlers = Tiddlers(store=store)
try:
candidate_tiddlers.title = tiddlers.title
candidate_tiddlers.is_search = tiddlers.is_search
candidate_tiddlers.is_revisions = tiddlers.is_revisions
except AttributeError:
pass
try:
for tiddler in recursive_filter(filters, tiddlers):
recipe = tiddler.recipe
if not tiddler.store:
tiddler = store.get(tiddler)
if recipe:
tiddler.recipe = recipe
candidate_tiddlers.add(tiddler)
except FilterError, exc:
raise HTTP400('malformed filter: %s' % exc)
def send_tiddlers(environ, start_response, tiddlers=None):
"""
Output the tiddlers contained in the provided
Tiddlers collection in a Negotiated representation.
Often, but not always, a wiki.
"""
last_modified = None
etag = None
download = environ['tiddlyweb.query'].get('download', [None])[0]
filters = environ['tiddlyweb.filters']
store = environ['tiddlyweb.store']
if tiddlers.store is None and not filters:
logging.warn('Incoming tiddlers no store set %s', inspect.stack()[1])
if filters:
candidate_tiddlers = Tiddlers(store=store)
try:
candidate_tiddlers.title = tiddlers.title
candidate_tiddlers.link = tiddlers.link
candidate_tiddlers.is_search = tiddlers.is_search
candidate_tiddlers.is_revisions = tiddlers.is_revisions
except AttributeError:
pass
try:
for tiddler in recursive_filter(filters, tiddlers):
candidate_tiddlers.add(tiddler)
except FilterError, exc:
raise HTTP400('malformed filter: %s' % exc)
def list_tiddlers(self, tiddlers):
"""
Turn the contents of a Tiddlers into an Atom Feed.
"""
authors = set()
try:
from tiddlyweb.model.collections import Tiddlers
config = self.environ['tiddlyweb.config']
default_filter = config['atom.default_filter']
filters, _ = parse_for_filters(default_filter, self.environ)
new_tiddlers = Tiddlers()
new_tiddlers.is_search = tiddlers.is_search
new_tiddlers.is_revisions = tiddlers.is_revisions
new_tiddlers.bag = tiddlers.bag
new_tiddlers.recipe = tiddlers.recipe
new_tiddlers.link = tiddlers.link
for tiddler in recursive_filter(filters, tiddlers):
new_tiddlers.add(tiddler)
authors.add(tiddler.modifier)
new_tiddlers.title = tiddlers.title
new_tiddlers.is_search = tiddlers.is_search
new_tiddlers.is_revisions = tiddlers.is_revisions
tiddlers = new_tiddlers
except (KeyError, ImportError):
pass
author_name = None
author_link = None
author_avatar = None
if len(authors) == 1:
author_name = authors.pop()
author_link = self._get_author_link(author_name)
author_avatar = self._get_author_avatar(author_name)
hub = self.environ.get('tiddlyweb.config', {}).get('atom.hub', None)
if tiddlers.link:
link = tiddlers.link
else:
link = self._current_url()
if not link.startswith('http'):
link = u'%s%s' % (self._host_url(), link)
feed = AtomFeed(link=link,
language=u'en',
hub=hub,
author_name=author_name,
author_link=author_link,
author_avatar=author_avatar,
title=tiddlers.title,
description=tiddlers.title)
for tiddler in tiddlers:
self._add_tiddler_to_feed(feed, tiddler)
# can we avoid sending utf-8 and let the wrapper handle it?
return feed.writeString('utf-8')
def _filter_tiddlers_from_bag(bag, filters, environ=None):
"""
Return the list of tiddlers resulting from filtering bag by filter.
The filter is a string that will be parsed to a list of filters.
"""
indexable = bag
if isinstance(filters, basestring):
filters, _ = parse_for_filters(filters, environ)
return recursive_filter(filters, bag.store.list_bag_tiddlers(bag), indexable=indexable)
def filter_tiddlers(tiddlers, filters, environ=None):
"""
Return a generator of tiddlers resulting from filtering the provided
iterator of tiddlers by the provided filters.
If filters is a string, it will be parsed for filters.
"""
if isinstance(filters, basestring):
filters, _ = parse_for_filters(filters, environ)
return recursive_filter(filters, tiddlers)
def filter_tiddlers(tiddlers, filters, environ=None):
"""
Return a generator of tiddlers resulting from filtering the provided
iterator of tiddlers by the provided filters.
If filters is a string, it will be parsed for filters.
"""
if isinstance(filters, basestring):
filters, _ = parse_for_filters(filters, environ)
return recursive_filter(filters, tiddlers)
def filter_tiddlers_from_bag(bag, filters):
"""
Return the list of tiddlers resulting from filtering
bag by filter. The filter is a string that will be
parsed to a list of filters.
"""
store = bag.store
if bag.tmpbag or bag.revbag or bag.searchbag:
indexable = False
else:
indexable = bag
# XXX isinstance considered harmful
if isinstance(filters, basestring):
filters, leftovers = parse_for_filters(filters)
if store:
return recursive_filter(filters, get_tiddlers_from_bag(bag), indexable=indexable)
else:
return recursive_filter(filters, bag.gen_tiddlers(), indexable=indexable)
def _filter_tiddlers_from_bag(bag, filters, environ=None):
"""
Return the list of tiddlers resulting from filtering bag by filter.
The filter is a string that will be parsed to a list of filters.
"""
indexable = bag
if isinstance(filters, basestring):
filters, _ = parse_for_filters(filters, environ)
return recursive_filter(filters, bag.store.list_bag_tiddlers(bag),
indexable=indexable)
def test_parsing():
"""
Incomplete testing of parsing the filter string
as part of the query string parsing, leaving the rest
of the query string intact.
"""
string = 'slag=absolute;foo=;select=tag:systemConfig;select=tag:blog;fat=1;sort=-modified;limit=0,10;select=title:monkey'
filters, leftovers = parse_for_filters(string)
assert len(filters) == 5
assert leftovers == 'slag=absolute;foo=;fat=1'
tiddlers = [Tiddler('a'), Tiddler('monkey')]
tiddlers[1].tags = ['systemConfig', 'blog']
tiddlers = recursive_filter(filters, tiddlers)
assert len(tiddlers) == 1
assert tiddlers[0].title == 'monkey'
def geo_near_tiddlers(lat, lng, radius, tiddlers, units="kms"):
# create bounding box around
if units == "miles":
degrees = radius / SINGLE_DEG_AT_ZERO_ZERO_IN_MILES
elif units == "kms":
degrees = radius / SINGLE_DEG_AT_ZERO_ZERO_IN_KMS
else:
raise "unknown unit type: please use kms or miles"
lat1 = lat - degrees
lat2 = lat + degrees
lng1 = lng - degrees
lng2 = lng + degrees
filter_string = "select=field:%s&select=field:%s&select=%s:>%s&select=%s:<%s&select=%s:>%s&select=%s:<%s" % (
LONGITUDE_FIELD,
LATITUDE_FIELD,
LATITUDE_FIELD,
lat1,
LATITUDE_FIELD,
lat2,
LONGITUDE_FIELD,
lng1,
LONGITUDE_FIELD,
lng2,
)
logging.debug("filter string lt gt %s" % filter_string)
filtered_sample = list(filters.recursive_filter(filters.parse_for_filters(filter_string)[0], tiddlers))
near_tiddlers = []
for tiddler in filtered_sample:
testlat = tiddler.fields[LATITUDE_FIELD]
testlng = tiddler.fields[LONGITUDE_FIELD]
if testlat and testlng:
try:
testlat = float(testlat)
testlng = float(testlng)
isNear = geoproximity(lat, lng, radius, testlat, testlng, units=units)
if isNear[0]:
tiddler.fields["_geo.proximity"] = "%.2f" % isNear[1]
yield tiddler
except ValueError:
# ignore tiddlers which have an empty string for this value
pass
return
def _filter_tiddlers(filters, store, tiddlers):
"""
Filter the tiddlers by filters provided by the enviornment.
"""
candidate_tiddlers = Tiddlers(store=store)
try:
candidate_tiddlers.title = tiddlers.title
candidate_tiddlers.link = tiddlers.link
candidate_tiddlers.is_search = tiddlers.is_search
candidate_tiddlers.is_revisions = tiddlers.is_revisions
candidate_tiddlers.bag = tiddlers.bag
candidate_tiddlers.recipe = tiddlers.recipe
except AttributeError:
pass
try:
for tiddler in recursive_filter(filters, tiddlers):
candidate_tiddlers.add(tiddler)
except FilterError, exc:
raise HTTP400('malformed filter: %s' % exc)
def _filter_tiddlers(filters, store, tiddlers):
"""
Filter the tiddlers by filters provided by the environment.
"""
candidate_tiddlers = Tiddlers(store=store)
try:
candidate_tiddlers.title = tiddlers.title
candidate_tiddlers.link = tiddlers.link
candidate_tiddlers.is_search = tiddlers.is_search
candidate_tiddlers.is_revisions = tiddlers.is_revisions
candidate_tiddlers.bag = tiddlers.bag
candidate_tiddlers.recipe = tiddlers.recipe
except AttributeError:
pass
try:
for tiddler in recursive_filter(filters, tiddlers):
candidate_tiddlers.add(tiddler)
except FilterError as exc:
raise HTTP400('malformed filter: %s' % exc)
return candidate_tiddlers
def generate_html(self, plugin_name, plugins, base_tiddlers):
"""
recurse through the template stack and generate the HTML on the way back out.
"""
plugin_html = {}
if isinstance(plugins, dict):
for template in plugins:
recipe_data = plugins[template].split('?', 1)
recipe = _get_recipe(self.environ, recipe_data[0])
plugin_tiddlers = control.get_tiddlers_from_recipe(
recipe, self.environ)
if len(recipe_data) == 2:
filters = parse_for_filters(recipe_data[1])[0]
plugin_tiddlers = recursive_filter(filters,
plugin_tiddlers)
try:
plugin_plugins = self.environ['tiddlyweb.config'][
'tw_pages_serializers'][template]['plugins']
plugin_html[template] = self.generate_html(
template, plugin_plugins, plugin_tiddlers)
except KeyError:
#there is no plugin by that name, so try a (non TiddlyWebPages) serializer instead
plugin_html[
template] = self.pass_through_external_serializer(
template, plugin_tiddlers)
server_prefix = self.get_server_prefix()
try:
self.template.set_template(plugin_name)
content = self.template.render(
tiddlers=base_tiddlers,
extra=plugin_html,
prefix=server_prefix,
query=self.query,
root_vars=self.environ['tiddlyweb.recipe_template'])
except KeyError:
content = self.pass_through_external_serializer(
plugin_name, base_tiddlers)
return content
def match_related_articles(title, matches, tiddlers):
def empty_generator():
return
yield 'never'
tiddlers = [tiddler for tiddler in tiddlers]
try:
source_tiddler = recursive_filter(
parse_for_filters('select=title:%s' % title)[0], tiddlers).next()
except StopIteration:
#nothing to match on, so return an empty generator
return empty_generator()
sort_set = []
for tiddler in tiddlers:
count = 0
for match in matches:
try:
source = getattr(source_tiddler, match)
test = getattr(tiddler, match)
test_func = ATTRIBUTE_SELECTOR.get(match, compare_text)
count += test_func(source, test)
except AttributeError:
count += compare_fields(source_tiddler.fields, tiddler.fields,
match)
if count > 0 and source_tiddler.title != tiddler.title:
sort_set.append([tiddler, count])
def sort_function(a, b):
return cmp(b[1], a[1])
sort_set.sort(sort_function)
result = (tiddler_set[0] for tiddler_set in sort_set)
return result
def filter(filter_string, tiddlers):
return recursive_filter(parse_for_filters(filter_string)[0], tiddlers)
def filter(environ, filter_string, entities):
return recursive_filter(
parse_for_filters(filter_string, environ)[0], entities)
def filter(filter_string, tiddlers):
return recursive_filter(parse_for_filters(filter_string)[0], tiddlers)
def test_exception():
filter, _ = parse_for_filters('limit=-1,2')
with py.test.raises(FilterError):
recursive_filter(filter, tiddlers)
def filter(environ, filter_string, entities):
return recursive_filter(parse_for_filters(
filter_string, environ)[0], entities)
