def get_search_query(self, user, permission, class_id=None):
# Special case: admins can see everything
user_groups, is_admin = self._get_user_groups(user)
if is_admin:
return AllQuery()
# 1. Back-office access rules
rules_query = OrQuery()
for rule in self.get_resources():
if rule.get_value('permission') != permission:
continue
if rule.get_value('group') not in user_groups:
continue
if permission == 'add':
r_format = rule.get_value('search_format')
if class_id and r_format and class_id != r_format:
continue
rules_query.append(rule.get_search_query())
# Case: anonymous
if not user:
return AndQuery(rules_query, PhraseQuery('share', 'everybody'))
# Case: authenticated
share_query = OrQuery(*[PhraseQuery('share', x) for x in user_groups])
share_query.append(PhraseQuery('share', str(user.abspath)))
query = AndQuery(rules_query, share_query)
if permission != 'share':
return OrQuery(PhraseQuery('owner', str(user.abspath)), query)
return query
def get_items_query(self, resource, context):
# Search in subtree
query = get_base_path_query(resource.abspath, max_depth=self.depth)
# Base classes
base_classes = self.base_classes
if base_classes is not None:
base_classes_query = OrQuery(
*[PhraseQuery('base_classes', x) for x in base_classes])
query = AndQuery(query, base_classes_query)
# Exclude non-content
if self.search_content_only(resource, context) is True:
query = AndQuery(query, PhraseQuery('is_content', True))
return query
def get_search_query(self):
permission = self.get_value('permission')
if permission == 'add':
names = ['path']
else:
names = ['path', 'format']
# Query
query = AndQuery()
for name in names:
field_name = 'search_%s' % name
field = self.get_field(field_name)
value = field.get_value(self, field_name)
if not value:
continue
if name == 'path':
depth = self.get_value('search_path_depth')
depth = None if depth == '*' else int(depth)
subquery = get_base_path_query(value, 0, depth)
elif field.multiple:
err = "access rules don't yet support multiple fields"
raise NotImplementedError, err
else:
subquery = PhraseQuery(name, value)
query.append(subquery)
# Ok
return query
def get_options(self):
context = get_context()
resource = context.resource
view = context.view
# 1. Build the query of all objects to search
query = get_base_path_query(resource.abspath)
if view.search_content_only(resource, context) is True:
content_query = PhraseQuery('is_content', True)
query = AndQuery(query, content_query)
# 2. Compute children_formats
children_formats = set()
for child in context.search(query).get_documents():
children_formats.add(child.format)
# 3. Do not show two options with the same title
formats = {}
for type in children_formats:
cls = context.database.get_resource_class(type)
title = cls.class_title.gettext()
formats.setdefault(title, []).append(type)
# 4. Build the namespace
types = []
for title, type in formats.items():
type = ','.join(type)
types.append({'name': type, 'value': title})
types.sort(key=lambda x: x['value'].lower())
return types
def test_AndQuery_with_AllQuery(self):
query1 = AndQuery(PhraseQuery('data', u'lion'), AllQuery())
results1 = self.database.search(query1)
# Must be equal to the same query without AllQuery
query2 = PhraseQuery('data', u'lion')
results2 = self.database.search(query2)
self.assertEqual(len(results1), len(results2))
def __init__(self, cls, database, user=None, username=None, email=None, commit_at_exit=True):
# Check if context is not already locked
if get_context() != None:
raise ValueError('Cannot acquire context. Already locked.')
# Acquire lock on database
DBSEM.acquire()
from server import get_server
self.context = cls()
self.context.database = database
self.context.server = get_server()
self.commit_at_exit = commit_at_exit
# Set context
set_context(self.context)
# Get user by user
if email:
query = AndQuery(
PhraseQuery('format', 'user'),
PhraseQuery('email', email),
)
search = database.search(query)
if search:
user = search.get_resources(size=1).next()
# Get user by username
if username:
user = self.context.root.get_user(username)
# Log user
if user:
self.context.login(user)
def has_permission(self, user, permission, resource, class_id=None):
# The query
query = AndQuery(self.get_search_query(user, permission, class_id),
PhraseQuery('abspath', str(resource.abspath)))
# Search
results = get_context().search(query)
return len(results) > 0
def test_multilingual_search(self):
with Database('demo.hforge.org', 19500, 20500) as database:
with database.init_context():
root = root = database.get_resource('/')
container = root.make_resource('test-multilingual', Folder)
# Create N resources
for i in range(0, 20):
kw = {'title': {'fr': u'Bonjour', 'en': u'Hello'}}
container.make_resource(str(i), Text, **kw)
database.save_changes()
# Check if resource exists
query = AndQuery(get_base_path_query('/test-multilingual'),
PhraseQuery('format', 'text'))
search = database.search(query)
self.assertEqual(len(search), 20)
# Check if resource exists
query = AndQuery(
PhraseQuery('format', 'text'),
PhraseQuery('title', u'Hello'),
get_base_path_query('/test-multilingual'),
)
search = database.search(query)
self.assertEqual(len(search), 20)
query = AndQuery(
PhraseQuery('format', 'text'),
PhraseQuery('title_en', u'Hello'),
get_base_path_query('/test-multilingual'),
)
search = database.search(query)
self.assertEqual(len(search), 20)
query = AndQuery(
PhraseQuery('format', 'text'),
PhraseQuery('title_fr', u'Bonjour'),
get_base_path_query('/test-multilingual'),
)
search = database.search(query)
self.assertEqual(len(search), 20)
query = AndQuery(
PhraseQuery('format', 'text'),
PhraseQuery('title_es', u'Hola'),
get_base_path_query('/test-multilingual'),
)
search = database.search(query)
self.assertEqual(len(search), 0)
# Close database
database.close()
def get_resource_by_uuid_query(uuid, base_class_id=None, class_id=None):
# Base query
query = AndQuery(PhraseQuery('uuid', uuid))
# Add filtering query elements
if base_class_id:
query.append(PhraseQuery('base_classes', base_class_id))
elif class_id:
query.append(PhraseQuery('format', class_id))
# Ok
return query
def get_items(self, resource, context):
# Base query
query = self.get_items_query(resource, context)
# Search form
search_query = self.get_search_query(resource, context)
if search_query:
query = AndQuery(query, search_query)
# Search
return context.search(query)
def get_items_query(self, resource, context):
# Search in subtree
query = get_base_path_query(resource.abspath, max_depth=self.depth)
# Base classes
base_classes = self.base_classes
if base_classes is not None:
base_classes_query = OrQuery(
*[PhraseQuery('base_classes', x) for x in base_classes])
query = AndQuery(query, base_classes_query)
# Ok
return query
def del_resource(self, name, soft=False, ref_action='restrict'):
"""ref_action allows to specify which action is done before deleting
the resource.
ref_action can take 2 values:
- 'restrict' (default value): do an integrity check
- 'force': do nothing
"""
database = self.database
resource = self.get_resource(name, soft=soft)
if soft and resource is None:
return
# Referential action
if ref_action == 'restrict':
# Check referencial-integrity
path = str(resource.abspath)
query = AndQuery(NotQuery(PhraseQuery('abspath', path)),
NotQuery(get_base_path_query(path)))
sub_search = database.search(query)
for sub_resource in resource.traverse_resources():
path = str(sub_resource.abspath)
query = PhraseQuery('links', path)
results = sub_search.search(query)
# A resource may have been updated in the same transaction,
# so not yet reindexed: we need to check that the resource
# really links.
for referrer in results.get_resources():
if path in referrer.get_links():
err = 'cannot delete, resource "{}" is referenced'
raise ConsistencyError(err.format(path))
elif ref_action == 'force':
# Do not check referencial-integrity
pass
else:
raise ValueError, ('Incorrect ref_action "{}"'.format(ref_action))
# Events, remove
path = str(resource.abspath)
database.remove_resource(resource)
# Remove
fs = database.fs
for handler in resource.get_handlers():
# Skip empty folders and phantoms
if fs.exists(handler.key):
database.del_handler(handler.key)
self.handler.del_handler('%s.metadata' % name)
# Clear cookie
context = get_context()
cut, paths = context.get_cookie('ikaaro_cp', datatype=CopyCookie)
if path in paths:
context.del_cookie('ikaaro_cp')
def get_items(self, resource, context):
# Build the Query
search_query = PhraseQuery('format', 'user')
search_term = context.query['search_term'].strip()
if search_term:
search_query = AndQuery(search_query)
or_query = OrQuery(TextQuery('lastname', search_term),
TextQuery('firstname', search_term),
StartQuery('email', search_term),
StartQuery('email_domain', search_term))
search_query.append(or_query)
# Ok
return context.search(search_query)
def test_close_transaction(self):
"""
Test if flush is done when we close database
"""
with Database('demo.hforge.org', 19500, 20500) as database:
with database.init_context():
root = database.get_resource('/')
container = root.make_resource('folder-test-close-transaction',
Folder)
kw = {
'title': {
'fr': u'Bonjour',
'en': u'Hello'
},
'data': 'this is text'
}
resource = container.make_resource(None, Text, **kw)
self.assertEqual(str(resource.abspath),
'/folder-test-close-transaction/0')
database.save_changes()
query = AndQuery(
get_base_path_query('/folder-test-close-transaction'),
PhraseQuery('format', 'text'))
search = database.search(query)
self.assertEqual(len(search), 1)
resource = root.make_resource(None, Text)
database.close()
with Database('demo.hforge.org', 19500, 20500) as database:
with database.init_context():
query = AndQuery(
get_base_path_query('/folder-test-close-transaction'),
PhraseQuery('format', 'text'))
search = database.search(query)
self.assertEqual(len(search), 1)
self.assertEqual(
root.get_resource('/folder-test-close-transaction/1',
soft=True), None)
def get_search_query(self, resource, context):
query = AndQuery()
form = context.query
for key, datatype in self.search_schema.items():
value = form[key]
if value is None or value == '':
continue
# Special case: search on text, title and name AS AndQuery
if key == 'text':
text_query = []
value = value.split(' ')
for v in value:
t_query = OrQuery(TextQuery('title', v),
TextQuery('text', v),
PhraseQuery('name', v))
text_query.append(t_query)
if len(text_query) == 1:
text_query = text_query[0]
else:
text_query = AndQuery(*text_query)
query.append(text_query)
# Special case: type
elif key == 'format':
squery = [PhraseQuery('format', x) for x in value.split(',')]
squery = squery[0] if len(squery) == 1 else OrQuery(*squery)
query.append(squery)
# Multiple
elif datatype.multiple is True:
query.append(OrQuery(*[PhraseQuery(key, x) for x in value]))
# Singleton
else:
if value is False:
# FIXME No value means False in xapian
query.append(NotQuery(PhraseQuery(key, True)))
else:
query.append(PhraseQuery(key, value))
return query
def get_events(self, day=None, *args):
query = AndQuery(*args)
query.append(PhraseQuery('base_classes', 'event'))
if day:
query.append(PhraseQuery('dates', day))
# Do not show hidden calendars
context = get_context()
for calendar in context.search(format='calendar').get_resources():
if context.user.name in calendar.get_value('hidden_for_users'):
abspath = str(calendar.abspath)
query.append(NotQuery(PhraseQuery('calendar', abspath)))
# Ok
search = self.context.search(query)
return search.get_resources(sort_by='dtstart')
def check(self, value):
from itools.database import AndQuery, NotQuery
from itools.database import PhraseQuery
if not value:
return
context = self.context
here = context.resource
query = AndQuery(
NotQuery(PhraseQuery('abspath', str(here.abspath))),
PhraseQuery(self.field_name, value))
if self.base_query:
query.append(self.base_query)
search = context.database.search(query)
nb_results = len(search)
if nb_results > 0:
kw = {'nb_results': nb_results}
self.raise_default_error(kw)
def get_content_containers(context, class_id=None):
query = AndQuery(PhraseQuery('base_classes', 'folder'),
PhraseQuery('is_content', True))
root = context.root
for container in context.search(query).get_resources():
if not root.has_permission(context.user, 'add', container, class_id):
continue
if class_id is None:
yield container
continue
for cls in container.get_document_types():
if class_id == cls.class_id:
yield container
break
def test_everything(self):
database = self.database
# Simple Search, hit
results = database.search(data=u'lion')
self.assertEqual(len(results), 5)
documents = [
x.abspath for x in results.get_documents(sort_by='abspath')
]
self.assertEqual(documents,
['03.txt', '08.txt', '10.txt', '23.txt', '99.txt.fr'])
# Simple Search, miss
self.assertEqual(len(database.search(data=u'tiger')), 0)
# Unindex, Search, Abort, Search
database.catalog.unindex_document('03.txt')
results = database.search(data=u'lion')
self.assertEqual(len(database.search(data=u'lion')), 4)
database.catalog.abort_changes()
self.assertEqual(len(database.search(data=u'lion')), 5)
# Query on indexed boolean
self.assertEqual(len(database.search(about_wolf=True)), 5)
# Query on stored boolean
results = database.search(about_wolf=True)
longer_stories = 0
for result in results.get_documents():
if result.is_long:
longer_stories += 1
self.assertEqual(longer_stories, 0)
# Phrase Query
results = database.search(data=u'this is a double death')
self.assertEqual(len(results), 1)
# Range Query
query = RangeQuery('abspath', '03.txt', '06.txt')
results = database.search(query)
self.assertEqual(len(results), 4)
# Not Query (1/2)
query = NotQuery(PhraseQuery('data', u'lion'))
results = database.search(query)
self.assertEqual(len(results), 31)
# Not Query (2/2)
query1 = PhraseQuery('data', u'mouse')
query2 = NotQuery(PhraseQuery('data', u'lion'))
query = AndQuery(query1, query2)
results = database.search(query)
self.assertEqual(len(results), 2)
def del_resource(self, name, soft=False, ref_action='restrict'):
"""ref_action allows to specify which action is done before deleting
the resource.
ref_action can take 2 values:
- 'restrict' (default value): do an integrity check
- 'force': do nothing
"""
database = self.database
resource = self.get_resource(name, soft=soft)
if soft and resource is None:
return
# Referential action
if ref_action == 'restrict':
# Check referencial-integrity
path = str(resource.abspath)
query = AndQuery(NotQuery(PhraseQuery('abspath', path)),
NotQuery(get_base_path_query(path)))
sub_search = database.search(query)
for sub_resource in resource.traverse_resources():
path = str(sub_resource.abspath)
query = PhraseQuery('links', path)
results = sub_search.search(query)
# A resource may have been updated in the same transaction,
# so not yet reindexed: we need to check that the resource
# really links.
for referrer in results.get_resources():
if path in referrer.get_links():
err = 'cannot delete, resource "{}" is referenced'
raise ConsistencyError(err.format(path))
elif ref_action == 'force':
# Do not check referencial-integrity
pass
else:
raise ValueError,('Incorrect ref_action "{}"'.format(ref_action))
# Events, remove
path = str(resource.abspath)
database.remove_resource(resource)
# Remove handlers
for r in list(resource.traverse_resources()):
for handler in [r.metadata] + r.get_fields_handlers():
if database.has_handler(handler.key):
database.del_handler(handler.key)
def GET(self, resource, context):
# Build the query
query = get_base_path_query(resource.abspath)
for key, value in context.uri.query.items():
if key == 'abspath' and value == 'myself':
value = str(context.user.abspath)
query = AndQuery(query, PhraseQuery(key, value))
# Search
items = []
for resource in context.search(query).get_resources():
item = {'abspath': {'value': str(resource.abspath)}}
for field_name in resource.fields:
value = field_to_json(resource, field_name)
if value is not None:
item[field_name] = value
items.append(item)
# Ok
return self.return_json(items)
def get_base_path_query(path, min_depth=1, max_depth=None):
"""Builds a query that will return all the objects within the given
absolute path, like it is returned by 'resource.abspath'.
The minimum and maximum depth parameters are relative to the given path:
- If the minimum depth is zero it means include the container
- If the maximum depth is None it means unlimited.
"""
# Preprocess input data
if type(path) is not str:
path = str(path)
if max_depth is not None and max_depth < min_depth:
err = 'maximum depth (%d) smaller than minimum depth (%d)'
raise ValueError, err % (max_depth, min_depth)
# Special case: everything
if path == '/' and min_depth == 0 and max_depth is None:
return AllQuery()
# Special case: just the given path
if min_depth == 0 and max_depth == 0:
return PhraseQuery('abspath', path)
# Standard case
query = PhraseQuery('parent_paths', path)
if min_depth > 1 or max_depth is not None:
path_depth = path.rstrip('/').count('/')
a = path_depth + min_depth
b = path_depth + max_depth if max_depth is not None else None
query = AndQuery(query, RangeQuery('abspath_depth', a, b))
if min_depth == 0:
return OrQuery(query, PhraseQuery('abspath', path))
return query
def find_versions_to_update(context, force=False):
database = context.database
cls_errors = []
cls_to_update = []
# Find classes
for cls in database.get_resource_classes():
# Class version
class_version = class_version_to_date(cls.class_version)
class_version_tomorrow = class_version + timedelta(days=1)
# Search for code older than the instance
query = AndQuery(
PhraseQuery('format', cls.class_id),
RangeQuery('class_version', class_version_tomorrow, None))
search = database.search(query)
if search:
resource = search.get_resources().next()
kw = {
'class_id': resource.class_id,
'class_title': resource.class_title,
'abspath': str(resource.abspath),
'resource_version': resource.metadata.version,
'cls_version': resource.class_version
}
cls_errors.append(kw)
if force is False:
break
# Find out the versions to upgrade
classes_and_versions = []
for sub_cls in cls.mro():
for name in sub_cls.__dict__.keys():
if not name.startswith('update_'):
continue
kk, version = name.split('_', 1)
if len(version) != 8:
continue
if not version.isdigit():
continue
class_version = class_version_to_date(version)
if class_version > class_version_to_date(cls.class_version):
raise ValueError('{0} class_version is bad'.format(
sub_cls.class_id))
class_version_yesterday = class_version - timedelta(days=1)
query = AndQuery(
PhraseQuery('format', cls.class_id),
RangeQuery('class_version', None, class_version_yesterday))
search = database.search(query)
if not search:
continue
class_and_version = (cls.class_id, version)
if class_and_version in classes_and_versions:
# Overriden update method in sub classes
# So update method should be done one time
continue
classes_and_versions.append(class_and_version)
update_title_name = 'update_{0}_title'.format(version)
update_title = getattr(cls, update_title_name, MSG(u'Unknow'))
kw = {
'class_id': cls.class_id,
'class_title': cls.class_title,
'class_version': version,
'class_version_date': class_version,
'class_version_pretty': context.format_date(class_version),
'update_title': update_title.gettext(),
'nb_resources': len(search)
}
cls_to_update.append(kw)
# Sort
cls_to_update.sort(key=itemgetter('class_version_date'))
# Ok
return {'cls_to_update': cls_to_update, 'cls_errors': cls_errors}
def run_next_update_method(context, force=False):
"""Update the database to the given versions.
"""
database = context.database
log = open('%s/log/update' % database.path, 'w')
messages = []
versions = find_versions_to_update(context, force)
if not versions['cls_to_update']:
return
# Update
version = versions['cls_to_update'][0]
class_version = version['class_version_date']
class_version_yesterday = class_version - timedelta(days=1)
query = AndQuery(
PhraseQuery('format', version['class_id']),
RangeQuery('class_version', None, class_version_yesterday))
search = context.database.search(query)
# Update
resources_old2new = database.resources_old2new
for resource in search.get_resources():
path = str(resource.abspath)
abspath = resources_old2new.get(path, path)
if abspath is None:
# resource deleted
continue
# Reindex on errors
obj_version = resource.metadata.version
cls_version = resource.class_version
next_versions = resource.get_next_versions()
if (obj_version == cls_version or not next_versions
or next_versions[0] != version['class_version']):
database.catalog.unindex_document(str(resource.abspath))
values = resource.get_catalog_values()
database.catalog.index_document(values)
continue
try:
if resource is not None:
# If resource has not been removed
resource.update(version['class_version'])
except Exception:
line = 'ERROR: "{0}" - class_id: "{1}"\n'.format(
resource.abspath, resource.__class__.class_id)
log.write(line)
print_exc(file=log)
log.write('\n')
# Add message
messages.append(line)
if force is False:
return messages
# Commit
if not database.has_changed:
# We reindex so the class_version is reindexed
database.catalog.save_changes()
else:
# Commit (Do not override the mtime/author)
git_message = u'Upgrade to version {0}'.format(
version['class_version'])
context.git_message = git_message
context.set_mtime = False
database.save_changes()
# Ok
return messages
def test_AndQuery_empty(self):
query = AndQuery()
query.append(PhraseQuery('data', u'mouse'))
query.append(NotQuery(PhraseQuery('data', u'lion')))
results = self.database.search(query)
self.assertEqual(len(results), 2)
def test_AndQuery_multiple(self):
query = AndQuery(PhraseQuery('data', u'mouse'),
NotQuery(PhraseQuery('data', u'lion')))
results = self.database.search(query)
self.assertEqual(len(results), 2)
def search_forms(self):
query = AndQuery(PhraseQuery('base_classes', 'form'),
get_base_path_query(self.abspath))
return get_context().search(query)
