def cron_manager(self):
database = self.database
# Build fake context
context = get_fake_context(database, self.root.context_cls)
context.server = self
context.init_context()
context.is_cron = True
# Go
query = RangeQuery('next_time_event', None, context.timestamp)
for brain in database.search(query).get_documents():
payload = pickle.loads(brain.next_time_event_payload)
resource = database.get_resource(brain.abspath)
try:
resource.time_event(payload)
except Exception:
# Log error
log_error('Cron error\n' + format_exc())
context.root.alert_on_internal_server_error(context)
# Reindex resource without committing
catalog = database.catalog
catalog.unindex_document(str(resource.abspath))
catalog.index_document(resource.get_catalog_values())
catalog.save_changes()
# Save changes
database.save_changes()
# Again, and again
return self.config.get_value('cron-interval')
def cron_manager(self):
database = self.database
# Build fake context
context = get_fake_context(database)
context.server = self
context.init_context()
# Go
query = RangeQuery('next_time_event', None, context.timestamp)
for brain in database.search(query).get_documents():
payload = pickle.loads(brain.next_time_event_payload)
resource = database.get_resource(brain.abspath)
resource.time_event(payload)
# Reindex resource without committing
catalog = database.catalog
catalog.unindex_document(str(resource.abspath))
catalog.index_document(resource.get_catalog_values())
catalog.save_changes()
# Save changes
database.save_changes()
# Again, and again
return self.config.get_value('cron-interval')
def test_everything(self):
database = self.database
# Simple Search, hit
results = database.search(data=u'lion')
self.assertEqual(len(results), 5)
documents = [
x.abspath for x in results.get_documents(sort_by='abspath')
]
self.assertEqual(documents,
['03.txt', '08.txt', '10.txt', '23.txt', '99.txt.fr'])
# Simple Search, miss
self.assertEqual(len(database.search(data=u'tiger')), 0)
# Unindex, Search, Abort, Search
database.catalog.unindex_document('03.txt')
results = database.search(data=u'lion')
self.assertEqual(len(database.search(data=u'lion')), 4)
database.catalog.abort_changes()
self.assertEqual(len(database.search(data=u'lion')), 5)
# Query on indexed boolean
self.assertEqual(len(database.search(about_wolf=True)), 5)
# Query on stored boolean
results = database.search(about_wolf=True)
longer_stories = 0
for result in results.get_documents():
if result.is_long:
longer_stories += 1
self.assertEqual(longer_stories, 0)
# Phrase Query
results = database.search(data=u'this is a double death')
self.assertEqual(len(results), 1)
# Range Query
query = RangeQuery('abspath', '03.txt', '06.txt')
results = database.search(query)
self.assertEqual(len(results), 4)
# Not Query (1/2)
query = NotQuery(PhraseQuery('data', u'lion'))
results = database.search(query)
self.assertEqual(len(results), 31)
# Not Query (2/2)
query1 = PhraseQuery('data', u'mouse')
query2 = NotQuery(PhraseQuery('data', u'lion'))
query = AndQuery(query1, query2)
results = database.search(query)
self.assertEqual(len(results), 2)
def get_base_path_query(path, min_depth=1, max_depth=None):
"""Builds a query that will return all the objects within the given
absolute path, like it is returned by 'resource.abspath'.
The minimum and maximum depth parameters are relative to the given path:
- If the minimum depth is zero it means include the container
- If the maximum depth is None it means unlimited.
"""
# Preprocess input data
if type(path) is not str:
path = str(path)
if max_depth is not None and max_depth < min_depth:
err = 'maximum depth (%d) smaller than minimum depth (%d)'
raise ValueError, err % (max_depth, min_depth)
# Special case: everything
if path == '/' and min_depth == 0 and max_depth is None:
return AllQuery()
# Special case: just the given path
if min_depth == 0 and max_depth == 0:
return PhraseQuery('abspath', path)
# Standard case
query = PhraseQuery('parent_paths', path)
if min_depth > 1 or max_depth is not None:
path_depth = path.rstrip('/').count('/')
a = path_depth + min_depth
b = path_depth + max_depth if max_depth is not None else None
query = AndQuery(query, RangeQuery('abspath_depth', a, b))
if min_depth == 0:
return OrQuery(query, PhraseQuery('abspath', path))
return query
def find_versions_to_update(context, force=False):
database = context.database
cls_errors = []
cls_to_update = []
# Find classes
for cls in database.get_resource_classes():
# Class version
class_version = class_version_to_date(cls.class_version)
class_version_tomorrow = class_version + timedelta(days=1)
# Search for code older than the instance
query = AndQuery(
PhraseQuery('format', cls.class_id),
RangeQuery('class_version', class_version_tomorrow, None))
search = database.search(query)
if search:
resource = search.get_resources().next()
kw = {
'class_id': resource.class_id,
'class_title': resource.class_title,
'abspath': str(resource.abspath),
'resource_version': resource.metadata.version,
'cls_version': resource.class_version
}
cls_errors.append(kw)
if force is False:
break
# Find out the versions to upgrade
classes_and_versions = []
for sub_cls in cls.mro():
for name in sub_cls.__dict__.keys():
if not name.startswith('update_'):
continue
kk, version = name.split('_', 1)
if len(version) != 8:
continue
if not version.isdigit():
continue
class_version = class_version_to_date(version)
if class_version > class_version_to_date(cls.class_version):
raise ValueError('{0} class_version is bad'.format(
sub_cls.class_id))
class_version_yesterday = class_version - timedelta(days=1)
query = AndQuery(
PhraseQuery('format', cls.class_id),
RangeQuery('class_version', None, class_version_yesterday))
search = database.search(query)
if not search:
continue
class_and_version = (cls.class_id, version)
if class_and_version in classes_and_versions:
# Overriden update method in sub classes
# So update method should be done one time
continue
classes_and_versions.append(class_and_version)
update_title_name = 'update_{0}_title'.format(version)
update_title = getattr(cls, update_title_name, MSG(u'Unknow'))
kw = {
'class_id': cls.class_id,
'class_title': cls.class_title,
'class_version': version,
'class_version_date': class_version,
'class_version_pretty': context.format_date(class_version),
'update_title': update_title.gettext(),
'nb_resources': len(search)
}
cls_to_update.append(kw)
# Sort
cls_to_update.sort(key=itemgetter('class_version_date'))
# Ok
return {'cls_to_update': cls_to_update, 'cls_errors': cls_errors}
def run_next_update_method(context, force=False):
"""Update the database to the given versions.
"""
database = context.database
log = open('%s/log/update' % database.path, 'w')
messages = []
versions = find_versions_to_update(context, force)
if not versions['cls_to_update']:
return
# Update
version = versions['cls_to_update'][0]
class_version = version['class_version_date']
class_version_yesterday = class_version - timedelta(days=1)
query = AndQuery(
PhraseQuery('format', version['class_id']),
RangeQuery('class_version', None, class_version_yesterday))
search = context.database.search(query)
# Update
resources_old2new = database.resources_old2new
for resource in search.get_resources():
path = str(resource.abspath)
abspath = resources_old2new.get(path, path)
if abspath is None:
# resource deleted
continue
# Reindex on errors
obj_version = resource.metadata.version
cls_version = resource.class_version
next_versions = resource.get_next_versions()
if (obj_version == cls_version or not next_versions
or next_versions[0] != version['class_version']):
database.catalog.unindex_document(str(resource.abspath))
values = resource.get_catalog_values()
database.catalog.index_document(values)
continue
try:
if resource is not None:
# If resource has not been removed
resource.update(version['class_version'])
except Exception:
line = 'ERROR: "{0}" - class_id: "{1}"\n'.format(
resource.abspath, resource.__class__.class_id)
log.write(line)
print_exc(file=log)
log.write('\n')
# Add message
messages.append(line)
if force is False:
return messages
# Commit
if not database.has_changed:
# We reindex so the class_version is reindexed
database.catalog.save_changes()
else:
# Commit (Do not override the mtime/author)
git_message = u'Upgrade to version {0}'.format(
version['class_version'])
context.git_message = git_message
context.set_mtime = False
database.save_changes()
# Ok
return messages