def reset_comment_count(root, registry): # pragma: no cover """Reset comment_count for all ICommentables - See #2194, #2188.""" from adhocracy_core.resources.comment import ICommentVersion from adhocracy_core.sheets.comment import ICommentable from adhocracy_core.resources.subscriber import update_comments_count catalogs = find_service(root, 'catalogs') query = search_query._replace(interfaces=ICommentable, only_visible=True, resolve=True) commentables = catalogs.search(query).elements count = len(commentables) for index, comment in enumerate(commentables): logger.info('Set comment_count to 0 for resource {0} of {1}' .format(index + 1, count)) commentable_sheet = registry.content.get_sheet(comment, ICommentable) commentable_sheet.set({'comments_count': 0}, omit_readonly=False) query = search_query._replace(interfaces=ICommentVersion, only_visible=True, resolve=True, indexes={'tag': 'FIRST'}) comment_versions = catalogs.search(query).elements count = len(comment_versions) for index, comment in enumerate(comment_versions): logger.info('Recalculate comment_count for resource {0} of {1}' .format(index + 1, count)) update_comments_count(comment, 1, registry)
def _export_comments(root: IResource, registry: Registry): catalogs = find_service(root, 'catalogs') query = search_query._replace(interfaces=IMercatorProposal, resolve=True, ) proposals = catalogs.search(query).elements filename = create_filename(directory='./var/export', prefix='ae-2016-comments', suffix='.csv') result_file = open(filename, 'w', newline='') wr = csv.writer(result_file, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL) fields = \ [('URL', partial(_get_url, registry)), ('Title', partial(get_sheet_field_for_partial, ITitle, 'title')), ('Comments', partial(_get_comments, registry))] wr.writerow([name for (name, _) in fields]) for proposal in proposals: result = [] append_field = partial(append_cvs_field, result) for name, get_field in fields: append_field(get_field(proposal)) wr.writerow(result) print('Exported mercator comments to %s' % filename)
def _get_references_query(self, params: dict) -> SearchQuery: """Might be overridden in subclasses.""" default_params = {"only_visible": False, "resolve": True, "allows": (), "references": []} query = search_query._replace(**default_params) if params: query = query._replace(**params) return query
def _search_user(self, index_name: str, value: str) -> IUser: catalogs = find_service(self.context, 'catalogs') query = search_query._replace(indexes={index_name: value}, resolve=True) users = catalogs.search(query).elements if len(users) == 1: return users[0]
def update_comments_count(resource: ICommentVersion, delta: int, registry: Registry): """Update all commentable resources related to `resource`. Traverse all commentable resources that have a IComment or ISubresource reference to `resource` and update the comment_count value with `delta`. Example reference structure that is traversed: comment <-IComment- comment <-IComment- comment """ catalogs = find_service(resource, 'catalogs') traverse = ReferenceComparator.traverse.value query = search_query._replace( only_visible=True, references=((traverse, (resource, sheets.comment.IComment, '', None)), ), resolve=True, ) commentables = catalogs.search(query).elements for commentable in commentables: commentable_sheet = registry.content.get_sheet(commentable, ICommentable) old_count = commentable_sheet.get()['comments_count'] commentable_sheet.set({'comments_count': old_count + delta}, omit_readonly=False)
def _get_references_query(self, params: dict) -> SearchQuery: """Might be overridden in subclasses.""" query = search_query._replace(resolve=True) if params: query = query._replace(**params) query = query._replace(allows=()) # no view permission check return query
def _delete_rate_items(catalogs: ICatalogsService, user: IUser): query = search_query._replace( interfaces=IRate, references=[(None, IMetadata, 'creator', user)], ) user_rates = catalogs.search(query).elements for rate in user_rates: _delete(rate)
def index_badge(resource, default) -> [str]: """Return value for the badge index.""" catalogs = find_service(resource, 'catalogs') reference = (None, IBadgeAssignment, 'object', resource) query = search_query._replace(references=[reference], only_visible=True, ) assignments = catalogs.search(query).elements badge_names = [] for assignment in assignments: reference = (assignment, IBadgeAssignment, 'badge', None) query = search_query._replace(references=[reference], only_visible=True, ) badges = catalogs.search(query).elements badge_names += [b.__name__ for b in badges] return badge_names
def add_badge_assignments_services_to_proposal_items(root, registry): """Add badge assignments services to proposals.""" catalogs = find_service(root, 'catalogs') query = search_query._replace(interfaces=IMercatorProposal) proposals = catalogs.search(query).elements for proposal in proposals: if find_service(proposal, 'badge_assignments') is None: logger.info('add badge assignments to {0}'.format(proposal)) add_badge_assignments_service(proposal, registry, {})
def _get_processes_with_auto_transition(root: IRoot, registry: Registry) -> [IResource]: catalogs = find_service(root, 'catalogs') query = search_query._replace(interfaces=IProcess) processes = catalogs.search(query).elements processes_with_auto_transition = filter( lambda r: _auto_transition_enabled(r, registry), processes) return processes_with_auto_transition
def update_workflow_state_acls(context: IPool, registry: Registry): """Update :term:`acl` of current workflow state for all resources.""" catalog = find_service(context, 'catalogs') query = search_query._replace(interfaces=IWorkflowAssignment) resources = catalog.search(query).elements for resource in resources: workflow = registry.content.get_workflow(resource) if workflow is None: continue workflow.update_acl(resource)
def index_badge(resource, default) -> [str]: """Return value for the badge index.""" catalogs = find_service(resource, 'catalogs') reference = (None, IBadgeAssignment, 'object', resource) query = search_query._replace( references=[reference], only_visible=True, ) assignments = catalogs.search(query).elements badge_names = [] for assignment in assignments: reference = (assignment, IBadgeAssignment, 'badge', None) query = search_query._replace( references=[reference], only_visible=True, ) badges = catalogs.search(query).elements badge_names += [b.__name__ for b in badges] return badge_names
def migrate_stadtforum_proposals_to_ipolls(root, registry): """Migrate stadtforum proposals to ipolls.""" from adhocracy_core.resources.proposal import IProposal from adhocracy_meinberlin.resources.stadtforum import IProcess from adhocracy_meinberlin.resources.stadtforum import IPoll from adhocracy_meinberlin.resources.stadtforum import poll_meta catalogs = find_service(root, 'catalogs') query = search_query._replace(interfaces=(IProcess, ), resolve=True) stadtforums = catalogs.search(query).elements for stadtforum in stadtforums: proposals_query = search_query._replace(interfaces=(IProposal, ), root=stadtforum, resolve=True) proposals = catalogs.search(proposals_query).elements for proposal in proposals: directlyProvides(proposal, IPoll) for sheet in poll_meta.basic_sheets + poll_meta.extended_sheets: alsoProvides(proposal, sheet) catalogs.reindex_index(proposal, 'interfaces')
def add_image_reference_to_organisations(root, registry): # pragma: no cover """Add image reference to organisations and add assets service.""" catalogs = find_service(root, 'catalogs') query = search_query._replace(interfaces=(IOrganisation,), resolve=True) organisations = catalogs.search(query).elements for organisation in organisations: if not IHasAssetPool.providedBy(organisation): logger.info('Add assets service to {0}'.format(organisation)) add_assets_service(organisation, registry, {}) migrate_new_sheet(root, IOrganisation, IHasAssetPool) migrate_new_sheet(root, IOrganisation, IImageReference)
def _get_rates_user_non_anonymized(context: IResource, request: IRequest, value: dict) -> [IRate]: catalogs = find_service(context, 'catalogs') authenticated_user = request.anonymized_user or request.user query = search_query._replace( references=(Reference(None, IRate, 'subject', authenticated_user), Reference(None, IRate, 'object', value['object'])), resolve=True, ) rates = catalogs.search(query).elements return rates
def _get_follow_subscriptions(streams: [tuple], request: IRequest) -> dict: context = request.root catalogs = find_service(context, 'catalogs') subscriptions = defaultdict(set) for resource in [x for x, y in streams]: ref = Reference(None, INotification, 'follow_resources', resource) query = search_query._replace(references=(ref, )) followers = catalogs.search(query).elements for follower in followers: subscriptions[resource].add(follower) return subscriptions
def add_logbook_service_to_proposal_items(root): # pragma: no cover """Add logbook service to proposals.""" catalogs = find_service(root, 'catalogs') query = search_query._replace(interfaces=IMercatorProposal) proposals = catalogs.search(query).elements registry = get_current_registry(root) for proposal in proposals: if find_service(proposal, 'logbook') is None: logger.info('add logbook service to {0}'.format(proposal)) creator = get_sheet_field(proposal, IMetadata, 'creator') add_logbook_service(proposal, registry, {'creator': creator})
def _search_user(self, index_name: str, value: str) -> IUser: catalogs = find_service(self.context, 'catalogs') query = search_query._replace(indexes={index_name: value}, resolve=True) users = catalogs.search(query).elements users_count = len(users) if users_count == 1: return users[0] elif users_count > 1: raise ValueError('{} users are indexed by `{}` with value `{}`.' .format(users_count, index_name, value))
def migrate_stadtforum_proposals_to_ipolls(root, registry): """Migrate stadtforum proposals to ipolls.""" from adhocracy_core.resources.proposal import IProposal from adhocracy_meinberlin.resources.stadtforum import IProcess from adhocracy_meinberlin.resources.stadtforum import IPoll from adhocracy_meinberlin.resources.stadtforum import poll_meta catalogs = find_service(root, 'catalogs') query = search_query._replace(interfaces=(IProcess,), resolve=True) stadtforums = catalogs.search(query).elements for stadtforum in stadtforums: proposals_query = search_query._replace(interfaces=(IProposal,), root=stadtforum, resolve=True) proposals = catalogs.search(proposals_query).elements for proposal in proposals: directlyProvides(proposal, IPoll) for sheet in poll_meta.basic_sheets + poll_meta.extended_sheets: alsoProvides(proposal, sheet) catalogs.reindex_index(proposal, 'interfaces')
def _get_references_query(self, params: dict) -> SearchQuery: """Might be overridden in subclasses.""" default_params = {'only_visible': False, 'resolve': True, 'allows': (), 'references': [], } query = search_query._replace(**default_params) if params: query = query._replace(**params) return query
def _get_follow_subscriptions(streams: [tuple], request: IRequest) -> dict: context = request.root catalogs = find_service(context, 'catalogs') subscriptions = defaultdict(set) for resource in [x for x, y in streams]: ref = Reference(None, INotification, 'follow_resources', resource) query = search_query._replace(references=(ref,)) followers = catalogs.search(query).elements for follower in followers: subscriptions[resource].add(follower) return subscriptions
def get_choices_by_interface(interface: IInterface, context: IResource, request: IRequest, ) -> []: """Get choices for resource paths by interface.""" catalogs = find_service(context, 'catalogs') query = search_query._replace(interfaces=interface) resources = catalogs.search(query).elements choices = [(request.resource_url(r, route_name=API_ROUTE_NAME), resource_path(r)) for r in resources] return choices
def _search_user(self, index_name: str, value: str) -> IUser: catalogs = find_service(self.context, 'catalogs') query = search_query._replace(indexes={index_name: value}, resolve=True) users = catalogs.search(query).elements users_count = len(users) if users_count == 1: return users[0] elif users_count > 1: raise ValueError( '{} users are indexed by `{}` with value `{}`.'.format( users_count, index_name, value))
def set_comment_count(root): # pragma: no cover """Set comment_count for all ICommentables.""" from adhocracy_core.resources.subscriber import update_comments_count registry = get_current_registry(root) catalogs = find_service(root, "catalogs") query = search_query._replace(interfaces=ICommentVersion, only_visible=True, resolve=True) comment_versions = catalogs.search(query).elements count = len(comment_versions) for index, comment in enumerate(comment_versions): logger.info("Set comment_count for resource {0} of {1}".format(index + 1, count)) update_comments_count(comment, 1, registry)
def _get_references_query(self, params: dict) -> SearchQuery: """Might be overridden in subclasses.""" default_params = { 'only_visible': False, 'resolve': True, 'allows': (), 'references': [], } query = search_query._replace(**default_params) if params: query = query._replace(**params) return query
def get_assignable_badges(context: IBadgeable, request: Request) -> [IBadge]: """Get assignable badges for the IBadgeAssignment sheet.""" badges = find_service(context, 'badges') if badges is None: return [] catalogs = find_service(context, 'catalogs') query = search_query._replace( root=badges, interfaces=IBadge, allows=(request.effective_principals, 'assign_badge'), ) result = catalogs.search(query) return result.elements
def delete_not_referenced_images( root, max_age: int, ): """Delete images older than `max_age` that are not referenced.""" catalogs = find_service(root, 'catalogs') max_date = now() - timedelta(days=max_age) query = search_query._replace( interfaces=IImage, resolve=True, indexes={'item_creation_date': (FieldComparator.lt.value, max_date)}) images = catalogs.search(query).elements msg = 'Found {0} images older then {1} days'.format(len(images), max_age) logger.info(msg) for image in images: picture_reference = Reference(None, IImageReference, '', image) query = search_query._replace(references=(picture_reference, )) referencing = catalogs.search(query) if referencing.count > 0: msg = 'Deleting image {0} that is not referenced'.format(image) logger.info(msg) del image.__parent__[image.__name__]
def _get_rate_date(user: IUser, rateable: IRateable, registry: Registry) -> str: catalogs = find_service(rateable, 'catalogs') references = [(None, IRate, 'subject', user), (None, IRate, 'object', rateable)] query = search_query._replace(interfaces=IRate, resolve=True, references=references) rate = catalogs.search(query).elements[0] creation_date = registry.content.get_sheet_field(rate, IMetadata, 'item_creation_date') creation_date_str = creation_date.strftime('%Y-%m-%d_%H:%M:%S') return creation_date_str
def get_assignable_badges(context: IBadgeable, request: Request) -> [IBadge]: """Get assignable badges for the IBadgeAssignment sheet.""" badges = find_service(context, 'badges') if badges is None: return [] catalogs = find_service(context, 'catalogs') principals = request.effective_principals query = search_query._replace(root=badges, interfaces=IBadge, allows=(principals, 'assign_badge'), ) result = catalogs.search(query) return result.elements
def _delete_not_referenced_images(root, max_age: int, ): catalogs = find_service(root, 'catalogs') max_date = now() - timedelta(days=max_age) query = search_query._replace(interfaces=IImage, resolve=True, indexes={'item_creation_date': (FieldComparator.lt.value, max_date) } ) images = catalogs.search(query).elements msg = 'Found {0} images older then {1} days'.format(len(images), max_age) logger.info(msg) for image in images: picture_reference = Reference(None, IImageReference, '', image) query = search_query._replace(references=(picture_reference,)) referencing = catalogs.search(query) if referencing.count > 0: msg = 'Deleting image {0} that is not referenced'.format(image) logger.info(msg) del image.__parent__[image.__name__]
def _get_children_sort_by_rates(context) -> []: catalog = find_service(context, 'catalogs') if catalog is None: return [] # ease testing result = catalog.search(search_query._replace(root=context, depth=2, only_visible=True, interfaces=(IRateable, IVersionable), sort_by='rates', indexes={'tag': 'LAST', 'workflow_state': 'voteable'}, )) return (r.__parent__ for r in result.elements)
def _get_references_query(self, params: dict) -> SearchQuery: reftype = self._fields['reference']['elements'].reftype target_isheet = reftype.getTaggedValue('target_isheet') default_params = {'interfaces': target_isheet, 'root': self.context, 'depth': 1, 'only_visible': False, 'resolve': True, 'allows': (), } query = search_query._replace(**default_params) if params: query = query._replace(**params) return query
def _map_rating_users(rateables: [IRateable], registry: Registry) -> [(IRateable, set(IUser))]: rateables_users_map = [] get_sheet_field = registry.content.get_sheet_field for rateable in rateables: catalogs = find_service(rateable, 'catalogs') references = [(None, IRate, 'object', rateable)] query = search_query._replace(interfaces=IRate, resolve=True, references=references) rates = catalogs.search(query).elements users = [get_sheet_field(x, IRate, 'subject') for x in rates] rateables_users_map.append((rateable, set(users))) return rateables_users_map
def index_comments(resource, default) -> int: """ Return aggregated values of comments below the `item` parent of `resource`. Only the LAST version of each rate is counted. """ item = find_interface(resource, IItem) catalogs = find_service(resource, 'catalogs') query = search_query._replace(root=item, interfaces=ICommentVersion, indexes={'tag': 'LAST'}, ) result = catalogs.search(query) return result.count
def index_comments(resource, default) -> int: """ Return aggregated values of comments below the `item` parent of `resource`. Only the LAST version of each rate is counted. """ item = find_interface(resource, IItem) catalogs = find_service(resource, 'catalogs') query = search_query._replace( root=item, interfaces=ICommentVersion, indexes={'tag': 'LAST'}, ) result = catalogs.search(query) return result.count
def _get_references_query(self, params: dict) -> SearchQuery: reftype = self._fields['reference']['elements'].reftype target_isheet = reftype.getTaggedValue('target_isheet') default_params = { 'interfaces': target_isheet, 'root': self.context, 'depth': 1, 'only_visible': False, 'resolve': True, 'allows': (), } query = search_query._replace(**default_params) if params: query = query._replace(**params) return query
def _get_comments(registry: Registry, proposal: IMercatorProposal): item = find_interface(proposal, IItem) catalogs = find_service(proposal, 'catalogs') query = search_query._replace(root=item, interfaces=ICommentVersion, indexes={'tag': 'LAST'}, sort_by='item_creation_date', ) result = catalogs.search(query) comments = list(result.elements) comment_content = [registry.context.get_sheet_field(comment, IComment, 'content') for comment in comments] comment_content_flat = '\n----------\n'.join(comment_content) return comment_content_flat
def test_get_reference_without_permission_check( self, inst, context, sheet_catalogs, mock_node_single_reference): from adhocracy_core.interfaces import ISheet from adhocracy_core.interfaces import search_query from adhocracy_core.interfaces import Reference node = mock_node_single_reference inst.schema.children.append(node) appstruct = inst.get() reference = Reference(context, ISheet, 'reference', None) query = search_query._replace(references=[reference], resolve=True, allows=(('admin'), 'view') ) assert sheet_catalogs.search.call_args[0][0] == query._replace(allows=())
def index_controversiality(resource, default) -> int: """Return metric based on number up/down rates and comments. Only the LAST version of each rate is counted. """ catalogs = find_service(resource, 'catalogs') query = search_query._replace(interfaces=IRate, frequency_of='rate', indexes={'tag': 'LAST'}, references=[(None, IRate, 'object', resource) ], ) result = catalogs.search(query) up_rates = result.frequency_of.get(1, 0) down_rates = result.frequency_of.get(-1, 0) controversiality = math.sqrt(up_rates * down_rates) return controversiality
def _get_children_sort_by_rates(context) -> []: catalog = find_service(context, 'catalogs') if catalog is None: return [] # ease testing result = catalog.search( search_query._replace( root=context, depth=2, only_visible=True, interfaces=(IRateable, IVersionable), sort_by='rates', indexes={ 'tag': 'LAST', 'workflow_state': 'voteable' }, )) return (r.__parent__ for r in result.elements)
def index_rates(resource, default) -> int: """ Return aggregated values of referenceing :class:`IRate` resources. Only the LAST version of each rate is counted. """ catalogs = find_service(resource, 'catalogs') query = search_query._replace(interfaces=IRate, frequency_of='rate', indexes={'tag': 'LAST'}, references=[(None, IRate, 'object', resource) ], ) result = catalogs.search(query) rate_sum = 0 for value, count in result.frequency_of.items(): rate_sum += value * count return rate_sum
def index_rates(resource, default) -> int: """ Return aggregated values of referenceing :class:`IRate` resources. Only the LAST version of each rate is counted. """ catalogs = find_service(resource, 'catalogs') query = search_query._replace( interfaces=IRate, frequency_of='rate', indexes={'tag': 'LAST'}, references=[(None, IRate, 'object', resource)], ) result = catalogs.search(query) rate_sum = 0 for value, count in result.frequency_of.items(): rate_sum += value * count return rate_sum
def index_comments(resource, default) -> int: """ Return aggregated values of comments below the `item` parent of `resource`. Only the LAST version of each rate is counted. """ catalogs = find_service(resource, 'catalogs') query = search_query._replace( interfaces=ICommentVersion, indexes={'tag': 'LAST'}, only_visible=True, references=[(None, IComment, 'refers_to', resource)], ) result = catalogs.search(query) comment_count = result.count if comment_count: comment_count += _index_comment_replies(result.elements, default) return comment_count
def index_controversiality(resource, default) -> int: """Return metric based on number up/down rates and comments. Only the LAST version of each rate is counted. """ catalogs = find_service(resource, 'catalogs') query = search_query._replace( interfaces=IRate, frequency_of='rate', indexes={'tag': 'LAST'}, only_visible=True, references=[(None, IRate, 'object', resource)], ) result = catalogs.search(query) up_rates = result.frequency_of.get(1, 0) down_rates = result.frequency_of.get(-1, 0) controversiality = math.sqrt(up_rates * down_rates) return controversiality
def _get_rates_user_anonymized(context: IResource, request: IRequest, value: dict) -> [IRate]: from adhocracy_core.resources.principal import get_system_user_anonymous catalogs = find_service(context, 'catalogs') anonymous = get_system_user_anonymous(request) query = search_query._replace( references=(Reference(None, IRate, 'subject', anonymous), Reference(None, IRate, 'object', value['object'])), resolve=True, ) rates = catalogs.search(query).elements rates_deanonymized = [] authenticated_user = request.anonymized_user or request.user for rate in rates: anonymized_creator = get_anonymized_creator(rate) if anonymized_creator == resource_path(authenticated_user): rates_deanonymized.append(rate) return rates_deanonymized
def validator(node, value): catalogs = find_service(context, 'catalogs') query = search_query._replace( references=(Reference(None, IRate, 'subject', value['subject']), Reference(None, IRate, 'object', value['object'])), resolve=True, ) same_rates = catalogs.search(query).elements if not same_rates: return item = find_interface(context, IRateItem) old_versions = get_sheet_field(item, IVersions, 'elements', registry=registry) for rate in same_rates: if rate not in old_versions: err = colander.Invalid(node, msg='') err['object'] = 'Another rate by the same user already exists' raise err
def test_ignore_if_no_rated_children(self, context, request_, mock_sheet, mock_catalogs): from adhocracy_core.interfaces import search_query from adhocracy_core.sheets.rate import IRateable from adhocracy_core.sheets.versions import IVersionable self.call_fut(context, request_) wanted_query = search_query._replace(\ interfaces=(IRateable, IVersionable), root=context, depth=2, only_visible=True, sort_by='rates', indexes = {'tag': 'LAST', 'workflow_state': 'voteable'}, ) assert not mock_sheet.set.called assert mock_catalogs.search.call_args[0][0] == wanted_query