def test_namespaces(self): item_name_n = u'normal' item = self.imw[item_name_n] rev_n = item.store_revision(dict( name=[ item_name_n, ], contenttype=u'text/plain;charset=utf-8'), StringIO(str(item_name_n)), return_rev=True) item_name_u = u'%s/userprofile' % NAMESPACE_USERPROFILES fqname_u = split_fqname(item_name_u) item = self.imw.get_item(**fqname_u.query) rev_u = item.store_revision(dict( name=[fqname_u.value], namespace=fqname_u.namespace, contenttype=u'text/plain;charset=utf-8'), StringIO(str(item_name_u)), return_rev=True) item = self.imw[item_name_n] rev_n = item.get_revision(rev_n.revid) assert rev_n.meta[NAMESPACE] == u'' assert rev_n.meta[NAME] == [ item_name_n, ] item = self.imw[item_name_u] rev_u = item.get_revision(rev_u.revid) assert rev_u.meta[NAMESPACE] == NAMESPACE_USERPROFILES assert rev_u.meta[NAME] == [item_name_u.split('/')[1]]
def location_breadcrumbs(self, fqname): """ Assemble the location using breadcrumbs (was: title) :rtype: list :returns: location breadcrumbs items in tuple (segment_name, fq_name, exists) """ breadcrumbs = [] current_item = '' if not isinstance(fqname, CompositeName): fqname = split_fqname(fqname) if fqname.field != NAME_EXACT: return [(fqname, fqname, bool(self.storage.get_item(**fqname.query)))] namespace = fqname.namespace fq_current = CompositeName(u'', NAME_EXACT, namespace) fq_segment = CompositeName(u'', NAME_EXACT, namespace or '~') breadcrumbs.append((fq_segment, fq_current, False)) item_name = fqname.value if not item_name: return breadcrumbs for segment in item_name.split('/'): current_item += segment fq_current = CompositeName(namespace, NAME_EXACT, current_item) fq_segment = CompositeName(namespace, NAME_EXACT, segment) breadcrumbs.append((fq_segment, fq_current, bool(self.storage.get_item(**fq_current.query)))) current_item += '/' return breadcrumbs
def location_breadcrumbs(self, fqname): """ Assemble the location using breadcrumbs (was: title) :rtype: list :returns: location breadcrumbs items in tuple (segment_name, fq_name, exists) """ breadcrumbs = [] current_item = '' if not isinstance(fqname, CompositeName): fqname = split_fqname(fqname) if fqname.field != NAME_EXACT: return [(fqname, fqname, bool(self.storage.get_item(**fqname.query)))] namespace = fqname.namespace fq_current = CompositeName(u'', NAME_EXACT, namespace) fq_segment = CompositeName(u'', NAME_EXACT, namespace or '~') breadcrumbs.append((fq_segment, fq_current, False)) item_name = fqname.value if not item_name: return breadcrumbs for segment in item_name.split('/'): current_item += segment fq_current = CompositeName(namespace, NAME_EXACT, current_item) fq_segment = CompositeName(namespace, NAME_EXACT, segment) breadcrumbs.append( (fq_segment, fq_current, bool(self.storage.get_item(**fq_current.query)))) current_item += '/' return breadcrumbs
def location_breadcrumbs(self, fqname): """ Split the incoming fqname into segments. Reassemble into a list of tuples. If the fqname has a namespace, the first tuple's segment_name will have the namespace as a prefix. :rtype: list :returns: location breadcrumbs items in tuple (segment_name, fq_name, exists) """ breadcrumbs = [] current_item = '' if not isinstance(fqname, CompositeName): fqname = split_fqname(fqname) if fqname.field != NAME_EXACT: return [ (fqname, fqname, bool(self.storage.get_item(**fqname.query))) ] # flaskg.unprotected_storage.get_item(**fqname.query) namespace = segment1_namespace = fqname.namespace item_name = fqname.value if not item_name: return breadcrumbs for segment in item_name.split('/'): current_item += segment fq_current = CompositeName(namespace, NAME_EXACT, current_item) fq_segment = CompositeName(segment1_namespace, NAME_EXACT, segment) breadcrumbs.append( (fq_segment, fq_current, bool(self.storage.get_item(**fq_current.query)))) current_item += '/' segment1_namespace = u'' return breadcrumbs
def modify_acl(item_name): fqname = split_fqname(item_name) item = Item.create(item_name) meta = dict(item.meta) new_acl = request.form.get(fqname.value) meta[ACL] = new_acl item._save(meta=meta) flash("Changes successfully applied", "info") return redirect(url_for('.item_acl_report'))
def test_split_fqname(self): app.cfg.namespace_mapping = [(u'', 'default_backend'), (u'ns1/', 'default_backend'), (u'ns1/ns2/', 'other_backend')] tests = [('ns1/ns2/@itemid/SomeItemID', ('ns1/ns2', 'itemid', 'SomeItemID')), ('ns3/@itemid/SomeItemID', ('', 'name_exact', 'ns3/@itemid/SomeItemID')), ('Page', ('', 'name_exact', 'Page')), ('ns1/ns2/@tags/SomeTag', ('ns1/ns2', 'tags', 'SomeTag')), ('@tags/SomeTag', ('', 'tags', 'SomeTag')), ('ns1/ns2/@notid', ('ns1/ns2', 'name_exact', '@notid')), ('ns1/ns2/ns3/Thisisapagename/ns4', ('ns1/ns2', 'name_exact', 'ns3/Thisisapagename/ns4')), ] for url, (namespace, field, pagename) in tests: assert split_fqname(url) == (namespace, field, pagename)
def subitem_index(self, fqname): """ Get a list of subitems for the given fqname :rtype: list :returns: list of item tuples (item_name, item_title, item_mime_type, has_children) """ from MoinMoin.items import Item if not isinstance(fqname, CompositeName): fqname = split_fqname(fqname) item = Item.create(fqname.fullname) return item.get_mixed_index()
def may(self, fqname, capability, usernames=None): if usernames is not None and isinstance(usernames, (str, unicode)): # we got a single username (maybe str), make a list of unicode: if isinstance(usernames, str): usernames = usernames.decode('utf-8') usernames = [usernames, ] # TODO Make sure that fqname must be a CompositeName class instance, not unicode or list. fqname = fqname[0] if isinstance(fqname, list) else fqname if isinstance(fqname, unicode): fqname = split_fqname(fqname) item = self.get_item(**fqname.query) allowed = item.allows(capability, user_names=usernames) return allowed
def do_show(self, revid): """ Show a blog item and a list of its blog entries below it. If tag GET-parameter is defined, the list of blog entries consists only of those entries that contain the tag value in their lists of tags. """ # for now it is just one tag=value, later it could be tag=value1&tag=value2&... tag = request.values.get('tag') prefix = self.name + u'/' current_timestamp = int(time.time()) terms = [ Term(WIKINAME, app.cfg.interwikiname), # Only blog entry itemtypes Term(ITEMTYPE, ITEMTYPE_BLOG_ENTRY), # Only sub items of this item Prefix(NAME_EXACT, prefix), ] if tag: terms.append(Term(TAGS, tag)) query = And(terms) def ptime_sort_key(searcher, docnum): """ Compute the publication time key for blog entries sorting. If PTIME is not defined, we use MTIME. """ fields = searcher.stored_fields(docnum) ptime = fields.get(PTIME, fields[MTIME]) return ptime ptime_sort_facet = FunctionFacet(ptime_sort_key) revs = flaskg.storage.search(query, sortedby=ptime_sort_facet, reverse=True, limit=None) blog_entry_items = [ Item.create(rev.name, rev_id=rev.revid) for rev in revs ] return render_template( 'blog/main.html', item_name=self.name, fqname=split_fqname(self.name), blog_item=self, blog_entry_items=blog_entry_items, tag=tag, item=self, )
def create(cls, name=u'', itemtype=None, contenttype=None, rev_id=CURRENT, item=None): """ Create a highlevel Item by looking up :name or directly wrapping :item and extract the Revision designated by :rev_id revision. The highlevel Item is created by creating an instance of Content subclass according to the item's contenttype metadata entry; The :contenttype argument can be used to override contenttype. It is used only when handling +convert (when deciding the contenttype of target item), +modify (when creating a new item whose contenttype is not yet decided), +diff and +diffraw (to coerce the Content to a common super-contenttype of both revisions). After that the Content instance, an instance of Item subclass is created according to the item's itemtype metadata entry, and the previously created Content instance is assigned to its content property. """ fqname = split_fqname(name) if fqname.field not in UFIELDS: # Need a unique key to extract stored item. raise FieldNotUniqueError("field {0} is not in UFIELDS".format( fqname.field)) rev = get_storage_revision(fqname, itemtype, contenttype, rev_id, item) contenttype = rev.meta.get(CONTENTTYPE) or contenttype logging.debug( "Item {0!r}, got contenttype {1!r} from revision meta".format( name, contenttype)) # logging.debug("Item %r, rev meta dict: %r" % (name, dict(rev.meta))) # XXX Cannot pass item=item to Content.__init__ via # content_registry.get yet, have to patch it later. content = Content.create(contenttype) itemtype = rev.meta.get(ITEMTYPE) or itemtype or ITEMTYPE_DEFAULT logging.debug( "Item {0!r}, got itemtype {1!r} from revision meta".format( name, itemtype)) item = item_registry.get(itemtype, fqname, rev=rev, content=content) logging.debug("Item class {0!r} handles {1!r}".format( item.__class__, itemtype)) content.item = item return item
def run(self, name, meta_file, data_file, revid): fqname = split_fqname(name) item = app.storage.get_item(**fqname.query) rev = item[revid] meta = json.dumps(dict(rev.meta), sort_keys=True, indent=2, ensure_ascii=False) meta = meta.encode('utf-8') meta_lines = meta.split('\n') meta_lines = [x.rstrip() for x in meta_lines] meta = '\n'.join(meta_lines) with open(meta_file, 'wb') as mf: mf.write(meta) data = rev.data.read().replace('\r\n', '\n') with open(data_file, 'wb') as df: df.write(data)
def test_namespaces(self): item_name_n = u'normal' item = self.imw[item_name_n] rev_n = item.store_revision(dict(name=[item_name_n, ], contenttype=u'text/plain;charset=utf-8'), StringIO(str(item_name_n)), return_rev=True) item_name_u = u'%s/userprofile' % NAMESPACE_USERPROFILES fqname_u = split_fqname(item_name_u) item = self.imw.get_item(**fqname_u.query) rev_u = item.store_revision(dict(name=[fqname_u.value], namespace=fqname_u.namespace, contenttype=u'text/plain;charset=utf-8'), StringIO(str(item_name_u)), return_rev=True) item = self.imw[item_name_n] rev_n = item.get_revision(rev_n.revid) assert rev_n.meta[NAMESPACE] == u'' assert rev_n.meta[NAME] == [item_name_n, ] item = self.imw[item_name_u] rev_u = item.get_revision(rev_u.revid) assert rev_u.meta[NAMESPACE] == NAMESPACE_USERPROFILES assert rev_u.meta[NAME] == [item_name_u.split('/')[1]]
def modify_acl(item_name): fqname = split_fqname(item_name) item = Item.create(item_name) meta = dict(item.meta) new_acl = request.form.get(fqname.fullname) is_valid = acl_validate(new_acl) if is_valid: if new_acl in ('Empty', ''): meta[ACL] = '' elif new_acl == 'None' and ACL in meta: del(meta[ACL]) else: meta[ACL] = new_acl item._save(meta=meta) flash(L_("Success! ACL saved.<br>Item: %(item_name)s<br>ACL: %(acl_rule)s", item_name=fqname.fullname, acl_rule=new_acl), "info") else: flash(L_("Nothing changed, invalid ACL.<br>Item: %(item_name)s<br>ACL: %(acl_rule)s", item_name=fqname.fullname, acl_rule=new_acl), "error") return redirect(url_for('.item_acl_report'))
def do_show(self, revid): """ Show a blog item and a list of its blog entries below it. If tag GET-parameter is defined, the list of blog entries consists only of those entries that contain the tag value in their lists of tags. """ # for now it is just one tag=value, later it could be tag=value1&tag=value2&... tag = request.values.get('tag') prefix = self.name + u'/' current_timestamp = int(time.time()) terms = [Term(WIKINAME, app.cfg.interwikiname), # Only blog entry itemtypes Term(ITEMTYPE, ITEMTYPE_BLOG_ENTRY), # Only sub items of this item Prefix(NAME_EXACT, prefix), ] if tag: terms.append(Term(TAGS, tag)) query = And(terms) def ptime_sort_key(searcher, docnum): """ Compute the publication time key for blog entries sorting. If PTIME is not defined, we use MTIME. """ fields = searcher.stored_fields(docnum) ptime = fields.get(PTIME, fields[MTIME]) return ptime ptime_sort_facet = FunctionFacet(ptime_sort_key) revs = flaskg.storage.search(query, sortedby=ptime_sort_facet, reverse=True, limit=None) blog_entry_items = [Item.create(rev.name, rev_id=rev.revid) for rev in revs] return render_template('blog/main.html', item_name=self.name, fqname=split_fqname(self.name), blog_item=self, blog_entry_items=blog_entry_items, tag=tag, item=self, )
def create(cls, name=u'', itemtype=None, contenttype=None, rev_id=CURRENT, item=None): """ Create a highlevel Item by looking up :name or directly wrapping :item and extract the Revision designated by :rev_id revision. The highlevel Item is created by creating an instance of Content subclass according to the item's contenttype metadata entry; The :contenttype argument can be used to override contenttype. It is used only when handling +convert (when deciding the contenttype of target item), +modify (when creating a new item whose contenttype is not yet decided), +diff and +diffraw (to coerce the Content to a common super-contenttype of both revisions). After that the Content instance, an instance of Item subclass is created according to the item's itemtype metadata entry, and the previously created Content instance is assigned to its content property. """ fqname = split_fqname(name) if fqname.field not in UFIELDS: # Need a unique key to extract stored item. raise FieldNotUniqueError("field {0} is not in UFIELDS".format(fqname.field)) rev = get_storage_revision(fqname, itemtype, contenttype, rev_id, item) contenttype = rev.meta.get(CONTENTTYPE) or contenttype logging.debug("Item {0!r}, got contenttype {1!r} from revision meta".format(name, contenttype)) # logging.debug("Item %r, rev meta dict: %r" % (name, dict(rev.meta))) # XXX Cannot pass item=item to Content.__init__ via # content_registry.get yet, have to patch it later. content = Content.create(contenttype) itemtype = rev.meta.get(ITEMTYPE) or itemtype or ITEMTYPE_DEFAULT logging.debug("Item {0!r}, got itemtype {1!r} from revision meta".format(name, itemtype)) item = item_registry.get(itemtype, fqname, rev=rev, content=content) logging.debug("Item class {0!r} handles {1!r}".format(item.__class__, itemtype)) content.item = item return item
def convert_to_indexable(meta, data, item_name=None, is_new=False): """ Convert revision data to a indexable content. :param meta: revision metadata (gets updated as a side effect) :param data: revision data (file-like) please make sure that the content file is ready to read all indexable content from it. if you have just written that content or already read from it, you need to call rev.seek(0) before calling convert_to_indexable(rev). :param is_new: if this is for a new revision and we shall modify metadata as a side effect :returns: indexable content, text/plain, unicode object """ fqname = split_fqname(item_name) class PseudoRev(object): def __init__(self, meta, data): self.meta = meta self.data = data self.revid = meta.get(REVID) class PseudoItem(object): def __init__(self, fqname): self.fqname = fqname self.name = fqname.value self.item = PseudoItem(fqname) def read(self, *args, **kw): return self.data.read(*args, **kw) def seek(self, *args, **kw): return self.data.seek(*args, **kw) def tell(self, *args, **kw): return self.data.tell(*args, **kw) if meta[CONTENTTYPE] in app.cfg.mimetypes_to_index_as_empty: logging.info("not indexing content of %r as requested by configuration".format(meta[NAME])) return u'' if not item_name: try: item_name = get_names(meta)[0] except IndexError: item_name = u'DoesNotExist' rev = PseudoRev(meta, data) try: # TODO use different converter mode? # Maybe we want some special mode for the input converters so they emit # different output than for normal rendering), esp. for the non-markup # content types (images, etc.). input_contenttype = meta[CONTENTTYPE] output_contenttype = 'text/plain' type_input_contenttype = Type(input_contenttype) type_output_contenttype = Type(output_contenttype) reg = default_registry # first try a direct conversion (this could be useful for extraction # of (meta)data from binary types, like from images or audio): conv = reg.get(type_input_contenttype, type_output_contenttype) if conv: doc = conv(rev, input_contenttype) return doc # otherwise try via DOM as intermediate format (this is useful if # input type is markup, to get rid of the markup): input_conv = reg.get(type_input_contenttype, type_moin_document) refs_conv = reg.get(type_moin_document, type_moin_document, items='refs') output_conv = reg.get(type_moin_document, type_output_contenttype) if input_conv and output_conv: doc = input_conv(rev, input_contenttype) # We do not convert smileys, includes, macros, links, because # it does not improve search results or even makes results worse. # We do run the referenced converter, though, to extract links and # transclusions. if is_new: # we only can modify new, uncommitted revisions, not stored revs i = Iri(scheme='wiki', authority='', path='/' + item_name) doc.set(moin_page.page_href, unicode(i)) refs_conv(doc) # side effect: we update some metadata: meta[ITEMLINKS] = refs_conv.get_links() meta[ITEMTRANSCLUSIONS] = refs_conv.get_transclusions() meta[EXTERNALLINKS] = refs_conv.get_external_links() doc = output_conv(doc) return doc # no way raise TypeError("No converter for {0} --> {1}".format(input_contenttype, output_contenttype)) except Exception as e: # catch all exceptions, we don't want to break an indexing run logging.exception("Exception happened in conversion of item {0!r} rev {1} contenttype {2}:".format( item_name, meta.get(REVID, 'new'), meta.get(CONTENTTYPE, ''))) doc = u'ERROR [{0!s}]'.format(e) return doc
def custom_setup(self): self.imw = flaskg.unprotected_storage self.item_name = u"foo" self.fqname = split_fqname(self.item_name)
def navibar(self, fqname): """ Assemble the navibar :rtype: list :returns: list of tuples (css_class, url, link_text, title) """ if not isinstance(fqname, CompositeName): fqname = split_fqname(fqname) item_name = fqname.value current = item_name # Process config navi_bar items = [] for cls, endpoint, args, link_text, title in self.cfg.navi_bar: if endpoint == "frontend.show_root": endpoint = "frontend.show_item" root_fqname = fqname.get_root_fqname() default_root = app.cfg.root_mapping.get(NAMESPACE_DEFAULT, app.cfg.default_root) args['item_name'] = root_fqname.fullname if fqname.namespace != NAMESPACE_ALL else default_root elif endpoint in ["frontend.global_history", "frontend.global_tags"]: args['namespace'] = fqname.namespace elif endpoint == "frontend.index": args['item_name'] = fqname.namespace items.append((cls, url_for(endpoint, **args), link_text, title)) # Add user links to wiki links. for text in self.user.quicklinks: url, link_text, title = self.split_navilink(text) items.append(('userlink', url, link_text, title)) # Add sister pages (see http://usemod.com/cgi-bin/mb.pl?SisterSitesImplementationGuide ) for sistername, sisterurl in self.cfg.sistersites: if is_local_wiki(sistername): items.append(('sisterwiki current', sisterurl, sistername, '')) else: cid = cache_key(usage="SisterSites", sistername=sistername) sisteritems = app.cache.get(cid) if sisteritems is None: uo = urllib.URLopener() uo.version = 'MoinMoin SisterItem list fetcher 1.0' try: sisteritems = {} f = uo.open(sisterurl) for line in f: line = line.strip() try: item_url, item_name = line.split(' ', 1) sisteritems[item_name.decode('utf-8')] = item_url except: pass # ignore invalid lines f.close() app.cache.set(cid, sisteritems) logging.info("Site: {0!r} Status: Updated. Pages: {1}".format(sistername, len(sisteritems))) except IOError as err: (title, code, msg, headers) = err.args # code e.g. 304 logging.warning("Site: {0!r} Status: Not updated.".format(sistername)) logging.exception("exception was:") if current in sisteritems: url = sisteritems[current] items.append(('sisterwiki', url, sistername, '')) return items
def navibar(self, fqname): """ Assemble the navibar :rtype: list :returns: list of tuples (css_class, url, link_text, title) """ if not isinstance(fqname, CompositeName): fqname = split_fqname(fqname) item_name = fqname.value current = item_name # Process config navi_bar items = [] for cls, endpoint, args, link_text, title in self.cfg.navi_bar: if endpoint == "frontend.show_root": endpoint = "frontend.show_item" root_fqname = fqname.get_root_fqname() default_root = app.cfg.root_mapping.get( NAMESPACE_DEFAULT, app.cfg.default_root) args[ 'item_name'] = root_fqname.fullname if fqname.namespace != NAMESPACE_ALL else default_root elif endpoint in [ "frontend.global_history", "frontend.global_tags" ]: args['namespace'] = fqname.namespace elif endpoint == "frontend.index": args['item_name'] = fqname.namespace items.append((cls, url_for(endpoint, **args), link_text, title)) # Add user links to wiki links. for text in self.user.quicklinks: url, link_text, title = self.split_navilink(text) items.append(('userlink', url, link_text, title)) # Add sister pages (see http://usemod.com/cgi-bin/mb.pl?SisterSitesImplementationGuide ) for sistername, sisterurl in self.cfg.sistersites: if is_local_wiki(sistername): items.append(('sisterwiki current', sisterurl, sistername, '')) else: cid = cache_key(usage="SisterSites", sistername=sistername) sisteritems = app.cache.get(cid) if sisteritems is None: uo = urllib.URLopener() uo.version = 'MoinMoin SisterItem list fetcher 1.0' try: sisteritems = {} f = uo.open(sisterurl) for line in f: line = line.strip() try: item_url, item_name = line.split(' ', 1) sisteritems[item_name.decode( 'utf-8')] = item_url except: pass # ignore invalid lines f.close() app.cache.set(cid, sisteritems) logging.info( "Site: {0!r} Status: Updated. Pages: {1}".format( sistername, len(sisteritems))) except IOError as err: (title, code, msg, headers) = err.args # code e.g. 304 logging.warning( "Site: {0!r} Status: Not updated.".format( sistername)) logging.exception("exception was:") if current in sisteritems: url = sisteritems[current] items.append(('sisterwiki', url, sistername, '')) return items