def parse_data(title, itemtype, body): matches = {"name": title, "schema": schema.get_itemtype_path(itemtype)} # parse data in yaml/schema section m = re.search(PageOperationMixin.re_yaml_schema, body) if m: parsed_yaml = yaml.load(m.group(1)) if type(parsed_yaml) != dict: raise ValueError("YAML must be a dictionary") for name, value in parsed_yaml.items(): if name in matches: if type(matches[name]) != list: matches[name] = [matches[name]] if type(value) == list: matches[name] += value else: matches[name].append(value) else: matches[name] = value # parse data embedded in body text for m in re.finditer(WikiPage.re_data, body): name = m.group("name") value = m.group("value") if name in matches: if type(matches[name]) != list: matches[name] = [matches[name]] matches[name].append(value) else: matches[name] = value return matches
def parse_data(cls, title, body, itemtype=u'Article'): body = body.replace('\r\n', '\n') default_data = { 'name': title, 'schema': schema.get_itemtype_path(itemtype) } # collect yaml_data = cls.parse_schema_yaml(body) body_data = pairs_to_dict((m.group('name'), m.group('value')) for m in re.finditer(cls.re_data, body)) if itemtype == u'Article' or u'Article' in schema.get_schema( itemtype)[u'ancestors']: default_section = u'articleBody' else: default_section = u'longDescription' section_data = cls.parse_sections(body, default_section) # merge data = merge_dicts([default_data, yaml_data, body_data, section_data]) # validation and type conversion typed = schema.SchemaConverter.convert(itemtype, data) return typed
def _evaluate_page_query_term(cls, name, value): if name == "schema" and value.find("/") == -1: value = schema.get_itemtype_path(value) pages = {} for index in SchemaDataIndex.query(SchemaDataIndex.name == name, SchemaDataIndex.value == value): pages[index.title] = index.data return pages
def _update_content_all(self, body, base_revision, comment, user, force_update, dont_create_rev, dont_defer): # do not update if the body is not changed if not force_update and self.body == body: return False # validate and prepare new contents new_data, new_md = self.validate_new_content(base_revision, body, user) new_body = self._merge_if_needed(base_revision, body) # get old data and metadata old_md = self.metadata.copy() old_data = self.data.copy() # delete caches caching.del_rendered_body(self.title) caching.del_hashbangs(self.title) caching.del_metadata(self.title) caching.del_data(self.title) # update model and save self.body = new_body self.modifier = user self.description = PageOperationMixin.make_description(new_body) self.acl_read = new_md.get('read', '') self.acl_write = new_md.get('write', '') self.comment = comment self.itemtype_path = schema.get_itemtype_path(new_md['schema']) self._update_pub_state(new_md, old_md) if not dont_create_rev: self.revision += 1 if not force_update: self.updated_at = datetime.now() self.put() # create revision if not dont_create_rev: rev_key = self._rev_key() rev = WikiPageRevision(parent=rev_key, title=self.title, body=self.body, created_at=self.updated_at, revision=self.revision, comment=self.comment, modifier=self.modifier, acl_read=self.acl_read, acl_write=self.acl_write) rev.put() # update inlinks, outlinks and schema data index self.update_links_and_data(old_md.get('redirect'), new_md.get('redirect'), old_data, new_data, dont_defer) # delete config cache if self.title == '.config': caching.del_config() # delete title cache if it's a new page if self.revision == 1: caching.del_titles() return True
def parse_data(cls, title, body, itemtype=u'Article'): # collect data default_data = {'name': title, 'schema': schema.get_itemtype_path(itemtype)} yaml_data = cls.parse_schema_yaml(body) body_data = pairs_to_dict((m.group('name'), m.group('value')) for m in re.finditer(cls.re_data, body)) data = merge_dicts([default_data, yaml_data, body_data]) # validation and type conversion typed = schema.SchemaConverter.convert(itemtype, data) return typed
def parse_data(cls, title, body, itemtype=u"Article"): body = body.replace("\r\n", "\n") default_data = {"name": title, "schema": schema.get_itemtype_path(itemtype)} # collect yaml_data = cls.parse_schema_yaml(body) body_data = pairs_to_dict((m.group("name"), m.group("value")) for m in re.finditer(cls.re_data, body)) if itemtype == u"Article" or u"Article" in schema.get_schema(itemtype)[u"ancestors"]: default_section = u"articleBody" else: default_section = u"longDescription" section_data = cls.parse_sections(body, default_section) # merge data = merge_dicts([default_data, yaml_data, body_data, section_data]) # validation and type conversion typed = schema.SchemaConverter.convert(itemtype, data) return typed
def parse_data(cls, title, body, itemtype=u'Article'): body = body.replace('\r\n', '\n') default_data = {'name': title, 'schema': schema.get_itemtype_path(itemtype)} # collect yaml_data = cls.parse_schema_yaml(body) body_data = pairs_to_dict((m.group('name'), m.group('value')) for m in re.finditer(cls.re_data, body)) if itemtype == u'Article' or u'Article' in schema.get_schema(itemtype)[u'ancestors']: default_section = u'articleBody' else: default_section = u'longDescription' section_data = cls.parse_sections(body, default_section) # merge data = merge_dicts([default_data, yaml_data, body_data, section_data]) # validation and type conversion typed = schema.SchemaConverter.convert(itemtype, data) return typed
def test_itemtype_path(self): self.assertEqual('Thing/', schema.get_itemtype_path('Thing')) self.assertEqual('Thing/CreativeWork/Article/', schema.get_itemtype_path('Article'))
def _evaluate_page_query_term(cls, name, value): if name == 'schema' and value.find('/') == -1: value = schema.get_itemtype_path(value) return SchemaDataIndex.query_titles(name, value)
def _update_content_all(self, body, base_revision, comment, user, force_update, dont_create_rev): # do not update if the body is not changed if not force_update and self.body == body: return False now = datetime.utcnow().replace(tzinfo=utc) # validate and prepare new contents new_data, new_md = self.validate_new_content(base_revision, body, user) new_body = self._merge_if_needed(base_revision, body) # get old data and metadata try: old_md = self.metadata.copy() except ValueError: old_md = {} try: old_data = self.data.copy() except ValueError: old_data = {} # delete caches caching.del_rendered_body(self.title) caching.del_hashbangs(self.title) caching.del_metadata(self.title) caching.del_data(self.title) # update model and save self.body = new_body if user and not user.is_anonymous(): self.modifier = user self.description = PageOperationMixin.make_description(new_body) self.acl_read = new_md.get('read', '') self.acl_write = new_md.get('write', '') self.comment = comment self.itemtype_path = schema.get_itemtype_path(new_md['schema']) self._update_pub_state(new_md, old_md) if not dont_create_rev: self.revision += 1 if not force_update: self.updated_at = now self.save() # create revision if not dont_create_rev: rev = WikiPageRevision(page=self, title=self.title, body=self.body, created_at=self.updated_at, revision=self.revision, comment=self.comment, modifier=self.modifier) rev.save() self.update_links_and_data(old_md.get('redirect'), new_md.get('redirect'), old_data, new_data) # delete config cache if self.title == '.config': caching.del_config() # delete title cache if it's a new page if self.revision == 1: caching.del_titles() return True
def _update_content_all(self, body, base_revision, comment, user, force_update, dont_create_rev, dont_defer): # do not update if the body is not changed if not force_update and self.body == body: return False now = datetime.now() # validate and prepare new contents new_data, new_md = self.validate_new_content(base_revision, body, user) new_body = self._merge_if_needed(base_revision, body) # get old data and metadata try: old_md = self.metadata.copy() except ValueError: old_md = {} try: old_data = self.data.copy() except ValueError: old_data = {} # delete caches caching.del_rendered_body(self.title) caching.del_hashbangs(self.title) caching.del_metadata(self.title) caching.del_data(self.title) # update model and save self.body = new_body self.modifier = user self.description = PageOperationMixin.make_description(new_body) self.acl_read = new_md.get('read', '') self.acl_write = new_md.get('write', '') self.comment = comment self.itemtype_path = schema.get_itemtype_path(new_md['schema']) self._update_pub_state(new_md, old_md) if not dont_create_rev: self.revision += 1 if not force_update: self.updated_at = now self.put() # create revision if not dont_create_rev: rev_key = self._rev_key() rev = WikiPageRevision(parent=rev_key, title=self.title, body=self.body, created_at=self.updated_at, revision=self.revision, comment=self.comment, modifier=self.modifier, acl_read=self.acl_read, acl_write=self.acl_write) rev.put() # update inlinks, outlinks and schema data index self.update_links_and_data(old_md.get('redirect'), new_md.get('redirect'), old_data, new_data, dont_defer) # delete config cache if self.title == '.config': caching.del_config() # delete title cache if it's a new page if self.revision == 1: caching.del_titles() return True
def test_every_itemtype_should_have_a_parent_except_for_root(self): for item in schema.SUPPORTED_SCHEMA.keys(): self.assertEqual("Thing/", schema.get_itemtype_path(item)[:6])
def test_itemtype_path(self): self.assertEqual("Thing/", schema.get_itemtype_path("Thing")) self.assertEqual("Thing/CreativeWork/Article/", schema.get_itemtype_path("Article"))
def update_content(self, new_body, base_revision, comment="", user=None, force_update=False, dont_create_rev=False): if not force_update and self.body == new_body: return False # get old data amd metadata old_md = self.metadata old_data = self.data # validate contents ## validate schema data new_md = PageOperationMixin.parse_metadata(new_body) try: PageOperationMixin.parse_data(self.title, new_md["schema"], new_body) except Exception: raise ValueError("Invalid schema data") ## validate metadata if u"pub" in new_md and u"redirect" in new_md: raise ValueError('You cannot use "pub" and "redirect" metadata at ' "the same time.") if u"redirect" in new_md and len(PageOperationMixin.remove_metadata(new_body).strip()) != 0: raise ValueError('Page with "redirect" metadata cannot have a body ' "content.") if u"read" in new_md and new_md["content-type"] != "text/x-markdown": raise ValueError("You cannot restrict read access of custom content-typed page.") ## validate revision if self.revision < base_revision: raise ValueError("Invalid revision number: %d" % base_revision) ## validate ToC if not TocGenerator(md.convert(new_body)).validate(): raise ValueError("Duplicate paths not allowed") if self.revision != base_revision: # perform 3-way merge if needed base = ( WikiPageRevision.query(WikiPageRevision.title == self.title, WikiPageRevision.revision == base_revision) .get() .body ) merged = "".join(Merge3(base, self.body, new_body).merge_lines()) conflicted = len(re.findall(PageOperationMixin.re_conflicted, merged)) > 0 if conflicted: raise ConflictError("Conflicted", base, new_body, merged) else: new_body = merged # delete rendered body, metadata, data cache cache.del_rendered_body(self.title) cache.del_hashbangs(self.title) cache.del_metadata(self.title) cache.del_data(self.title) # update model fields self.body = new_body self.modifier = user self.description = self.make_description(200) self.acl_read = new_md.get("read", "") self.acl_write = new_md.get("write", "") self.comment = comment if not dont_create_rev: self.revision += 1 if not force_update: self.updated_at = datetime.now() # publish pub_old = u"pub" in old_md pub_new = u"pub" in new_md pub_old_title = None pub_new_title = None if pub_old: pub_old_title = old_md["pub"] if pub_new: pub_new_title = new_md["pub"] if pub_old and pub_new and (pub_old_title != pub_new_title): # if target page is changed self._unpublish(save=False) self._publish(title=pub_new_title, save=False) else: if pub_new: self._publish(title=pub_new_title, save=False) else: self._unpublish(save=False) # update itemtype_path self.itemtype_path = schema.get_itemtype_path(new_md["schema"]) # save self.put() # create revision if not dont_create_rev: rev_key = self._rev_key() rev = WikiPageRevision( parent=rev_key, title=self.title, body=self.body, created_at=self.updated_at, revision=self.revision, comment=self.comment, modifier=self.modifier, acl_read=self.acl_read, acl_write=self.acl_write, ) rev.put() # deferred update schema data index new_data = self.data deferred.defer(self.rebuild_data_index_deferred, old_data, new_data) # update inlinks and outlinks old_redir = old_md.get("redirect") new_redir = new_md.get("redirect") self.update_links(old_redir, new_redir) # delete config and tittle cache if self.title == ".config": cache.del_config() if self.revision == 1: cache.del_titles() return True
def update_content(self, new_body, base_revision, comment, user=None, force_update=False): if not force_update and self.body == new_body: return False # delete rendered body cache cache.del_rendered_body(self.title) # update body old_md = self.metadata new_md = WikiPage.parse_metadata(new_body) # validate contents if u'pub' in new_md and u'redirect' in new_md: raise ValueError('You cannot use "pub" and "redirect" metadata at ' 'the same time.') if u'redirect' in new_md and len(WikiPage.remove_metadata(new_body).strip()) != 0: raise ValueError('Page with "redirect" metadata cannot have a body ' 'content.') if u'read' in new_md and new_md['content-type'] != 'text/x-markdown': raise ValueError('You cannot restrict read access of custom content-typed page.') if self.revision < base_revision: raise ValueError('Invalid revision number: %d' % base_revision) # update model fields if self.revision != base_revision: # perform 3-way merge if needed base = WikiPageRevision.query(WikiPageRevision.title == self.title, WikiPageRevision.revision == base_revision).get().body merged = ''.join(Merge3(base, self.body, new_body).merge_lines()) self.body = merged else: self.body = new_body self.modifier = user self.description = self.make_description(200) self.acl_read = new_md.get('read', '') self.acl_write = new_md.get('write', '') self.comment = comment self.revision += 1 if not force_update: self.updated_at = datetime.now() # publish pub_old = u'pub' in old_md pub_new = u'pub' in new_md pub_old_title = None pub_new_title = None if pub_old: pub_old_title = old_md['pub'] if pub_new: pub_new_title = new_md['pub'] if pub_old and pub_new and (pub_old_title != pub_new_title): # if target page is changed self._unpublish(save=False) self._publish(title=pub_new_title, save=False) else: if pub_new: self._publish(title=pub_new_title, save=False) else: self._unpublish(save=False) # update related pages if it's first time if self.revision == 1: for _ in range(5): self.update_related_links() # update itemtype_path self.itemtype_path = schema.get_itemtype_path(self.itemtype) # update hashbangs self.hashbangs = WikiPage.extract_hashbangs(self.rendered_body) # save self.put() # create revision rev_key = self._rev_key() rev = WikiPageRevision(parent=rev_key, title=self.title, body=self.body, created_at=self.updated_at, revision=self.revision, comment=self.comment, modifier=self.modifier, acl_read=self.acl_read, acl_write=self.acl_write) rev.put() # update inlinks and outlinks old_redir = old_md.get('redirect') new_redir = new_md.get('redirect') self.update_links(old_redir, new_redir) # invalidate cache cache.del_yaml(self.title) if self.revision == 1: cache.del_titles() return True
def test_every_itemtype_should_have_a_parent_except_for_root(self): for item in schema.SUPPORTED_SCHEMA.keys(): self.assertEqual('Thing/', schema.get_itemtype_path(item)[:6])