def parse_data(title, itemtype, body):
matches = {"name": title, "schema": schema.get_itemtype_path(itemtype)}
# parse data in yaml/schema section
m = re.search(PageOperationMixin.re_yaml_schema, body)
if m:
parsed_yaml = yaml.load(m.group(1))
if type(parsed_yaml) != dict:
raise ValueError("YAML must be a dictionary")
for name, value in parsed_yaml.items():
if name in matches:
if type(matches[name]) != list:
matches[name] = [matches[name]]
if type(value) == list:
matches[name] += value
else:
matches[name].append(value)
else:
matches[name] = value
# parse data embedded in body text
for m in re.finditer(WikiPage.re_data, body):
name = m.group("name")
value = m.group("value")
if name in matches:
if type(matches[name]) != list:
matches[name] = [matches[name]]
matches[name].append(value)
else:
matches[name] = value
return matches
def parse_data(cls, title, body, itemtype=u'Article'):
body = body.replace('\r\n', '\n')
default_data = {
'name': title,
'schema': schema.get_itemtype_path(itemtype)
}
# collect
yaml_data = cls.parse_schema_yaml(body)
body_data = pairs_to_dict((m.group('name'), m.group('value'))
for m in re.finditer(cls.re_data, body))
if itemtype == u'Article' or u'Article' in schema.get_schema(
itemtype)[u'ancestors']:
default_section = u'articleBody'
else:
default_section = u'longDescription'
section_data = cls.parse_sections(body, default_section)
# merge
data = merge_dicts([default_data, yaml_data, body_data, section_data])
# validation and type conversion
typed = schema.SchemaConverter.convert(itemtype, data)
return typed
def _evaluate_page_query_term(cls, name, value):
if name == "schema" and value.find("/") == -1:
value = schema.get_itemtype_path(value)
pages = {}
for index in SchemaDataIndex.query(SchemaDataIndex.name == name, SchemaDataIndex.value == value):
pages[index.title] = index.data
return pages
def _update_content_all(self, body, base_revision, comment, user, force_update, dont_create_rev, dont_defer):
# do not update if the body is not changed
if not force_update and self.body == body:
return False
# validate and prepare new contents
new_data, new_md = self.validate_new_content(base_revision, body, user)
new_body = self._merge_if_needed(base_revision, body)
# get old data and metadata
old_md = self.metadata.copy()
old_data = self.data.copy()
# delete caches
caching.del_rendered_body(self.title)
caching.del_hashbangs(self.title)
caching.del_metadata(self.title)
caching.del_data(self.title)
# update model and save
self.body = new_body
self.modifier = user
self.description = PageOperationMixin.make_description(new_body)
self.acl_read = new_md.get('read', '')
self.acl_write = new_md.get('write', '')
self.comment = comment
self.itemtype_path = schema.get_itemtype_path(new_md['schema'])
self._update_pub_state(new_md, old_md)
if not dont_create_rev:
self.revision += 1
if not force_update:
self.updated_at = datetime.now()
self.put()
# create revision
if not dont_create_rev:
rev_key = self._rev_key()
rev = WikiPageRevision(parent=rev_key, title=self.title, body=self.body,
created_at=self.updated_at, revision=self.revision,
comment=self.comment, modifier=self.modifier,
acl_read=self.acl_read, acl_write=self.acl_write)
rev.put()
# update inlinks, outlinks and schema data index
self.update_links_and_data(old_md.get('redirect'), new_md.get('redirect'), old_data, new_data, dont_defer)
# delete config cache
if self.title == '.config':
caching.del_config()
# delete title cache if it's a new page
if self.revision == 1:
caching.del_titles()
return True
def parse_data(cls, title, body, itemtype=u'Article'):
# collect data
default_data = {'name': title, 'schema': schema.get_itemtype_path(itemtype)}
yaml_data = cls.parse_schema_yaml(body)
body_data = pairs_to_dict((m.group('name'), m.group('value')) for m in re.finditer(cls.re_data, body))
data = merge_dicts([default_data, yaml_data, body_data])
# validation and type conversion
typed = schema.SchemaConverter.convert(itemtype, data)
return typed
def parse_data(cls, title, body, itemtype=u"Article"):
body = body.replace("\r\n", "\n")
default_data = {"name": title, "schema": schema.get_itemtype_path(itemtype)}
# collect
yaml_data = cls.parse_schema_yaml(body)
body_data = pairs_to_dict((m.group("name"), m.group("value")) for m in re.finditer(cls.re_data, body))
if itemtype == u"Article" or u"Article" in schema.get_schema(itemtype)[u"ancestors"]:
default_section = u"articleBody"
else:
default_section = u"longDescription"
section_data = cls.parse_sections(body, default_section)
# merge
data = merge_dicts([default_data, yaml_data, body_data, section_data])
# validation and type conversion
typed = schema.SchemaConverter.convert(itemtype, data)
return typed
def parse_data(cls, title, body, itemtype=u'Article'):
body = body.replace('\r\n', '\n')
default_data = {'name': title, 'schema': schema.get_itemtype_path(itemtype)}
# collect
yaml_data = cls.parse_schema_yaml(body)
body_data = pairs_to_dict((m.group('name'), m.group('value')) for m in re.finditer(cls.re_data, body))
if itemtype == u'Article' or u'Article' in schema.get_schema(itemtype)[u'ancestors']:
default_section = u'articleBody'
else:
default_section = u'longDescription'
section_data = cls.parse_sections(body, default_section)
# merge
data = merge_dicts([default_data, yaml_data, body_data, section_data])
# validation and type conversion
typed = schema.SchemaConverter.convert(itemtype, data)
return typed
def test_itemtype_path(self):
self.assertEqual('Thing/',
schema.get_itemtype_path('Thing'))
self.assertEqual('Thing/CreativeWork/Article/',
schema.get_itemtype_path('Article'))
def _evaluate_page_query_term(cls, name, value):
if name == 'schema' and value.find('/') == -1:
value = schema.get_itemtype_path(value)
return SchemaDataIndex.query_titles(name, value)
def _update_content_all(self, body, base_revision, comment, user, force_update, dont_create_rev):
# do not update if the body is not changed
if not force_update and self.body == body:
return False
now = datetime.utcnow().replace(tzinfo=utc)
# validate and prepare new contents
new_data, new_md = self.validate_new_content(base_revision, body, user)
new_body = self._merge_if_needed(base_revision, body)
# get old data and metadata
try:
old_md = self.metadata.copy()
except ValueError:
old_md = {}
try:
old_data = self.data.copy()
except ValueError:
old_data = {}
# delete caches
caching.del_rendered_body(self.title)
caching.del_hashbangs(self.title)
caching.del_metadata(self.title)
caching.del_data(self.title)
# update model and save
self.body = new_body
if user and not user.is_anonymous():
self.modifier = user
self.description = PageOperationMixin.make_description(new_body)
self.acl_read = new_md.get('read', '')
self.acl_write = new_md.get('write', '')
self.comment = comment
self.itemtype_path = schema.get_itemtype_path(new_md['schema'])
self._update_pub_state(new_md, old_md)
if not dont_create_rev:
self.revision += 1
if not force_update:
self.updated_at = now
self.save()
# create revision
if not dont_create_rev:
rev = WikiPageRevision(page=self, title=self.title, body=self.body,
created_at=self.updated_at, revision=self.revision,
comment=self.comment, modifier=self.modifier)
rev.save()
self.update_links_and_data(old_md.get('redirect'), new_md.get('redirect'), old_data, new_data)
# delete config cache
if self.title == '.config':
caching.del_config()
# delete title cache if it's a new page
if self.revision == 1:
caching.del_titles()
return True
def _evaluate_page_query_term(cls, name, value):
if name == 'schema' and value.find('/') == -1:
value = schema.get_itemtype_path(value)
return SchemaDataIndex.query_titles(name, value)
def _update_content_all(self, body, base_revision, comment, user,
force_update, dont_create_rev, dont_defer):
# do not update if the body is not changed
if not force_update and self.body == body:
return False
now = datetime.now()
# validate and prepare new contents
new_data, new_md = self.validate_new_content(base_revision, body, user)
new_body = self._merge_if_needed(base_revision, body)
# get old data and metadata
try:
old_md = self.metadata.copy()
except ValueError:
old_md = {}
try:
old_data = self.data.copy()
except ValueError:
old_data = {}
# delete caches
caching.del_rendered_body(self.title)
caching.del_hashbangs(self.title)
caching.del_metadata(self.title)
caching.del_data(self.title)
# update model and save
self.body = new_body
self.modifier = user
self.description = PageOperationMixin.make_description(new_body)
self.acl_read = new_md.get('read', '')
self.acl_write = new_md.get('write', '')
self.comment = comment
self.itemtype_path = schema.get_itemtype_path(new_md['schema'])
self._update_pub_state(new_md, old_md)
if not dont_create_rev:
self.revision += 1
if not force_update:
self.updated_at = now
self.put()
# create revision
if not dont_create_rev:
rev_key = self._rev_key()
rev = WikiPageRevision(parent=rev_key,
title=self.title,
body=self.body,
created_at=self.updated_at,
revision=self.revision,
comment=self.comment,
modifier=self.modifier,
acl_read=self.acl_read,
acl_write=self.acl_write)
rev.put()
# update inlinks, outlinks and schema data index
self.update_links_and_data(old_md.get('redirect'),
new_md.get('redirect'), old_data, new_data,
dont_defer)
# delete config cache
if self.title == '.config':
caching.del_config()
# delete title cache if it's a new page
if self.revision == 1:
caching.del_titles()
return True
def test_every_itemtype_should_have_a_parent_except_for_root(self):
for item in schema.SUPPORTED_SCHEMA.keys():
self.assertEqual("Thing/", schema.get_itemtype_path(item)[:6])
def test_itemtype_path(self):
self.assertEqual("Thing/", schema.get_itemtype_path("Thing"))
self.assertEqual("Thing/CreativeWork/Article/", schema.get_itemtype_path("Article"))
def test_itemtype_path(self):
self.assertEqual('Thing/', schema.get_itemtype_path('Thing'))
self.assertEqual('Thing/CreativeWork/Article/',
schema.get_itemtype_path('Article'))
def update_content(self, new_body, base_revision, comment="", user=None, force_update=False, dont_create_rev=False):
if not force_update and self.body == new_body:
return False
# get old data amd metadata
old_md = self.metadata
old_data = self.data
# validate contents
## validate schema data
new_md = PageOperationMixin.parse_metadata(new_body)
try:
PageOperationMixin.parse_data(self.title, new_md["schema"], new_body)
except Exception:
raise ValueError("Invalid schema data")
## validate metadata
if u"pub" in new_md and u"redirect" in new_md:
raise ValueError('You cannot use "pub" and "redirect" metadata at ' "the same time.")
if u"redirect" in new_md and len(PageOperationMixin.remove_metadata(new_body).strip()) != 0:
raise ValueError('Page with "redirect" metadata cannot have a body ' "content.")
if u"read" in new_md and new_md["content-type"] != "text/x-markdown":
raise ValueError("You cannot restrict read access of custom content-typed page.")
## validate revision
if self.revision < base_revision:
raise ValueError("Invalid revision number: %d" % base_revision)
## validate ToC
if not TocGenerator(md.convert(new_body)).validate():
raise ValueError("Duplicate paths not allowed")
if self.revision != base_revision:
# perform 3-way merge if needed
base = (
WikiPageRevision.query(WikiPageRevision.title == self.title, WikiPageRevision.revision == base_revision)
.get()
.body
)
merged = "".join(Merge3(base, self.body, new_body).merge_lines())
conflicted = len(re.findall(PageOperationMixin.re_conflicted, merged)) > 0
if conflicted:
raise ConflictError("Conflicted", base, new_body, merged)
else:
new_body = merged
# delete rendered body, metadata, data cache
cache.del_rendered_body(self.title)
cache.del_hashbangs(self.title)
cache.del_metadata(self.title)
cache.del_data(self.title)
# update model fields
self.body = new_body
self.modifier = user
self.description = self.make_description(200)
self.acl_read = new_md.get("read", "")
self.acl_write = new_md.get("write", "")
self.comment = comment
if not dont_create_rev:
self.revision += 1
if not force_update:
self.updated_at = datetime.now()
# publish
pub_old = u"pub" in old_md
pub_new = u"pub" in new_md
pub_old_title = None
pub_new_title = None
if pub_old:
pub_old_title = old_md["pub"]
if pub_new:
pub_new_title = new_md["pub"]
if pub_old and pub_new and (pub_old_title != pub_new_title):
# if target page is changed
self._unpublish(save=False)
self._publish(title=pub_new_title, save=False)
else:
if pub_new:
self._publish(title=pub_new_title, save=False)
else:
self._unpublish(save=False)
# update itemtype_path
self.itemtype_path = schema.get_itemtype_path(new_md["schema"])
# save
self.put()
# create revision
if not dont_create_rev:
rev_key = self._rev_key()
rev = WikiPageRevision(
parent=rev_key,
title=self.title,
body=self.body,
created_at=self.updated_at,
revision=self.revision,
comment=self.comment,
modifier=self.modifier,
acl_read=self.acl_read,
acl_write=self.acl_write,
)
rev.put()
# deferred update schema data index
new_data = self.data
deferred.defer(self.rebuild_data_index_deferred, old_data, new_data)
# update inlinks and outlinks
old_redir = old_md.get("redirect")
new_redir = new_md.get("redirect")
self.update_links(old_redir, new_redir)
# delete config and tittle cache
if self.title == ".config":
cache.del_config()
if self.revision == 1:
cache.del_titles()
return True
def update_content(self, new_body, base_revision, comment, user=None, force_update=False):
if not force_update and self.body == new_body:
return False
# delete rendered body cache
cache.del_rendered_body(self.title)
# update body
old_md = self.metadata
new_md = WikiPage.parse_metadata(new_body)
# validate contents
if u'pub' in new_md and u'redirect' in new_md:
raise ValueError('You cannot use "pub" and "redirect" metadata at '
'the same time.')
if u'redirect' in new_md and len(WikiPage.remove_metadata(new_body).strip()) != 0:
raise ValueError('Page with "redirect" metadata cannot have a body '
'content.')
if u'read' in new_md and new_md['content-type'] != 'text/x-markdown':
raise ValueError('You cannot restrict read access of custom content-typed page.')
if self.revision < base_revision:
raise ValueError('Invalid revision number: %d' % base_revision)
# update model fields
if self.revision != base_revision:
# perform 3-way merge if needed
base = WikiPageRevision.query(WikiPageRevision.title == self.title, WikiPageRevision.revision == base_revision).get().body
merged = ''.join(Merge3(base, self.body, new_body).merge_lines())
self.body = merged
else:
self.body = new_body
self.modifier = user
self.description = self.make_description(200)
self.acl_read = new_md.get('read', '')
self.acl_write = new_md.get('write', '')
self.comment = comment
self.revision += 1
if not force_update:
self.updated_at = datetime.now()
# publish
pub_old = u'pub' in old_md
pub_new = u'pub' in new_md
pub_old_title = None
pub_new_title = None
if pub_old:
pub_old_title = old_md['pub']
if pub_new:
pub_new_title = new_md['pub']
if pub_old and pub_new and (pub_old_title != pub_new_title):
# if target page is changed
self._unpublish(save=False)
self._publish(title=pub_new_title, save=False)
else:
if pub_new:
self._publish(title=pub_new_title, save=False)
else:
self._unpublish(save=False)
# update related pages if it's first time
if self.revision == 1:
for _ in range(5):
self.update_related_links()
# update itemtype_path
self.itemtype_path = schema.get_itemtype_path(self.itemtype)
# update hashbangs
self.hashbangs = WikiPage.extract_hashbangs(self.rendered_body)
# save
self.put()
# create revision
rev_key = self._rev_key()
rev = WikiPageRevision(parent=rev_key, title=self.title, body=self.body,
created_at=self.updated_at, revision=self.revision,
comment=self.comment, modifier=self.modifier,
acl_read=self.acl_read, acl_write=self.acl_write)
rev.put()
# update inlinks and outlinks
old_redir = old_md.get('redirect')
new_redir = new_md.get('redirect')
self.update_links(old_redir, new_redir)
# invalidate cache
cache.del_yaml(self.title)
if self.revision == 1:
cache.del_titles()
return True
def test_every_itemtype_should_have_a_parent_except_for_root(self):
for item in schema.SUPPORTED_SCHEMA.keys():
self.assertEqual('Thing/', schema.get_itemtype_path(item)[:6])
