def test_update_manifest(self):
opts = {}
shutil.copy(
os.path.join(settings.BASE_DIR, 'fixtures', 'tuto', 'balise_audio', 'manifest.json'),
os.path.join(settings.BASE_DIR, 'fixtures', 'tuto', 'balise_audio', 'manifest2.json')
)
LicenceFactory(code='CC BY')
args = [os.path.join(settings.BASE_DIR, 'fixtures', 'tuto', 'balise_audio', 'manifest2.json')]
call_command('upgrade_manifest_to_v2', *args, **opts)
manifest = open(os.path.join(settings.BASE_DIR, 'fixtures', 'tuto', 'balise_audio', 'manifest2.json'), 'r')
json = json_handler.loads(manifest.read())
self.assertTrue('version' in json)
self.assertTrue('licence' in json)
self.assertTrue('children' in json)
self.assertEqual(len(json['children']), 3)
self.assertEqual(json['children'][0]['object'], 'extract')
os.unlink(args[0])
args = [os.path.join(settings.BASE_DIR, 'fixtures', 'tuto', 'big_tuto_v1', 'manifest2.json')]
shutil.copy(
os.path.join(settings.BASE_DIR, 'fixtures', 'tuto', 'big_tuto_v1', 'manifest.json'),
os.path.join(settings.BASE_DIR, 'fixtures', 'tuto', 'big_tuto_v1', 'manifest2.json')
)
call_command('upgrade_manifest_to_v2', *args, **opts)
manifest = open(os.path.join(settings.BASE_DIR, 'fixtures', 'tuto', 'big_tuto_v1', 'manifest2.json'), 'r')
json = json_handler.loads(manifest.read())
os.unlink(args[0])
self.assertTrue('version' in json)
self.assertTrue('licence' in json)
self.assertTrue('children' in json)
self.assertEqual(len(json['children']), 5)
self.assertEqual(json['children'][0]['object'], 'container')
self.assertEqual(len(json['children'][0]['children']), 3)
self.assertEqual(len(json['children'][0]['children'][0]['children']), 3)
args = [os.path.join(settings.BASE_DIR, 'fixtures', 'tuto', 'article_v1', 'manifest2.json')]
shutil.copy(
os.path.join(settings.BASE_DIR, 'fixtures', 'tuto', 'article_v1', 'manifest.json'),
os.path.join(settings.BASE_DIR, 'fixtures', 'tuto', 'article_v1', 'manifest2.json')
)
call_command('upgrade_manifest_to_v2', *args, **opts)
manifest = open(os.path.join(settings.BASE_DIR, 'fixtures', 'tuto', 'article_v1', 'manifest2.json'), 'r')
json = json_handler.loads(manifest.read())
self.assertTrue('version' in json)
self.assertTrue('licence' in json)
self.assertTrue('children' in json)
self.assertEqual(len(json['children']), 1)
os.unlink(args[0])
def test_get_similar_topics(self):
"""Get similar topics lists"""
if not self.manager.connected_to_es:
return
text = "Clem ne se mange pas"
topic_1 = TopicFactory(forum=self.forum, author=self.user, title=text)
post_1 = PostFactory(topic=topic_1, author=self.user, position=1)
post_1.text = post_1.text_html = text
post_1.save()
text = "Clem est la meilleure mascotte"
topic_2 = TopicFactory(forum=self.forum, author=self.user, title=text)
post_2 = PostFactory(topic=topic_2, author=self.user, position=1)
post_2.text = post_1.text_html = text
post_2.save()
# 1. Should not get any result
result = self.client.get(reverse("search:similar") + "?q=est",
follow=False)
self.assertEqual(result.status_code, 200)
content = json_handler.loads(result.content.decode("utf-8"))
self.assertEqual(len(content["results"]), 0)
# index
for model in self.indexable:
if model is FakeChapter:
continue
self.manager.es_bulk_indexing_of_model(model)
self.manager.refresh_index()
# 2. Should get exactly one result
result = self.client.get(reverse("search:similar") + "?q=mange",
follow=False)
self.assertEqual(result.status_code, 200)
content = json_handler.loads(result.content.decode("utf-8"))
self.assertEqual(len(content["results"]), 1)
# 2. Should get exactly two results
result = self.client.get(reverse("search:similar") + "?q=Clem",
follow=False)
self.assertEqual(result.status_code, 200)
content = json_handler.loads(result.content.decode("utf-8"))
self.assertEqual(len(content["results"]), 2)
def test_update_manifest(self):
opts = {}
path_manifest1 = settings.BASE_DIR / 'fixtures' / 'tuto' / 'balise_audio' / 'manifest.json'
path_manifest2 = settings.BASE_DIR / 'fixtures' / 'tuto' / 'balise_audio' / 'manifest2.json'
args = [str(path_manifest2)]
shutil.copy(path_manifest1, path_manifest2)
LicenceFactory(code='CC BY')
call_command('upgrade_manifest_to_v2', *args, **opts)
manifest = path_manifest2.open('r')
json = json_handler.loads(manifest.read())
self.assertTrue('version' in json)
self.assertTrue('licence' in json)
self.assertTrue('children' in json)
self.assertEqual(len(json['children']), 3)
self.assertEqual(json['children'][0]['object'], 'extract')
os.unlink(args[0])
path_manifest1 = settings.BASE_DIR / 'fixtures' / 'tuto' / 'big_tuto_v1' / 'manifest.json'
path_manifest2 = settings.BASE_DIR / 'fixtures' / 'tuto' / 'big_tuto_v1' / 'manifest2.json'
args = [str(path_manifest2)]
shutil.copy(path_manifest1, path_manifest2)
call_command('upgrade_manifest_to_v2', *args, **opts)
manifest = path_manifest2.open('r')
json = json_handler.loads(manifest.read())
os.unlink(args[0])
self.assertTrue('version' in json)
self.assertTrue('licence' in json)
self.assertTrue('children' in json)
self.assertEqual(len(json['children']), 5)
self.assertEqual(json['children'][0]['object'], 'container')
self.assertEqual(len(json['children'][0]['children']), 3)
self.assertEqual(len(json['children'][0]['children'][0]['children']),
3)
path_manifest1 = settings.BASE_DIR / 'fixtures' / 'tuto' / 'article_v1' / 'manifest.json'
path_manifest2 = settings.BASE_DIR / 'fixtures' / 'tuto' / 'article_v1' / 'manifest2.json'
args = [path_manifest2]
shutil.copy(path_manifest1, path_manifest2)
call_command('upgrade_manifest_to_v2', *args, **opts)
manifest = path_manifest2.open('r')
json = json_handler.loads(manifest.read())
self.assertTrue('version' in json)
self.assertTrue('licence' in json)
self.assertTrue('children' in json)
self.assertEqual(len(json['children']), 1)
os.unlink(args[0])
def test_update_manifest(self):
opts = {}
path_manifest1 = settings.BASE_DIR / "fixtures" / "tuto" / "balise_audio" / "manifest.json"
path_manifest2 = settings.BASE_DIR / "fixtures" / "tuto" / "balise_audio" / "manifest2.json"
args = [str(path_manifest2)]
shutil.copy(path_manifest1, path_manifest2)
LicenceFactory(code="CC BY")
call_command("upgrade_manifest_to_v2", *args, **opts)
manifest = path_manifest2.open("r")
json = json_handler.loads(manifest.read())
self.assertTrue("version" in json)
self.assertTrue("licence" in json)
self.assertTrue("children" in json)
self.assertEqual(len(json["children"]), 3)
self.assertEqual(json["children"][0]["object"], "extract")
os.unlink(args[0])
path_manifest1 = settings.BASE_DIR / "fixtures" / "tuto" / "big_tuto_v1" / "manifest.json"
path_manifest2 = settings.BASE_DIR / "fixtures" / "tuto" / "big_tuto_v1" / "manifest2.json"
args = [str(path_manifest2)]
shutil.copy(path_manifest1, path_manifest2)
call_command("upgrade_manifest_to_v2", *args, **opts)
manifest = path_manifest2.open("r")
json = json_handler.loads(manifest.read())
os.unlink(args[0])
self.assertTrue("version" in json)
self.assertTrue("licence" in json)
self.assertTrue("children" in json)
self.assertEqual(len(json["children"]), 5)
self.assertEqual(json["children"][0]["object"], "container")
self.assertEqual(len(json["children"][0]["children"]), 3)
self.assertEqual(len(json["children"][0]["children"][0]["children"]),
3)
path_manifest1 = settings.BASE_DIR / "fixtures" / "tuto" / "article_v1" / "manifest.json"
path_manifest2 = settings.BASE_DIR / "fixtures" / "tuto" / "article_v1" / "manifest2.json"
args = [path_manifest2]
shutil.copy(path_manifest1, path_manifest2)
call_command("upgrade_manifest_to_v2", *args, **opts)
manifest = path_manifest2.open("r")
json = json_handler.loads(manifest.read())
self.assertTrue("version" in json)
self.assertTrue("licence" in json)
self.assertTrue("children" in json)
self.assertEqual(len(json["children"]), 1)
os.unlink(args[0])
def test_get_similar_topics(self):
"""Get similar topics lists"""
if not self.manager.connected_to_es:
return
text = 'Clem ne se mange pas'
topic_1 = TopicFactory(forum=self.forum, author=self.user, title=text)
post_1 = PostFactory(topic=topic_1, author=self.user, position=1)
post_1.text = post_1.text_html = text
post_1.save()
text = 'Clem est la meilleure mascotte'
topic_2 = TopicFactory(forum=self.forum, author=self.user, title=text)
post_2 = PostFactory(topic=topic_2, author=self.user, position=1)
post_2.text = post_1.text_html = text
post_2.save()
# 1. Should not get any result
result = self.client.get(reverse('search:similar') + '?q=est', follow=False)
self.assertEqual(result.status_code, 200)
content = json_handler.loads(result.content.decode('utf-8'))
self.assertEqual(len(content['results']), 0)
# index
for model in self.indexable:
if model is FakeChapter:
continue
self.manager.es_bulk_indexing_of_model(model)
self.manager.refresh_index()
# 2. Should get exactly one result
result = self.client.get(reverse('search:similar') + '?q=mange', follow=False)
self.assertEqual(result.status_code, 200)
content = json_handler.loads(result.content.decode('utf-8'))
self.assertEqual(len(content['results']), 1)
# 2. Should get exactly two results
result = self.client.get(reverse('search:similar') + '?q=Clem', follow=False)
self.assertEqual(result.status_code, 200)
content = json_handler.loads(result.content.decode('utf-8'))
self.assertEqual(len(content['results']), 2)
def extract_content_from_zip(zip_archive):
"""Check if the data in the zip file are coherent
:param zip_archive: the zip archive to analyze
:type zip_archive: zipfile.ZipFile
:raise BadArchiveError: if something is wrong in the archive
:return: the content in the archive
:rtype: VersionedContent
"""
try:
manifest = str(zip_archive.read("manifest.json"), "utf-8")
except KeyError:
raise BadArchiveError(
_("Cette archive ne contient pas de fichier manifest.json."))
except UnicodeDecodeError:
raise BadArchiveError(
_("L'encodage du manifest.json n'est pas de l'UTF-8."))
# is the manifest ok ?
try:
json_ = json_handler.loads(manifest)
except ValueError:
raise BadArchiveError(
_("Une erreur est survenue durant la lecture du manifest, "
"vérifiez qu'il s'agit de JSON correctement formaté."))
try:
versioned = get_content_from_json(
json_,
None,
"",
max_title_len=PublishableContent._meta.get_field(
"title").max_length)
except BadManifestError as e:
raise BadArchiveError(e.message)
except InvalidSlugError as e:
e1 = _('Le slug "{}" est invalide').format(e)
e2 = ""
if e.had_source:
e2 = _(' (slug généré à partir de "{}")').format(e.source)
raise BadArchiveError(_("{}{} !").format(e1, e2))
except Exception as e:
raise BadArchiveError(
_("Une erreur est survenue lors de la lecture de l'archive : {}."
).format(e))
# is there everything in the archive ?
for f in UpdateContentWithArchive.walk_content(versioned):
try:
zip_archive.getinfo(f)
except KeyError:
raise BadArchiveError(
_("Le fichier '{}' n'existe pas dans l'archive.").format(
f))
return versioned
def form_valid(self, form):
if self.request.FILES["archive"]:
try:
zfile = zipfile.ZipFile(self.request.FILES["archive"], "r")
except zipfile.BadZipfile:
messages.error(self.request,
_("Cette archive n'est pas au format ZIP."))
return self.form_invalid(form)
try:
new_content = UpdateContentWithArchive.extract_content_from_zip(
zfile)
except BadArchiveError as e:
messages.error(self.request, e.message)
return super(CreateContentFromArchive, self).form_invalid(form)
except KeyError as e:
messages.error(
self.request,
_(e.message + " n'est pas correctement renseigné."))
return super(CreateContentFromArchive, self).form_invalid(form)
else:
# Warn the user if the license has been changed
manifest = json_handler.loads(
str(zfile.read("manifest.json"), "utf-8"))
if new_content.licence and "licence" in manifest and manifest[
"licence"] != new_content.licence.code:
messages.info(
self.request,
_("la licence « {} » a été appliquée.".format(
new_content.licence.code)))
# first, create DB object (in order to get a slug)
self.object = PublishableContent()
self.object.title = new_content.title
self.object.description = new_content.description
self.object.licence = new_content.licence
self.object.type = new_content.type # change of type is then allowed !!
self.object.creation_date = datetime.now()
self.object.save()
new_content.slug = self.object.slug # new slug (choosen via DB)
# Creating the gallery
gal = Gallery()
gal.title = new_content.title
gal.slug = slugify(new_content.title)
gal.pubdate = datetime.now()
gal.save()
# Attach user to gallery
self.object.gallery = gal
self.object.save()
# Add subcategories on tutorial
for subcat in form.cleaned_data["subcategory"]:
self.object.subcategory.add(subcat)
# We need to save the tutorial before changing its author list since it's a many-to-many relationship
self.object.authors.add(self.request.user)
self.object.save()
self.object.ensure_author_gallery()
# ok, now we can import
introduction = ""
conclusion = ""
if new_content.introduction:
introduction = str(zfile.read(new_content.introduction),
"utf-8")
if new_content.conclusion:
conclusion = str(zfile.read(new_content.conclusion),
"utf-8")
commit_message = _("Création de « {} »").format(
new_content.title)
init_new_repo(self.object,
introduction,
conclusion,
commit_message=commit_message)
# copy all:
versioned = self.object.load_version()
try:
UpdateContentWithArchive.update_from_new_version_in_zip(
versioned, new_content, zfile)
except BadArchiveError as e:
self.object.delete() # abort content creation
messages.error(self.request, e.message)
return super(CreateContentFromArchive,
self).form_invalid(form)
# and end up by a commit !!
commit_message = form.cleaned_data["msg_commit"]
if not commit_message:
commit_message = _(
"Importation d'une archive contenant « {} »").format(
new_content.title)
versioned.slug = self.object.slug # force slug to ensure path resolution
sha = versioned.repo_update(
versioned.title,
versioned.get_introduction(),
versioned.get_conclusion(),
commit_message,
update_slug=True,
)
# This HAVE TO happen after commiting files (if not, content are None)
if "image_archive" in self.request.FILES:
try:
zfile = zipfile.ZipFile(
self.request.FILES["image_archive"], "r")
except zipfile.BadZipfile:
messages.error(
self.request,
_("L'archive contenant les images n'est pas au format ZIP."
))
return self.form_invalid(form)
UpdateContentWithArchive.use_images_from_archive(
self.request, zfile, versioned, self.object.gallery)
commit_message = _(
"Utilisation des images de l'archive pour « {} »"
).format(new_content.title)
sha = versioned.commit_changes(
commit_message) # another commit
# of course, need to update sha
self.object.sha_draft = sha
self.object.update_date = datetime.now()
self.object.save()
self.success_url = reverse("content:view",
args=[versioned.pk, versioned.slug])
return super(CreateContentFromArchive, self).form_valid(form)
def form_valid(self, form):
versioned = self.versioned_object
if self.request.FILES["archive"]:
try:
zfile = zipfile.ZipFile(self.request.FILES["archive"], "r")
except zipfile.BadZipfile:
messages.error(self.request,
_("Cette archive n'est pas au format ZIP."))
return super(UpdateContentWithArchive, self).form_invalid(form)
try:
new_version = UpdateContentWithArchive.extract_content_from_zip(
zfile)
except BadArchiveError as e:
messages.error(self.request, e.message)
return super(UpdateContentWithArchive, self).form_invalid(form)
else:
# Warn the user if the license has been changed
manifest = json_handler.loads(
str(zfile.read("manifest.json"), "utf-8"))
if new_version.licence and "licence" in manifest and manifest[
"licence"] != new_version.licence.code:
messages.info(
self.request,
_("la licence « {} » a été appliquée.").format(
new_version.licence.code))
# first, update DB object (in order to get a new slug if needed)
title_is_changed = self.object.title != new_version.title
self.object.title = new_version.title
self.object.description = new_version.description
self.object.licence = new_version.licence
self.object.type = new_version.type # change of type is then allowed !!
self.object.save(force_slug_update=title_is_changed)
new_version.slug = self.object.slug # new slug if any !!
# ok, then, let's do the import. First, remove everything in the repository
while True:
if versioned.children:
versioned.children[0].repo_delete(do_commit=False)
else:
break # this weird construction ensure that everything is removed
versioned.slug_pool = default_slug_pool(
) # slug pool to its initial value (to avoid weird stuffs)
# start by copying extra information
self.object.insert_data_in_versioned(
versioned) # better have a clean version of those one
versioned.description = new_version.description
versioned.type = new_version.type
versioned.licence = new_version.licence
# update container (and repo)
introduction = ""
conclusion = ""
if new_version.introduction:
introduction = str(zfile.read(new_version.introduction),
"utf-8")
if new_version.conclusion:
conclusion = str(zfile.read(new_version.conclusion),
"utf-8")
versioned.repo_update_top_container(new_version.title,
new_version.slug,
introduction,
conclusion,
do_commit=False)
# then do the dirty job:
try:
UpdateContentWithArchive.update_from_new_version_in_zip(
versioned, new_version, zfile)
except BadArchiveError as e:
versioned.repository.index.reset()
messages.error(self.request, e.message)
return super(UpdateContentWithArchive,
self).form_invalid(form)
# and end up by a commit !!
commit_message = form.cleaned_data["msg_commit"]
if not commit_message:
commit_message = _(
"Importation d'une archive contenant « {} ».").format(
new_version.title)
sha = versioned.commit_changes(commit_message)
# now, use the images from the archive if provided. To work, this HAVE TO happen after commiting files !
if "image_archive" in self.request.FILES:
try:
zfile = zipfile.ZipFile(
self.request.FILES["image_archive"], "r")
except zipfile.BadZipfile:
messages.error(
self.request,
_("L'archive contenant les images n'est pas au format ZIP."
))
return self.form_invalid(form)
UpdateContentWithArchive.use_images_from_archive(
self.request, zfile, versioned, self.object.gallery)
commit_message = _(
"Utilisation des images de l'archive pour « {} »"
).format(new_version.title)
sha = versioned.commit_changes(
commit_message) # another commit
# of course, need to update sha
self.object.sha_draft = sha
self.object.update_date = datetime.now()
self.object.save()
self.success_url = reverse("content:view",
args=[versioned.pk, versioned.slug])
return super(UpdateContentWithArchive, self).form_valid(form)
def test_update_manifest(self):
opts = {}
shutil.copy(
os.path.join(BASE_DIR, 'fixtures', 'tuto', 'balise_audio',
'manifest.json'),
os.path.join(BASE_DIR, 'fixtures', 'tuto', 'balise_audio',
'manifest2.json'))
LicenceFactory(code='CC BY')
args = [
os.path.join(BASE_DIR, 'fixtures', 'tuto', 'balise_audio',
'manifest2.json')
]
call_command('upgrade_manifest_to_v2', *args, **opts)
manifest = open(
os.path.join(BASE_DIR, 'fixtures', 'tuto', 'balise_audio',
'manifest2.json'), 'r')
json = json_handler.loads(manifest.read())
self.assertTrue('version' in json)
self.assertTrue('licence' in json)
self.assertTrue('children' in json)
self.assertEqual(len(json['children']), 3)
self.assertEqual(json['children'][0]['object'], 'extract')
os.unlink(args[0])
args = [
os.path.join(BASE_DIR, 'fixtures', 'tuto', 'big_tuto_v1',
'manifest2.json')
]
shutil.copy(
os.path.join(BASE_DIR, 'fixtures', 'tuto', 'big_tuto_v1',
'manifest.json'),
os.path.join(BASE_DIR, 'fixtures', 'tuto', 'big_tuto_v1',
'manifest2.json'))
call_command('upgrade_manifest_to_v2', *args, **opts)
manifest = open(
os.path.join(BASE_DIR, 'fixtures', 'tuto', 'big_tuto_v1',
'manifest2.json'), 'r')
json = json_handler.loads(manifest.read())
os.unlink(args[0])
self.assertTrue('version' in json)
self.assertTrue('licence' in json)
self.assertTrue('children' in json)
self.assertEqual(len(json['children']), 5)
self.assertEqual(json['children'][0]['object'], 'container')
self.assertEqual(len(json['children'][0]['children']), 3)
self.assertEqual(len(json['children'][0]['children'][0]['children']),
3)
args = [
os.path.join(BASE_DIR, 'fixtures', 'tuto', 'article_v1',
'manifest2.json')
]
shutil.copy(
os.path.join(BASE_DIR, 'fixtures', 'tuto', 'article_v1',
'manifest.json'),
os.path.join(BASE_DIR, 'fixtures', 'tuto', 'article_v1',
'manifest2.json'))
call_command('upgrade_manifest_to_v2', *args, **opts)
manifest = open(
os.path.join(BASE_DIR, 'fixtures', 'tuto', 'article_v1',
'manifest2.json'), 'r')
json = json_handler.loads(manifest.read())
self.assertTrue('version' in json)
self.assertTrue('licence' in json)
self.assertTrue('children' in json)
self.assertEqual(len(json['children']), 1)
os.unlink(args[0])
def load_version(self, sha=None, public=None):
"""Using git, load a specific version of the content. if ``sha`` is ``None``,
the draft/public version is used (if ``public`` is ``True``).
.. attention::
for practical reason, the returned object is filled with information from DB.
:param sha: version
:param public: if set with the right object, return the public version
:type public: PublishedContent
:raise BadObject: if sha is not None and related version could not be found
:raise OSError: if the path to the repository is wrong
:raise NotAPublicVersion: if the sha does not correspond to a public version
:return: the versioned content
:rtype: zds.tutorialv2.models.versioned.VersionedContent
"""
# load the good manifest.json
if sha is None:
if not public:
sha = self.sha_draft
else:
sha = self.sha_public
max_title_length = PublishableContent._meta.get_field('title').max_length
if public and isinstance(public, PublishedContent): # use the public (altered and not versioned) repository
path = public.get_prod_path()
slug = public.content_public_slug
if not os.path.isdir(path):
raise OSError(path)
if sha != public.sha_public:
raise NotAPublicVersion
with open(os.path.join(path, 'manifest.json'), 'r', encoding='utf-8') as manifest:
json = json_handler.loads(manifest.read())
versioned = get_content_from_json(
json,
public.sha_public,
slug,
public=True,
max_title_len=max_title_length,
hint_licence=self.licence,
)
else: # draft version, use the repository (slower, but allows manipulation)
path = self.get_repo_path()
if not os.path.isdir(path):
raise OSError(path)
repo = Repo(path)
data = get_blob(repo.commit(sha).tree, 'manifest.json')
try:
json = json_handler.loads(data)
except ValueError:
raise BadManifestError(
_('Une erreur est survenue lors de la lecture du manifest.json, est-ce du JSON ?'))
versioned = get_content_from_json(json, sha, self.slug, max_title_len=max_title_length)
self.insert_data_in_versioned(versioned)
return versioned