def test_update_manifest(self): opts = {} shutil.copy( os.path.join(settings.BASE_DIR, 'fixtures', 'tuto', 'balise_audio', 'manifest.json'), os.path.join(settings.BASE_DIR, 'fixtures', 'tuto', 'balise_audio', 'manifest2.json') ) LicenceFactory(code='CC BY') args = [os.path.join(settings.BASE_DIR, 'fixtures', 'tuto', 'balise_audio', 'manifest2.json')] call_command('upgrade_manifest_to_v2', *args, **opts) manifest = open(os.path.join(settings.BASE_DIR, 'fixtures', 'tuto', 'balise_audio', 'manifest2.json'), 'r') json = json_handler.loads(manifest.read()) self.assertTrue('version' in json) self.assertTrue('licence' in json) self.assertTrue('children' in json) self.assertEqual(len(json['children']), 3) self.assertEqual(json['children'][0]['object'], 'extract') os.unlink(args[0]) args = [os.path.join(settings.BASE_DIR, 'fixtures', 'tuto', 'big_tuto_v1', 'manifest2.json')] shutil.copy( os.path.join(settings.BASE_DIR, 'fixtures', 'tuto', 'big_tuto_v1', 'manifest.json'), os.path.join(settings.BASE_DIR, 'fixtures', 'tuto', 'big_tuto_v1', 'manifest2.json') ) call_command('upgrade_manifest_to_v2', *args, **opts) manifest = open(os.path.join(settings.BASE_DIR, 'fixtures', 'tuto', 'big_tuto_v1', 'manifest2.json'), 'r') json = json_handler.loads(manifest.read()) os.unlink(args[0]) self.assertTrue('version' in json) self.assertTrue('licence' in json) self.assertTrue('children' in json) self.assertEqual(len(json['children']), 5) self.assertEqual(json['children'][0]['object'], 'container') self.assertEqual(len(json['children'][0]['children']), 3) self.assertEqual(len(json['children'][0]['children'][0]['children']), 3) args = [os.path.join(settings.BASE_DIR, 'fixtures', 'tuto', 'article_v1', 'manifest2.json')] shutil.copy( os.path.join(settings.BASE_DIR, 'fixtures', 'tuto', 'article_v1', 'manifest.json'), os.path.join(settings.BASE_DIR, 'fixtures', 'tuto', 'article_v1', 'manifest2.json') ) call_command('upgrade_manifest_to_v2', *args, **opts) manifest = open(os.path.join(settings.BASE_DIR, 'fixtures', 'tuto', 'article_v1', 'manifest2.json'), 'r') json = json_handler.loads(manifest.read()) self.assertTrue('version' in json) self.assertTrue('licence' in json) self.assertTrue('children' in json) self.assertEqual(len(json['children']), 1) os.unlink(args[0])
def test_get_similar_topics(self): """Get similar topics lists""" if not self.manager.connected_to_es: return text = "Clem ne se mange pas" topic_1 = TopicFactory(forum=self.forum, author=self.user, title=text) post_1 = PostFactory(topic=topic_1, author=self.user, position=1) post_1.text = post_1.text_html = text post_1.save() text = "Clem est la meilleure mascotte" topic_2 = TopicFactory(forum=self.forum, author=self.user, title=text) post_2 = PostFactory(topic=topic_2, author=self.user, position=1) post_2.text = post_1.text_html = text post_2.save() # 1. Should not get any result result = self.client.get(reverse("search:similar") + "?q=est", follow=False) self.assertEqual(result.status_code, 200) content = json_handler.loads(result.content.decode("utf-8")) self.assertEqual(len(content["results"]), 0) # index for model in self.indexable: if model is FakeChapter: continue self.manager.es_bulk_indexing_of_model(model) self.manager.refresh_index() # 2. Should get exactly one result result = self.client.get(reverse("search:similar") + "?q=mange", follow=False) self.assertEqual(result.status_code, 200) content = json_handler.loads(result.content.decode("utf-8")) self.assertEqual(len(content["results"]), 1) # 2. Should get exactly two results result = self.client.get(reverse("search:similar") + "?q=Clem", follow=False) self.assertEqual(result.status_code, 200) content = json_handler.loads(result.content.decode("utf-8")) self.assertEqual(len(content["results"]), 2)
def test_update_manifest(self): opts = {} path_manifest1 = settings.BASE_DIR / 'fixtures' / 'tuto' / 'balise_audio' / 'manifest.json' path_manifest2 = settings.BASE_DIR / 'fixtures' / 'tuto' / 'balise_audio' / 'manifest2.json' args = [str(path_manifest2)] shutil.copy(path_manifest1, path_manifest2) LicenceFactory(code='CC BY') call_command('upgrade_manifest_to_v2', *args, **opts) manifest = path_manifest2.open('r') json = json_handler.loads(manifest.read()) self.assertTrue('version' in json) self.assertTrue('licence' in json) self.assertTrue('children' in json) self.assertEqual(len(json['children']), 3) self.assertEqual(json['children'][0]['object'], 'extract') os.unlink(args[0]) path_manifest1 = settings.BASE_DIR / 'fixtures' / 'tuto' / 'big_tuto_v1' / 'manifest.json' path_manifest2 = settings.BASE_DIR / 'fixtures' / 'tuto' / 'big_tuto_v1' / 'manifest2.json' args = [str(path_manifest2)] shutil.copy(path_manifest1, path_manifest2) call_command('upgrade_manifest_to_v2', *args, **opts) manifest = path_manifest2.open('r') json = json_handler.loads(manifest.read()) os.unlink(args[0]) self.assertTrue('version' in json) self.assertTrue('licence' in json) self.assertTrue('children' in json) self.assertEqual(len(json['children']), 5) self.assertEqual(json['children'][0]['object'], 'container') self.assertEqual(len(json['children'][0]['children']), 3) self.assertEqual(len(json['children'][0]['children'][0]['children']), 3) path_manifest1 = settings.BASE_DIR / 'fixtures' / 'tuto' / 'article_v1' / 'manifest.json' path_manifest2 = settings.BASE_DIR / 'fixtures' / 'tuto' / 'article_v1' / 'manifest2.json' args = [path_manifest2] shutil.copy(path_manifest1, path_manifest2) call_command('upgrade_manifest_to_v2', *args, **opts) manifest = path_manifest2.open('r') json = json_handler.loads(manifest.read()) self.assertTrue('version' in json) self.assertTrue('licence' in json) self.assertTrue('children' in json) self.assertEqual(len(json['children']), 1) os.unlink(args[0])
def test_update_manifest(self): opts = {} path_manifest1 = settings.BASE_DIR / "fixtures" / "tuto" / "balise_audio" / "manifest.json" path_manifest2 = settings.BASE_DIR / "fixtures" / "tuto" / "balise_audio" / "manifest2.json" args = [str(path_manifest2)] shutil.copy(path_manifest1, path_manifest2) LicenceFactory(code="CC BY") call_command("upgrade_manifest_to_v2", *args, **opts) manifest = path_manifest2.open("r") json = json_handler.loads(manifest.read()) self.assertTrue("version" in json) self.assertTrue("licence" in json) self.assertTrue("children" in json) self.assertEqual(len(json["children"]), 3) self.assertEqual(json["children"][0]["object"], "extract") os.unlink(args[0]) path_manifest1 = settings.BASE_DIR / "fixtures" / "tuto" / "big_tuto_v1" / "manifest.json" path_manifest2 = settings.BASE_DIR / "fixtures" / "tuto" / "big_tuto_v1" / "manifest2.json" args = [str(path_manifest2)] shutil.copy(path_manifest1, path_manifest2) call_command("upgrade_manifest_to_v2", *args, **opts) manifest = path_manifest2.open("r") json = json_handler.loads(manifest.read()) os.unlink(args[0]) self.assertTrue("version" in json) self.assertTrue("licence" in json) self.assertTrue("children" in json) self.assertEqual(len(json["children"]), 5) self.assertEqual(json["children"][0]["object"], "container") self.assertEqual(len(json["children"][0]["children"]), 3) self.assertEqual(len(json["children"][0]["children"][0]["children"]), 3) path_manifest1 = settings.BASE_DIR / "fixtures" / "tuto" / "article_v1" / "manifest.json" path_manifest2 = settings.BASE_DIR / "fixtures" / "tuto" / "article_v1" / "manifest2.json" args = [path_manifest2] shutil.copy(path_manifest1, path_manifest2) call_command("upgrade_manifest_to_v2", *args, **opts) manifest = path_manifest2.open("r") json = json_handler.loads(manifest.read()) self.assertTrue("version" in json) self.assertTrue("licence" in json) self.assertTrue("children" in json) self.assertEqual(len(json["children"]), 1) os.unlink(args[0])
def test_get_similar_topics(self): """Get similar topics lists""" if not self.manager.connected_to_es: return text = 'Clem ne se mange pas' topic_1 = TopicFactory(forum=self.forum, author=self.user, title=text) post_1 = PostFactory(topic=topic_1, author=self.user, position=1) post_1.text = post_1.text_html = text post_1.save() text = 'Clem est la meilleure mascotte' topic_2 = TopicFactory(forum=self.forum, author=self.user, title=text) post_2 = PostFactory(topic=topic_2, author=self.user, position=1) post_2.text = post_1.text_html = text post_2.save() # 1. Should not get any result result = self.client.get(reverse('search:similar') + '?q=est', follow=False) self.assertEqual(result.status_code, 200) content = json_handler.loads(result.content.decode('utf-8')) self.assertEqual(len(content['results']), 0) # index for model in self.indexable: if model is FakeChapter: continue self.manager.es_bulk_indexing_of_model(model) self.manager.refresh_index() # 2. Should get exactly one result result = self.client.get(reverse('search:similar') + '?q=mange', follow=False) self.assertEqual(result.status_code, 200) content = json_handler.loads(result.content.decode('utf-8')) self.assertEqual(len(content['results']), 1) # 2. Should get exactly two results result = self.client.get(reverse('search:similar') + '?q=Clem', follow=False) self.assertEqual(result.status_code, 200) content = json_handler.loads(result.content.decode('utf-8')) self.assertEqual(len(content['results']), 2)
def extract_content_from_zip(zip_archive): """Check if the data in the zip file are coherent :param zip_archive: the zip archive to analyze :type zip_archive: zipfile.ZipFile :raise BadArchiveError: if something is wrong in the archive :return: the content in the archive :rtype: VersionedContent """ try: manifest = str(zip_archive.read("manifest.json"), "utf-8") except KeyError: raise BadArchiveError( _("Cette archive ne contient pas de fichier manifest.json.")) except UnicodeDecodeError: raise BadArchiveError( _("L'encodage du manifest.json n'est pas de l'UTF-8.")) # is the manifest ok ? try: json_ = json_handler.loads(manifest) except ValueError: raise BadArchiveError( _("Une erreur est survenue durant la lecture du manifest, " "vérifiez qu'il s'agit de JSON correctement formaté.")) try: versioned = get_content_from_json( json_, None, "", max_title_len=PublishableContent._meta.get_field( "title").max_length) except BadManifestError as e: raise BadArchiveError(e.message) except InvalidSlugError as e: e1 = _('Le slug "{}" est invalide').format(e) e2 = "" if e.had_source: e2 = _(' (slug généré à partir de "{}")').format(e.source) raise BadArchiveError(_("{}{} !").format(e1, e2)) except Exception as e: raise BadArchiveError( _("Une erreur est survenue lors de la lecture de l'archive : {}." ).format(e)) # is there everything in the archive ? for f in UpdateContentWithArchive.walk_content(versioned): try: zip_archive.getinfo(f) except KeyError: raise BadArchiveError( _("Le fichier '{}' n'existe pas dans l'archive.").format( f)) return versioned
def form_valid(self, form): if self.request.FILES["archive"]: try: zfile = zipfile.ZipFile(self.request.FILES["archive"], "r") except zipfile.BadZipfile: messages.error(self.request, _("Cette archive n'est pas au format ZIP.")) return self.form_invalid(form) try: new_content = UpdateContentWithArchive.extract_content_from_zip( zfile) except BadArchiveError as e: messages.error(self.request, e.message) return super(CreateContentFromArchive, self).form_invalid(form) except KeyError as e: messages.error( self.request, _(e.message + " n'est pas correctement renseigné.")) return super(CreateContentFromArchive, self).form_invalid(form) else: # Warn the user if the license has been changed manifest = json_handler.loads( str(zfile.read("manifest.json"), "utf-8")) if new_content.licence and "licence" in manifest and manifest[ "licence"] != new_content.licence.code: messages.info( self.request, _("la licence « {} » a été appliquée.".format( new_content.licence.code))) # first, create DB object (in order to get a slug) self.object = PublishableContent() self.object.title = new_content.title self.object.description = new_content.description self.object.licence = new_content.licence self.object.type = new_content.type # change of type is then allowed !! self.object.creation_date = datetime.now() self.object.save() new_content.slug = self.object.slug # new slug (choosen via DB) # Creating the gallery gal = Gallery() gal.title = new_content.title gal.slug = slugify(new_content.title) gal.pubdate = datetime.now() gal.save() # Attach user to gallery self.object.gallery = gal self.object.save() # Add subcategories on tutorial for subcat in form.cleaned_data["subcategory"]: self.object.subcategory.add(subcat) # We need to save the tutorial before changing its author list since it's a many-to-many relationship self.object.authors.add(self.request.user) self.object.save() self.object.ensure_author_gallery() # ok, now we can import introduction = "" conclusion = "" if new_content.introduction: introduction = str(zfile.read(new_content.introduction), "utf-8") if new_content.conclusion: conclusion = str(zfile.read(new_content.conclusion), "utf-8") commit_message = _("Création de « {} »").format( new_content.title) init_new_repo(self.object, introduction, conclusion, commit_message=commit_message) # copy all: versioned = self.object.load_version() try: UpdateContentWithArchive.update_from_new_version_in_zip( versioned, new_content, zfile) except BadArchiveError as e: self.object.delete() # abort content creation messages.error(self.request, e.message) return super(CreateContentFromArchive, self).form_invalid(form) # and end up by a commit !! commit_message = form.cleaned_data["msg_commit"] if not commit_message: commit_message = _( "Importation d'une archive contenant « {} »").format( new_content.title) versioned.slug = self.object.slug # force slug to ensure path resolution sha = versioned.repo_update( versioned.title, versioned.get_introduction(), versioned.get_conclusion(), commit_message, update_slug=True, ) # This HAVE TO happen after commiting files (if not, content are None) if "image_archive" in self.request.FILES: try: zfile = zipfile.ZipFile( self.request.FILES["image_archive"], "r") except zipfile.BadZipfile: messages.error( self.request, _("L'archive contenant les images n'est pas au format ZIP." )) return self.form_invalid(form) UpdateContentWithArchive.use_images_from_archive( self.request, zfile, versioned, self.object.gallery) commit_message = _( "Utilisation des images de l'archive pour « {} »" ).format(new_content.title) sha = versioned.commit_changes( commit_message) # another commit # of course, need to update sha self.object.sha_draft = sha self.object.update_date = datetime.now() self.object.save() self.success_url = reverse("content:view", args=[versioned.pk, versioned.slug]) return super(CreateContentFromArchive, self).form_valid(form)
def form_valid(self, form): versioned = self.versioned_object if self.request.FILES["archive"]: try: zfile = zipfile.ZipFile(self.request.FILES["archive"], "r") except zipfile.BadZipfile: messages.error(self.request, _("Cette archive n'est pas au format ZIP.")) return super(UpdateContentWithArchive, self).form_invalid(form) try: new_version = UpdateContentWithArchive.extract_content_from_zip( zfile) except BadArchiveError as e: messages.error(self.request, e.message) return super(UpdateContentWithArchive, self).form_invalid(form) else: # Warn the user if the license has been changed manifest = json_handler.loads( str(zfile.read("manifest.json"), "utf-8")) if new_version.licence and "licence" in manifest and manifest[ "licence"] != new_version.licence.code: messages.info( self.request, _("la licence « {} » a été appliquée.").format( new_version.licence.code)) # first, update DB object (in order to get a new slug if needed) title_is_changed = self.object.title != new_version.title self.object.title = new_version.title self.object.description = new_version.description self.object.licence = new_version.licence self.object.type = new_version.type # change of type is then allowed !! self.object.save(force_slug_update=title_is_changed) new_version.slug = self.object.slug # new slug if any !! # ok, then, let's do the import. First, remove everything in the repository while True: if versioned.children: versioned.children[0].repo_delete(do_commit=False) else: break # this weird construction ensure that everything is removed versioned.slug_pool = default_slug_pool( ) # slug pool to its initial value (to avoid weird stuffs) # start by copying extra information self.object.insert_data_in_versioned( versioned) # better have a clean version of those one versioned.description = new_version.description versioned.type = new_version.type versioned.licence = new_version.licence # update container (and repo) introduction = "" conclusion = "" if new_version.introduction: introduction = str(zfile.read(new_version.introduction), "utf-8") if new_version.conclusion: conclusion = str(zfile.read(new_version.conclusion), "utf-8") versioned.repo_update_top_container(new_version.title, new_version.slug, introduction, conclusion, do_commit=False) # then do the dirty job: try: UpdateContentWithArchive.update_from_new_version_in_zip( versioned, new_version, zfile) except BadArchiveError as e: versioned.repository.index.reset() messages.error(self.request, e.message) return super(UpdateContentWithArchive, self).form_invalid(form) # and end up by a commit !! commit_message = form.cleaned_data["msg_commit"] if not commit_message: commit_message = _( "Importation d'une archive contenant « {} ».").format( new_version.title) sha = versioned.commit_changes(commit_message) # now, use the images from the archive if provided. To work, this HAVE TO happen after commiting files ! if "image_archive" in self.request.FILES: try: zfile = zipfile.ZipFile( self.request.FILES["image_archive"], "r") except zipfile.BadZipfile: messages.error( self.request, _("L'archive contenant les images n'est pas au format ZIP." )) return self.form_invalid(form) UpdateContentWithArchive.use_images_from_archive( self.request, zfile, versioned, self.object.gallery) commit_message = _( "Utilisation des images de l'archive pour « {} »" ).format(new_version.title) sha = versioned.commit_changes( commit_message) # another commit # of course, need to update sha self.object.sha_draft = sha self.object.update_date = datetime.now() self.object.save() self.success_url = reverse("content:view", args=[versioned.pk, versioned.slug]) return super(UpdateContentWithArchive, self).form_valid(form)
def test_update_manifest(self): opts = {} shutil.copy( os.path.join(BASE_DIR, 'fixtures', 'tuto', 'balise_audio', 'manifest.json'), os.path.join(BASE_DIR, 'fixtures', 'tuto', 'balise_audio', 'manifest2.json')) LicenceFactory(code='CC BY') args = [ os.path.join(BASE_DIR, 'fixtures', 'tuto', 'balise_audio', 'manifest2.json') ] call_command('upgrade_manifest_to_v2', *args, **opts) manifest = open( os.path.join(BASE_DIR, 'fixtures', 'tuto', 'balise_audio', 'manifest2.json'), 'r') json = json_handler.loads(manifest.read()) self.assertTrue('version' in json) self.assertTrue('licence' in json) self.assertTrue('children' in json) self.assertEqual(len(json['children']), 3) self.assertEqual(json['children'][0]['object'], 'extract') os.unlink(args[0]) args = [ os.path.join(BASE_DIR, 'fixtures', 'tuto', 'big_tuto_v1', 'manifest2.json') ] shutil.copy( os.path.join(BASE_DIR, 'fixtures', 'tuto', 'big_tuto_v1', 'manifest.json'), os.path.join(BASE_DIR, 'fixtures', 'tuto', 'big_tuto_v1', 'manifest2.json')) call_command('upgrade_manifest_to_v2', *args, **opts) manifest = open( os.path.join(BASE_DIR, 'fixtures', 'tuto', 'big_tuto_v1', 'manifest2.json'), 'r') json = json_handler.loads(manifest.read()) os.unlink(args[0]) self.assertTrue('version' in json) self.assertTrue('licence' in json) self.assertTrue('children' in json) self.assertEqual(len(json['children']), 5) self.assertEqual(json['children'][0]['object'], 'container') self.assertEqual(len(json['children'][0]['children']), 3) self.assertEqual(len(json['children'][0]['children'][0]['children']), 3) args = [ os.path.join(BASE_DIR, 'fixtures', 'tuto', 'article_v1', 'manifest2.json') ] shutil.copy( os.path.join(BASE_DIR, 'fixtures', 'tuto', 'article_v1', 'manifest.json'), os.path.join(BASE_DIR, 'fixtures', 'tuto', 'article_v1', 'manifest2.json')) call_command('upgrade_manifest_to_v2', *args, **opts) manifest = open( os.path.join(BASE_DIR, 'fixtures', 'tuto', 'article_v1', 'manifest2.json'), 'r') json = json_handler.loads(manifest.read()) self.assertTrue('version' in json) self.assertTrue('licence' in json) self.assertTrue('children' in json) self.assertEqual(len(json['children']), 1) os.unlink(args[0])
def load_version(self, sha=None, public=None): """Using git, load a specific version of the content. if ``sha`` is ``None``, the draft/public version is used (if ``public`` is ``True``). .. attention:: for practical reason, the returned object is filled with information from DB. :param sha: version :param public: if set with the right object, return the public version :type public: PublishedContent :raise BadObject: if sha is not None and related version could not be found :raise OSError: if the path to the repository is wrong :raise NotAPublicVersion: if the sha does not correspond to a public version :return: the versioned content :rtype: zds.tutorialv2.models.versioned.VersionedContent """ # load the good manifest.json if sha is None: if not public: sha = self.sha_draft else: sha = self.sha_public max_title_length = PublishableContent._meta.get_field('title').max_length if public and isinstance(public, PublishedContent): # use the public (altered and not versioned) repository path = public.get_prod_path() slug = public.content_public_slug if not os.path.isdir(path): raise OSError(path) if sha != public.sha_public: raise NotAPublicVersion with open(os.path.join(path, 'manifest.json'), 'r', encoding='utf-8') as manifest: json = json_handler.loads(manifest.read()) versioned = get_content_from_json( json, public.sha_public, slug, public=True, max_title_len=max_title_length, hint_licence=self.licence, ) else: # draft version, use the repository (slower, but allows manipulation) path = self.get_repo_path() if not os.path.isdir(path): raise OSError(path) repo = Repo(path) data = get_blob(repo.commit(sha).tree, 'manifest.json') try: json = json_handler.loads(data) except ValueError: raise BadManifestError( _('Une erreur est survenue lors de la lecture du manifest.json, est-ce du JSON ?')) versioned = get_content_from_json(json, sha, self.slug, max_title_len=max_title_length) self.insert_data_in_versioned(versioned) return versioned