def test_get_articles_tlfp_parse_error(self, app, dossier_plfss2018,
lecture_senat, amendements_senat,
article1_an):
from zam_repondeur.models import Amendement, DBSession
from zam_repondeur.services.fetch.articles import get_articles
responses.add(
responses.GET,
"https://www.senat.fr/leg/pjl17-063.html",
body=(SAMPLE_DATA_DIR / "pjl18-218.html").read_text(
"utf-8", "ignore"),
status=200,
)
with transaction.manager:
DBSession.add(dossier_plfss2018)
DBSession.add(lecture_senat)
changed = get_articles(lecture_senat)
assert not changed
amendement = DBSession.query(Amendement).filter(
Amendement.num == 6666).first()
assert amendement.article.content == {}
# Events should NOT be created
assert len(amendement.article.events) == 0
def test_get_articles_senat_with_mult(self, app, dossier_plfss2018,
lecture_senat, amendements_senat):
from zam_repondeur.models import Amendement, DBSession
from zam_repondeur.services.fetch.articles import get_articles
responses.add(
responses.GET,
"https://www.senat.fr/leg/pjl17-063.html",
body=(SAMPLE_DATA_DIR / "pjl17-063.html").read_text(
"utf-8", "ignore"),
status=200,
)
with transaction.manager:
amendement = amendements_senat[0]
amendement.article.num = "4"
amendement.article.mult = "bis"
# The objects are no longer bound to a session here, as they were created in
# a previous transaction, so we add them to the current session to make sure
# that our changes will be committed with the current transaction
DBSession.add(amendement)
DBSession.add(dossier_plfss2018)
DBSession.add(lecture_senat)
changed = get_articles(lecture_senat)
assert changed
amendement = DBSession.query(Amendement).filter(
Amendement.num == 6666).first()
assert amendement.article.content["001"].startswith(
"Ne donnent pas lieu à")
def test_not_found(self, app, lecture_an, amendements_an):
from zam_repondeur.models import Amendement, DBSession
from zam_repondeur.services.fetch.articles import get_articles
responses.add(
responses.GET,
"http://www.assemblee-nationale.fr/15/projets/pl0269.asp",
status=404,
)
responses.add(
responses.GET,
"http://www.assemblee-nationale.fr/15/propositions/pion0269.asp",
status=404,
)
responses.add(
responses.GET,
"http://www.assemblee-nationale.fr/15/ta-commission/r0269-a0.asp",
status=404,
)
changed = get_articles(lecture_an)
assert not changed
amendement = DBSession.query(Amendement).filter(
Amendement.num == 666).first()
assert amendement.article.content == {}
def test_intersticial_articles_are_not_updated(self, app, lecture_an):
from zam_repondeur.models import Article, DBSession
from zam_repondeur.services.fetch.articles import get_articles
article_avant_2 = Article.create(lecture=lecture_an,
type="article",
num="2",
pos="avant")
DBSession.add(article_avant_2)
responses.add(
responses.GET,
"http://www.assemblee-nationale.fr/15/projets/pl0269.asp",
body=(SAMPLE_DATA_DIR / "pl0269.html").read_text(
"utf-8", "ignore"),
status=200,
)
changed = get_articles(lecture_an)
assert changed
article = DBSession.query(Article).filter(
Article.pos == "avant").first()
assert article.user_content.title == ""
assert article.content == {}
def test_custom_article_titles_are_preserved(self, app, lecture_an,
amendements_an):
from zam_repondeur.models import Amendement, DBSession
from zam_repondeur.services.fetch.articles import get_articles
responses.add(
responses.GET,
"http://www.assemblee-nationale.fr/15/projets/pl0269.asp",
body=(SAMPLE_DATA_DIR / "pl0269.html").read_text(
"utf-8", "ignore"),
status=200,
)
DBSession.add(lecture_an)
amendement = DBSession.query(Amendement).filter(
Amendement.num == 666).first()
assert amendement.article.user_content.title == ""
assert amendement.article.content == {}
# Let's set a custom article title
amendement.article.user_content.title = "My custom title"
changed = get_articles(lecture_an)
assert changed
# We can get the article contents from an amendement
amendement = DBSession.query(Amendement).filter(
Amendement.num == 666).first()
assert amendement.article.user_content.title == "My custom title"
assert amendement.article.content["001"].startswith(
"Au titre de l'exercice 2016")
def test_article_ranges(self, app, lecture_an):
from zam_repondeur.models import DBSession
from zam_repondeur.services.fetch.articles import get_articles
responses.add(
responses.GET,
"http://www.assemblee-nationale.fr/15/projets/pl0269.asp",
body=(SAMPLE_DATA_DIR /
"pl0387.html").read_text("latin-1").encode("latin-1"),
status=200,
)
DBSession.add(lecture_an)
# No articles initially
assert {article.num for article in lecture_an.articles} == set()
changed = get_articles(lecture_an)
assert changed
nums = {article.num for article in lecture_an.articles}
# "Articles 1er et 2"
assert {"1", "2"}.issubset(nums)
# "Articles 19 à 24"
assert {"19", "20", "21", "22", "23", "24"}.issubset(nums)
def test_fallback_to_alternative_url_pattern(self, app, dossier_plfss2018,
lecture_an, amendements_an):
from zam_repondeur.models import Amendement, DBSession
from zam_repondeur.services.fetch.articles import get_articles
with transaction.manager:
lecture_an.texte.numero = 575
lecture_an.organe = "PO717460"
lecture_an.titre = "Première lecture – Séance publique"
amendements_an[0].article.num = "2"
# The objects are no longer bound to a session here, as they were created in
# a previous transaction, so we add them to the current session to make sure
# that our changes will be committed with the current transaction
DBSession.add(lecture_an)
DBSession.add_all(amendements_an)
# These URL patterns will fail
responses.add(
responses.GET,
"http://www.assemblee-nationale.fr/15/projets/pl0575.asp",
status=404,
)
responses.add(
responses.GET,
"http://www.assemblee-nationale.fr/15/propositions/pion0575.asp",
status=404,
)
# So we want to use this one instead
responses.add(
responses.GET,
"http://www.assemblee-nationale.fr/15/ta-commission/r0575-a0.asp",
body=(SAMPLE_DATA_DIR / "r0575-a0.html").read_text(
"utf-8", "ignore"),
status=200,
)
with transaction.manager:
DBSession.add(dossier_plfss2018)
DBSession.add(lecture_an)
changed = get_articles(lecture_an)
assert changed
amendement = DBSession.query(Amendement).filter(
Amendement.num == 666).first()
assert amendement.article.content["001"].startswith(
"Le code des relations entre")
def test_get_articles_senat(self, app, dossier_plfss2018, lecture_senat,
amendements_senat, article1_an):
from zam_repondeur.models import Amendement, Article, DBSession
from zam_repondeur.models.events.article import (
ContenuArticleModifie,
TitreArticleModifie,
)
from zam_repondeur.services.fetch.articles import get_articles
responses.add(
responses.GET,
"https://www.senat.fr/leg/pjl17-063.html",
body=(SAMPLE_DATA_DIR / "pjl17-063.html").read_text(
"utf-8", "ignore"),
status=200,
)
with transaction.manager:
DBSession.add(dossier_plfss2018)
DBSession.add(lecture_senat)
changed = get_articles(lecture_senat)
assert changed
amendement = DBSession.query(Amendement).filter(
Amendement.num == 6666).first()
assert amendement.article.content["001"].startswith(
"Au titre de l'exercice 2016")
# We should not modify articles from unrelated lectures
article = DBSession.query(Article).filter_by(pk=article1_an.pk).one()
assert article is not amendement.article
assert article.content == {}
# Events should be created
assert len(amendement.article.events) == 2
assert isinstance(amendement.article.events[0], TitreArticleModifie)
assert amendement.article.events[0].created_at is not None
assert amendement.article.events[0].user is None
assert amendement.article.events[0].data["old_value"] == ""
assert (amendement.article.events[0].data["new_value"] ==
"Dispositions relatives à l'exercice 2016")
assert isinstance(amendement.article.events[1], ContenuArticleModifie)
assert amendement.article.events[1].created_at is not None
assert amendement.article.events[1].user is None
assert amendement.article.events[1].data["old_value"] == {}
assert (amendement.article.events[1].data["new_value"]["001"] ==
"Au titre de l'exercice 2016, sont approuvés :")
def test_new_articles_are_created(self, app, lecture_an, amendements_an):
from zam_repondeur.models import DBSession
from zam_repondeur.models.events.article import (
ContenuArticleModifie,
TitreArticleModifie,
)
from zam_repondeur.services.fetch.articles import get_articles
responses.add(
responses.GET,
"http://www.assemblee-nationale.fr/15/projets/pl0269.asp",
body=(SAMPLE_DATA_DIR / "pl0269.html").read_text(
"utf-8", "ignore"),
status=200,
)
DBSession.add(lecture_an)
# We only have the article mentioned by the amendement
assert {article.num for article in lecture_an.articles} == {"1"}
changed = get_articles(lecture_an)
assert changed
# We now also have article 2 after scraping the web page
assert {article.num for article in lecture_an.articles} == {"1", "2"}
# Events should be created
assert len(lecture_an.articles[0].events) == 2
assert isinstance(lecture_an.articles[0].events[0],
TitreArticleModifie)
assert lecture_an.articles[0].events[0].created_at is not None
assert lecture_an.articles[0].events[0].user is None
assert lecture_an.articles[0].events[0].data["old_value"] == ""
assert (lecture_an.articles[0].events[0].data["new_value"] ==
"Dispositions relatives l'exercice 2016")
assert isinstance(lecture_an.articles[0].events[1],
ContenuArticleModifie)
assert lecture_an.articles[0].events[1].created_at is not None
assert lecture_an.articles[0].events[1].user is None
assert lecture_an.articles[0].events[1].data["old_value"] == {}
assert (lecture_an.articles[0].events[1].data["new_value"]["001"] ==
"Au titre de l'exercice 2016, sont approuvs :")
def test_annexes_are_retrieved(self, app, lecture_an, amendements_an):
from zam_repondeur.models import Article, DBSession
from zam_repondeur.services.fetch.articles import get_articles
lecture_an.texte.numero = 1056
lecture_an.organe = "PO717460"
lecture_an.titre = "Première lecture – Séance publique"
DBSession.add(lecture_an)
responses.add(
responses.GET,
"http://www.assemblee-nationale.fr/15/projets/pl1056.asp",
status=404,
)
responses.add(
responses.GET,
"http://www.assemblee-nationale.fr/15/propositions/pion1056.asp",
status=404,
)
responses.add(
responses.GET,
"http://www.assemblee-nationale.fr/15/ta-commission/r1056-a0.asp",
body=(SAMPLE_DATA_DIR /
"r1056-a0.html").read_text("latin-1").encode("latin-1"),
status=200,
)
assert get_articles(lecture_an)
annexe = DBSession.query(Article).filter(
Article.type == "annexe").first()
assert annexe.num == "93"
assert (annexe.user_content.title ==
"Stratégie nationale d'orientation de l'action publique")
assert annexe.content["001"].startswith(
"ANNEXE Stratégie nationale d'orientation de l'action publique")
assert annexe.content["002"].startswith(
("La présente stratégie nationale énonce les orientations et les "
"objectifs de l'action publique vers une société de confiance, "
"d'ici à 2022."))
def test_get_articles_senat_with_dots(self, app, dossier_plfss2018,
lecture_senat, amendements_senat):
from zam_repondeur.models import Amendement, DBSession
from zam_repondeur.services.fetch.articles import get_articles
responses.add(
responses.GET,
"https://www.senat.fr/leg/pjl17-063.html",
body=(SAMPLE_DATA_DIR / "pjl17-659.html").read_text(
"utf-8", "ignore"),
status=200,
)
with transaction.manager:
DBSession.add(dossier_plfss2018)
DBSession.add(lecture_senat)
changed = get_articles(lecture_senat)
assert changed
amendement = DBSession.query(Amendement).filter(
Amendement.num == 6666).first()
assert amendement.article.content["001"].startswith(
"La stratégie nationale")
