def indicateurs_thematiques(
indicateurs_dir=paths.indicateurs_markdown_dir,
thematiques_file=paths.thematique_markdown_file,
) -> None:
"""
Regenerate (overwrite) markdown files using thematiques
"""
markdown = load_md(thematiques_file)
thematiques = build_thematiques(markdown)
thematiques_lookup = {thematiques[id]: id for id in thematiques.keys()}
indicateur_files = glob.glob(os.path.join(indicateurs_dir, "*.md"))
for filename in indicateur_files:
typer.echo(f"Processing {filename}...")
md = load_md(filename)
indicators = build_indicators(md)
for indicator in indicators:
# -- extract this for future regen function on indicator --
climat_pratic = indicator["climat_pratic"]
indicator["climat_pratic_ids"] = [
thematiques_lookup[name] for name in climat_pratic
]
# ---
md = indicators_to_markdown_legacy_2(indicators)
write(filename, md)
def mesures_thematiques(
mesures_dir=paths.mesures_markdown_dir,
thematiques_file=paths.thematique_markdown_file,
) -> None:
"""
Regenerate (overwrite) markdown files using thematiques
"""
markdown = load_md(thematiques_file)
thematiques = build_thematiques(markdown)
thematiques_lookup = {thematiques[id]: id for id in thematiques.keys()}
mesures_files = glob.glob(os.path.join(mesures_dir, "*.md"))
with typer.progressbar(mesures_files) as progress:
for filename in progress:
md = load_md(filename)
mesure = build_mesure(md)
# -- extract this for future regen function on mesures --
climat_pratic = mesure["climat_pratic"]
mesure["climat_pratic_id"] = thematiques_lookup[climat_pratic]
# ---
md = mesure_to_markdown(mesure)
write(filename, md)
typer.echo(f"All {len(mesures_files)} 'mesures' were regenerated.")
def actions(
mesures_markdown_dir=paths.mesures_markdown_dir,
orientations_markdown_dir=paths.orientations_markdown_dir,
client_output_dir=paths.shared_client_data_dir,
shared_api_data_dir=paths.shared_api_data_dir,
output_typescript=True,
output_python=True,
) -> None:
# citergie
files = sorted_files(mesures_markdown_dir, "md")
actions_citergie = []
for file in files:
md = load_md(file)
action = build_action(md)
actions_citergie.append(action)
relativize_ids(actions_citergie, "citergie")
citergie = referentiel_from_actions(actions_citergie,
id="citergie",
name="Climat Air Énergie")
# economie circulaire
files = sorted_files(orientations_markdown_dir, "md")
actions_economie_circulaire = []
for file in files:
md = load_md(file)
action = build_action(md)
actions_economie_circulaire.append(action)
relativize_ids(actions_economie_circulaire, "economie_circulaire")
economie_circulaire = referentiel_from_actions(
actions_economie_circulaire,
id="economie_circulaire",
name="Economie circulaire",
)
all_actions = actions_citergie + actions_economie_circulaire
all_actions = clean_thematiques(all_actions)
all_actions = propagate_thematiques(all_actions)
all_actions = remove_top_nodes(all_actions)
if output_typescript:
# actions list (soon to be deprecated)
typescript = render_actions_as_typescript(all_actions)
filename = os.path.join(client_output_dir, "actions_referentiels.ts")
write(filename, typescript)
# two referentiels
typescript = render_actions_as_typescript(
[citergie, economie_circulaire])
filename = os.path.join(client_output_dir, "referentiels.ts")
write(filename, typescript)
if output_python:
python = render_actions_as_python([citergie, economie_circulaire])
filename = os.path.join(shared_api_data_dir, "referentiels.py")
write(filename, python)
def thematiques(
markdown_file: str = typer.Option(paths.thematique_markdown_file,
"--markdown", "-md"),
output_dir: str = typer.Option(paths.thematique_client_output_dir,
"--output", "-o"),
output_typescript: bool = typer.Option(True, "--typescript", "-ts"),
output_json: bool = typer.Option(False, "--json"),
) -> None:
"""
Convert 'thematiques' markdown to code.
"""
markdown = load_md(markdown_file)
thematiques = build_thematiques(markdown)
if output_json:
data = json.dumps(thematiques, indent=4)
filename = os.path.join(output_dir, "thematiques.json")
write(filename, data)
if output_typescript:
typescript = render_thematiques_as_typescript(thematiques)
filename = os.path.join(output_dir, "thematiques.ts")
write(filename, typescript)
typer.echo(f"Rendered {len(thematiques)} 'thematiques' in {output_dir}.")
def shared(
markdown_dir: str = paths.shared_markdown_dir,
typescript: bool = True,
client_output_dir: str = paths.shared_client_models_dir,
python: bool = True,
api_output_dir: str = paths.shared_api_models_dir,
) -> None: # pragma: no cover
"""
Generate shared definitions.
"""
files = glob.glob(os.path.join(markdown_dir, "*.md"))
with typer.progressbar(files) as progress:
for filename in progress:
if filename[-6:] == "poc.md":
continue
typer.echo(f"Processing {filename}...")
md = load_md(filename)
if typescript:
outputs = render_markdown_as_typescript(md)
for name, content in outputs:
write(os.path.join(client_output_dir, name), content)
if python:
outputs = render_markdown_as_python(md)
for name, content in outputs:
write(os.path.join(api_output_dir, name), content)
typer.echo(f"Processed {len(files)} shared definitions.")
def test_build_thematiques() -> None:
"""Test that thématique markdown file is parsed"""
markdown = load_md(
"../referentiels/markdown/thematiques_climat_pratic/thematiques.md")
thematiques = build_thematiques(markdown)
assert "strategie" in thematiques.keys()
assert "Stratégie" in thematiques.values()
def test_build_action_with_mesure():
"""Test that a specific mesure is parsed correctly"""
md = load_md("../referentiels/markdown/mesures_citergie/1.1.1.md")
mesure = build_action(md)
assert mesure
assert set(mesure.keys()) == {
"nom",
"actions",
"description",
"exemples",
"contexte",
"ressources",
"preuve",
"id",
"points",
"climat_pratic_id",
}
assert mesure["id"] == "1.1.1"
assert len(mesure["description"]) > 10
assert len(mesure["actions"]) == 7
for action in mesure["actions"]:
assert str(action["id"]).startswith(mesure["id"])
assert len(action["nom"]) > 10
def indicateurs_universal(
indicateurs_old_dir="../referentiels/markdown/indicateurs_citergie",
indicateurs_dir="../referentiels/markdown/indicateurs",
) -> None:
"""
Regenerate (overwrite) markdown files in a new format
"""
old_files = glob.glob(os.path.join(indicateurs_old_dir, "*.md"))
for filename in old_files:
typer.echo(f"Processing {filename}...")
md = load_md(filename)
indicateurs = build_indicators(md)
new_md = ""
for indicateur in indicateurs:
# -- extract this for future regen function on indicator --
indicateur["id"] = "cae-" + str(indicateur["id"])
indicateur["actions"] = []
for mesure_id in indicateur["mesures"]:
indicateur["actions"].append("climat_air_energie/" + mesure_id)
indicateur["obligation_cae"] = indicateur["obligation_citergie"]
indicateur["programmes"] = ["climat_air_energie"]
if indicateur["pcaet"]:
indicateur["programmes"].append("pcaet")
new_md += indicateur_to_markdown(indicateur) + "\n"
# ---
base_name = os.path.basename(filename)
number = re.findall(r"\d+", base_name)[0]
new_filename = f"cae_{int(number):03d}.md"
write(os.path.join(indicateurs_dir, new_filename), new_md)
def test_parse_all_definitions():
"""Test that all definition files are not breaking"""
files = glob.glob(os.path.join("definitions/shared", "*.md"))
for filename in files:
md = load_md(filename)
definitions = parse_definitions(md)
assert definitions
for definition in definitions:
assert isinstance(definition["yaml"], dict)
def test_render_thematiques_as_typescript() -> None:
"""Test that parsed thématiques renders as TypeScript"""
markdown = load_md(
"../referentiels/markdown/thematiques_climat_pratic/thematiques.md")
thematiques = build_thematiques(markdown)
rendered = render_thematiques_as_typescript(thematiques)
for id, name in thematiques.items():
assert id in rendered
assert name in rendered
def _render_actions_as_typescript(files: List[str]) -> None:
assert files
actions = []
for file in files:
md = load_md(file)
action = build_action(md)
actions.append(action)
typescript = render_actions_as_typescript(actions)
env = build_jinja_environment()
to_json = env.get_template("tests/single_value_to_json.j2")
for action in actions:
nom_json = to_json.render(value=action["nom"])
assert escape_to_html(nom_json) in typescript
def test_build_action_with_orientation():
"""Test that a specific orientation is parsed correctly"""
md = load_md(
"../referentiels/markdown/orientations_economie_circulaire/1.1.md")
orientation = build_action(md)
assert orientation
assert orientation["id"] == "1.1"
assert (
orientation["nom"] ==
"Définir une stratégie globale de la politique Economie Circulaire et assurer un portage politique fort"
)
assert len(orientation["description"]) > 10
assert len(orientation["actions"]) == 5
for action in orientation["actions"]:
assert str(action["id"]).startswith(orientation["id"])
assert len(action["nom"]) >= 10
def mesures_nested_actions(mesures_dir=paths.mesures_markdown_dir, ) -> None:
"""
Regenerate (overwrite) markdown files using nested actions
"""
mesures_files = glob.glob(os.path.join(mesures_dir, "*.md"))
count = 0
with typer.progressbar(mesures_files) as progress:
for filename in progress:
md = load_md(filename)
# todo make this recursive.
mesure = build_mesure(md)
for action in mesure["actions"]:
if "description" not in action.keys():
continue
action["actions"] = []
description = str(action["description"])
stripped_description = ""
lines = description.splitlines()
for line in lines:
if line.startswith("- "):
index = len(action["actions"]) + 1
action["actions"].append({
"id": f"{action['id']}.{index}",
"nom": line.lstrip("- "),
})
elif action[
"actions"]: # the line does is not an item, add it to the current action description
line = line.strip()
if line:
stripped_description += f"{line}\n"
if action[
"actions"]: # we have consumed the description lines to create sub actions
count += len(action["actions"])
action["description"] = stripped_description
md = mesure_to_markdown(mesure)
# temp_name = os.path.join("../referentiels/markdown/mesures_temp", os.path.basename(filename))
write(filename, md)
typer.echo(
f"All {len(mesures_files)} 'mesures' were regenerated extracting {count} 'tasks'."
)
def indicateurs(
indicateurs_markdown_dir: str = typer.Option(
paths.indicateurs_markdown_dir, "--markdown", "-md"),
client_output_dir: str = paths.shared_client_data_dir,
typescript: bool = True,
) -> None:
"""
Convert 'indicateurs' markdown files to code.
"""
files = sorted_files(indicateurs_markdown_dir, "md")
indicators = []
for filename in files:
typer.echo(f"Processing {filename}...")
md = load_md(filename)
indicators.extend(indicateurs_builder(md))
if typescript:
rendered = render_indicators_as_typescript(indicators)
filename = os.path.join(client_output_dir,
"indicateurs_referentiels.ts")
write(filename, rendered)
typer.echo(f"Rendered {len(files)} 'indicateurs' in {client_output_dir}.")
def test_build_indicators():
"""Test that a specific collection of indicators is parsed correctly"""
md = load_md("../referentiels/markdown/indicateurs/cae_001.md")
indicators = indicateurs_builder(md)
assert len(indicators) == 10
def get_thematiques(
markdown_file="../referentiels/markdown/thematiques_climat_pratic/thematiques.md",
) -> Dict[str, str]:
"""Returns thématiques built from markdown file"""
markdown = load_md(markdown_file)
return build_thematiques(markdown)
def test_render_markdown_as_python():
md = load_md("definitions/shared/action_status.md")
output = render_markdown_as_python(md)
assert output
def correspondance_table(
orientations_dir=orientations_markdown_dir,
correspondance_file:
str = "../referentiels/sources/dteci_correspondance.json",
output_dir: str = "../referentiels/data",
) -> None:
"""
Regenerate (overwrite) markdown files in a new format
"""
files = sorted_files(orientations_dir, "md")
actions_economie_circulaire = []
for file in files:
md = load_md(file)
action = build_action(md)
action["climat_pratic_id"] = "eci"
actions_economie_circulaire.append(action)
# relativize_ids(actions_economie_circulaire, 'economie_circulaire')
economie_circulaire = referentiel_from_actions(
actions_economie_circulaire,
id="economie_circulaire",
name="Economie circulaire",
)
def actionById(id: str, actions: List[dict]) -> dict:
for action in actions:
if action["id"] == id:
return action
elif id.startswith(action["id"]):
return actionById(id, action["actions"])
def parentId(action: dict) -> str:
ns = action["id"].split(".")
ns.pop()
return ".".join(ns)
with open(correspondance_file, encoding="utf-8") as file:
axes = json.load(file)
for axe in axes:
for orientation in axe["orientations"]:
for niveau in orientation["niveaux"]:
id = niveau["id"]
action = actionById(id, economie_circulaire["actions"])
print(id)
if not action or "indicateur" not in niveau.keys():
continue
indicateur = niveau["indicateur"]
# handle oui non
if "question" in indicateur.keys():
question = indicateur["question"]
if action["actions"]:
# ex 3.1.1
indicateur[
"raison"] = f"{len(action['actions'])} actions pour un seul niveau en oui non"
continue
question["oui"]["faite"] = [action["id"]]
# handle many oui non
elif "questions" in indicateur.keys():
questions = indicateur["questions"]
if not action["actions"]:
indicateur[
"raison"] = "Pas de sous actions pour plusieurs oui non"
continue
noms = [action["nom"] for action in action["actions"]]
for question in questions.keys():
choice, score = process.extractOne(question, noms)
chosen = [
action for action in action["actions"]
if action["nom"] == choice
][0]
questions[question]["oui"]["faite"] = [
chosen["id"]
]
questions[question]["oui"]["raison"] = (
f'"{action["id"]} {option["nom"]}" '
f'ressemble à {score}% à "{choice}"')
# handle fonction
elif "fonction" in indicateur.keys():
indicateur[
"raison"] = "Pas de correspondance pour une fonction"
# handle interval
elif "interval" in indicateur.keys():
indicateur[
"raison"] = "Pas de correspondance pour des intervalles de valeurs"
# handle intervalles
elif "intervalles" in indicateur.keys():
indicateur[
"raison"] = "Pas de correspondance pour des intervalles de valeurs"
# handle checkboxes
elif "choix" in indicateur.keys():
choix = indicateur["choix"]
if not action["actions"]:
indicateur[
"raison"] = f"pas de sous actions à {action['id']} pour ce niveau à choix multiple"
continue
if len(choix) > len(action["actions"]):
indicateur[
"raison"] = f"plus d'options ({len(choix)}) que d'actions ({len(action['actions'])})"
continue
noms = [action["nom"] for action in action["actions"]]
for option in choix:
choice, score = process.extractOne(
option["nom"], noms)
chosen = [
action for action in action["actions"]
if action["nom"] == choice
][0]
option["faite"] = [chosen["id"]]
option[
"raison"] = f'"{action["id"]} {option["nom"]}" ressemble à {score}% à "{choice}"'
# handle dropdown
elif "liste" in indicateur.keys():
liste = indicateur["liste"]
if not action["actions"]:
indicateur[
"raison"] = f"pas de sous actions à {action['id']} pour ce niveau à liste"
continue
if len(liste) > len(action["actions"]):
indicateur[
"raison"] = f"plus d'options ({len(liste)}) que d'actions ({len(action['actions'])})"
continue
noms = [action["nom"] for action in action["actions"]]
for option in liste:
choice, score = process.extractOne(
option["nom"], noms)
chosen = [
action for action in action["actions"]
if action["nom"] == choice
][0]
i = int(chosen["id"].split(".")[-1])
option["faite"] = [chosen["id"]]
option["faite"] = [
f"{parentId(chosen)}.{n}"
for n in range(1, i + 1)
]
option[
"raison"] = f'"{action["id"]} {option["nom"]}" ressemble à {score}% à "{choice}"'
print(option["faite"])
write(
os.path.join(output_dir, "correspondance_table.json"),
json.dumps(axes, indent=4, ensure_ascii=False),
)