def migrate_articles(app, limit, article_filepath):
click.echo('Starting article migration.')
old_articles = load_json_file(article_filepath)
if limit:
old_articles = old_articles[:limit]
bar = ProgressBar(total=len(old_articles),
template=PROGRESSIST_TEMPLATE,
done_char='📃')
with app.app_context():
for old_article in bar.iter(old_articles):
create_article(old_article)
bar.done = 0 # Reset.
for old_article in bar.iter(old_articles):
create_article(old_article)
click.echo('Articles migrated.')
async def fetch_statistics(datasets: List[Dataset]) -> List[Dataset]:
print(f"Fetching statistics")
nb_updated_datasets = 0
bar = ProgressBar(total=len(datasets))
for dataset in bar.iter(datasets):
results = await fetch_stats_for(dataset.page)
if results["2020"]:
dataset.nb_hits = results["2020"][0]["nb_hits"]
nb_updated_datasets += 1
print(f"{nb_updated_datasets} datasets updated from Matomo")
return datasets
async def fetch_stats():
conn = await get_conn()
rows = await conn.fetch('SELECT id, url FROM datasets;')
bar = ProgressBar(total=len(rows))
for row in bar.iter(rows):
stats = await fetch_stats_for(row['url'])
if stats.get('2020'):
await conn.execute(
'UPDATE datasets SET nb_hits = $1 WHERE id = $2',
stats['2020'][0]['nb_hits'],
row['id']
)
def migrate_users(app, limit, users_filepath):
click.echo('Starting users migration.')
ds = app.user_datastore
old_users = load_json_file(users_filepath)
if limit:
old_users = old_users[:limit]
bar = ProgressBar(total=len(old_users),
template=PROGRESSIST_TEMPLATE,
done_char='👤')
editor_role = Role.objects.get(name='editor')
with app.app_context():
for old_user in bar.iter(old_users):
create_user(ds, old_user, editor_role)
click.echo('Users migrated.')
async def as_xlsx(max_rows=None, debug=False):
"""Export des données au format souhaité par la DGT.
:max_rows: Max number of rows to process.
:debug: Turn on debug to be able to read the generated Workbook
"""
print("Reading from DB")
records = await db.declaration.completed()
print("Flattening JSON")
if max_rows:
records = records[:max_rows]
wb = Workbook(write_only=not debug)
ws = wb.create_sheet()
ws.title = "BDD REPONDANTS"
wb.active = ws
ws_ues = wb.create_sheet()
ws_ues.title = "BDD UES détail entreprises"
ws_ues.append([
"Annee_indicateurs",
"Region",
"Departement",
"Adresse",
"CP",
"Commune",
"Tranche_effectif",
"Nom_UES",
"Siren_entreprise_declarante",
"Nom_entreprise_declarante",
"Nom_entreprise",
"Siren",
])
headers, columns = await get_headers_columns()
ws.append(headers)
bar = ProgressBar(prefix="Computing", total=len(records))
for record in bar.iter(records):
data = record.data
if not data:
continue
ues_data(ws_ues, data)
data = prepare_record(data)
data["modified_at"] = record["modified_at"]
ws.append([clean_cell(fmt(data.get(c))) for c, fmt in columns])
return wb
async def fetch_datasets_from_urls(dataset_urls: List[str]) -> List[Dataset]:
print("Fetching datasets from URLs.")
dataset_slugs = [
extract_slug(dataset_url) for dataset_url in dataset_urls
if dataset_url.startswith("https://www.data.gouv.fr/fr/datasets/")
]
datasets = []
bar = ProgressBar(total=len(dataset_slugs))
for i, dataset_slug in enumerate(bar.iter(dataset_slugs)):
data = await fetch_json_data(
f"/api/1/datasets/{dataset_slug}/",
headers={
"X-Fields": ("id,title,metrics,description,acronym,page,"
"owner{first_name,last_name,avatar_thumbnail},"
"organization{name,slug,logo_thumbnail}")
},
)
if data and "id" in data:
dataset = await convert_to_dataset(data, i)
datasets.append(dataset)
return datasets