def run(self):
self.base = DBMeta.get(DBMeta.key == 'base').value or 'LEGI'
self.unknown_folders = defaultdict(lambda: 0)
self.liste_suppression = []
self.counts = defaultdict(lambda: 0)
self.skipped = 0
entries_counts = self.get_entries_count()
chunks = self.get_chunks(entries_counts)
if len(chunks) > 1:
print(f"big archive will be processed in {len(chunks)} chunks...")
for chunk_idx, chunk in enumerate(chunks):
chunk_counts, chunk_skipped = self.process_chunk(chunk_idx, chunk)
merge_counts(chunk_counts, chunk_skipped, self.counts,
self.skipped)
if self.liste_suppression:
db = connect_db(self.db_url)
db_proxy.initialize(db)
suppress(self.base, self.db, self.liste_suppression)
print("made %s changes in the database:" % sum(self.counts.values()),
json.dumps(self.counts, indent=4, sort_keys=True))
if self.skipped:
print("skipped", self.skipped, "files that haven't changed")
if self.unknown_folders:
for d, x in self.unknown_folders.items():
print("skipped", x, "files in unknown folder `%s`" % d)
def process_xml_jobs_batch(jobs_args_batch, db_url):
# this will be ran in a separate process, thus we need to init our own DB connection
db = connect_db(db_url)
db_proxy.initialize(db)
batch_counts, batch_skipped = process_xml_jobs_sync(jobs_args_batch,
db=db,
commit=False)
db.commit() # limits commits to db
return batch_counts, batch_skipped
def process_xml_batch(process_xml_jobs_args, db_url):
if MAX_PROCESSES != 1 and len(
process_xml_jobs_args) > 10 * PARALLEL_BATCH_SIZE:
return process_xml_jobs_in_parallel(process_xml_jobs_args, db_url)
else:
db = connect_db(db_url)
return process_xml_jobs_sync(process_xml_jobs_args,
db=db,
commit=True,
progress=True)
def test_normalize():
db = connect_db(':memory:')
db_proxy.initialize(db)
for row in DATA:
TexteVersion.insert(**row).execute()
normalize_text_titles(db)
data_brutes = list(TexteVersionBrute.select().order_by(
TexteVersionBrute.rowid))
data_norm = list(TexteVersion.select().order_by(TexteVersion.rowid))
assert len(data_brutes) == 6
assert data_brutes[0].bits == 23
assert data_norm[0].nature == "DECRET"
assert data_norm[0].titre == "Décret n° 75-96 du 18 février 1975"
assert data_norm[
0].titrefull == "Décret n° 75-96 du 18 février 1975 fixant les modalités de …"
assert data_brutes[1].bits == 63
assert data_norm[1].nature == "DECISION"
assert data_norm[
1].titre == "Décision du Conseil d'État n° 344021, 344022 du 28 juin 2013"
assert data_norm[
1].titrefull == "Décision du Conseil d'État n° 344021, 344022 du 28 juin 2013 statuant au contentieux"
assert data_norm[1].autorite == "CONSEIL D'ETAT"
assert data_norm[1].num == "344021, 344022"
assert data_norm[1].date_texte == "2013-06-28"
assert data_brutes[2].bits == 7
assert data_norm[2].nature == "LOI_ORGANIQUE"
assert data_norm[2].titre == "Loi organique n° 2016-1086 du 8 août 2016"
assert data_norm[
2].titrefull == "Loi organique n° 2016-1086 du 8 août 2016 relative à la nomination …"
assert data_brutes[3].bits == 4
assert data_norm[3].titrefull == "Code minier (nouveau)"
assert data_brutes[4].bits == 6
assert data_norm[4].titre == "Arrêté du 18 décembre 2014"
assert data_norm[4].titrefull == "Arrêté du 18 décembre 2014 modifiant …"
assert data_brutes[5].bits == 4
assert data_norm[5].titrefull == "Arrêté du 5 septembre 2002"
file_path = f"active_idccs_{human_date}.csv"
with open(file_path, 'w') as f:
writer = csv.DictWriter(
f, fieldnames=["idcc", "name", "group", "in_kali"])
writer.writeheader()
writer.writerows(new_rows)
print(f"wrote {len(new_rows)} IDCCs to {file_path}")
def idcc_to_group(idcc):
if idcc >= 0 and idcc <= 3999:
return 'DGT'
elif idcc >= 5000 and idcc <= 5999:
return 'DARES'
elif idcc >= 7000 and idcc <= 9999:
return 'AGRICULTURE'
if __name__ == '__main__':
p = ArgumentParser()
p.add_argument('--db-url')
p.add_argument('--identify-missing', action='store_true')
args = p.parse_args()
db = connect_db(args.db_url)
db_proxy.initialize(db)
xls_sheet = download_file_and_open_xls_sheet()
active_idccs = get_active_idccs(xls_sheet)
identify_missing(xls_sheet, active_idccs)
def run_importer(db_url=DEFAULT_OPTIONS["db-url"],
base=DEFAULT_OPTIONS["base"],
raw=DEFAULT_OPTIONS["raw"],
anomalies=DEFAULT_OPTIONS["anomalies"],
anomalies_dir=DEFAULT_OPTIONS["anomalies-dir"],
skip_links=DEFAULT_OPTIONS["skip-links"],
dumps_directory=DEFAULT_OPTIONS["dumps-dir"]):
db = connect_db(db_url)
db_proxy.initialize(db)
db_meta_base = DBMeta.get_or_none(key='base')
base_from_db = db_meta_base.value if db_meta_base else None
db_meta_last_update = DBMeta.get_or_none(key='last_update')
last_update = db_meta_last_update.value if db_meta_last_update else None
if not base_from_db:
DBMeta.create(key='base', value=base)
elif base and base != base_from_db:
print(
f"!> Wrong database: requested '{base}' but existing database is '{base_from_db}"
)
raise SystemExit(1)
if base != 'LEGI' and not raw:
print(
"!> You need to use the --raw option when working with bases other than LEGI."
)
raise SystemExit(1)
if base != 'LEGI' and anomalies:
print("!> The --anomalies option can only be used with the LEGI base")
raise SystemExit(1)
# Check and record the data mode
db_meta_raw = DBMeta.get_or_none(key='raw')
db_meta_raw = db_meta_raw.value if db_meta_raw else None
if raw:
versions_brutes = bool(TexteVersionBrute.get_or_none())
data_is_not_raw = versions_brutes or db_meta_raw is False
if data_is_not_raw:
print(
"!> Can't honor --raw option, the data has already been modified previously."
)
raise SystemExit(1)
if db_meta_raw != raw:
DBMeta.insert(key='raw', value=raw) \
.on_conflict(conflict_target=[DBMeta.key], preserve=[DBMeta.value]) \
.execute()
# Handle the --skip-links option
has_links = bool(Lien.get_or_none())
if not skip_links and not has_links and last_update is not None:
skip_links = True
print(
"> Warning: links will not be processed because this DB was built with --skip-links."
)
elif skip_links and has_links:
print("> Deleting links...")
Lien.delete()
# Look for new archives in the given dumps_directory
print("> last_update is", last_update)
archive_re = re.compile(
r'(.+_)?' + base.lower() +
r'(?P<global>_global|_)?_(?P<date>[0-9]{8}-[0-9]{6})\..+',
flags=re.IGNORECASE)
skipped = 0
archives = sorted([
(m.group('date'), bool(m.group('global')), m.group(0)) for m in [
archive_re.match(fn) for fn in os.listdir(dumps_directory)
if fnmatch(fn.lower(), '*' + base.lower() + '_*.tar.*')
]
])
most_recent_global = [t[0] for t in archives if t[1]][-1]
if last_update and most_recent_global > last_update:
print(
"> There is a new global archive, recreating the DB from scratch!")
raise Exception("not implemented yet")
# db.close()
# os.rename(db.address, db.address + '.back')
# db = connect_db(db_name, pragmas=args.pragma)
archives, skipped = partition(
archives, lambda t: t[0] >= most_recent_global and t[0] >
(last_update or ''))
if skipped:
print("> Skipped %i old archives" % len(skipped))
# Process the new archives
for archive_date, is_global, archive_name in archives:
print("> Processing %s..." % archive_name)
archive_processor = ArchiveProcessor(db,
db_url,
dumps_directory + '/' +
archive_name,
process_links=not skip_links)
archive_processor.run()
DBMeta.insert(key='last_update', value=archive_date) \
.on_conflict(conflict_target=[DBMeta.key], preserve=[DBMeta.value]) \
.execute()
last_update = archive_date
print('last_update is now set to', last_update)
# Detect anomalies if requested
if anomalies:
if not os.path.isdir(anomalies_dir):
os.mkdir(anomalies_dir)
fpath = anomalies_dir + '/anomalies-' + last_update + '.txt'
with open(fpath, 'w') as f:
n_anomalies = detect_anomalies(db, f)
print("logged", n_anomalies, "anomalies in", fpath)
postprocess(db, base)
if not raw:
from .normalize import normalize_text_titles
normalize_text_titles(db)
from .factorize import main as factorize
factorize(db)
from .normalize import normalize_article_numbers
normalize_article_numbers(db)
LEFT JOIN articles ON articles.id = hierarchie.element
WHERE SUBSTR(hierarchie.element, 5, 4) = 'ARTI'
""")
for article in articles:
bloc_textuel, titre, num = article
article_counts[conteneur_id] += 1
if bloc_textuel is None:
continue
soup = BeautifulSoup(bloc_textuel)
text = soup.text
if text is None:
continue
txt_file.write(text + '\n')
count = len([x.strip() for x in text.split(" ")])
word_counts[conteneur_id] += count
txt_file.close()
csv_writer.writerow([
conteneur_id, conteneur_num, conteneur_titre, textes_count,
article_counts[conteneur_id], word_counts[conteneur_id]
])
csv_file.close()
if __name__ == '__main__':
p = ArgumentParser()
p.add_argument('db')
p.add_argument('--output-dir', default=DEFAULT_OUTPUT_DIR)
args = p.parse_args()
db = connect_db(args.db)
write_csv(db, args.output_dir)
def test_factorize():
db = connect_db('sqlite:///tests_factorize.sqlite')
main(db)