def mirror_old_style_records_etl():
"""Ensure the old-style records exist for all present new-style ones."""
start_time = datetime.datetime.now()
LOG.info("Start: Mirror record documents in old-style naming.")
conn = credential.UNCPathCredential(path.RLID_DATA_SHARE,
**credential.RLID_DATA_SHARE)
count = Counter()
with conn:
for doc_path in rlid_record_paths():
doc_name = os.path.basename(doc_path)
doc_id, ext = os.path.splitext(doc_name)
old_style_path = rlid_record_path_old(doc_id, ext)
if not old_style_path:
count["not in database"] += 1
elif os.path.exists(old_style_path):
count["already mirrored"] += 1
elif place_record_old(doc_path):
count["mirrored"] += 1
else:
count["failed to mirror"] += 1
LOG.warning("%r failed to mirror to %r.", doc_name,
old_style_path)
document.log_state_counts(count, documents_type="records")
LOG.info("End: Mirror.")
elapsed(start_time, LOG)
def property_cards_staging_update():
"""Run update for RLID assessor property card staging repository."""
LOG.info("Start: Update assessor property card staging repository.")
start_time = datetime.datetime.now()
source_paths = document.repository_file_paths(path.LANE_PROPERTY_CARDS)
conn = credential.UNCPathCredential(
path.RLID_DATA_STAGING_SHARE, **credential.RLID_DATA_SHARE
)
with conn:
count = Counter()
for source_path in source_paths:
staging_path = os.path.join(
REPO_PATH["property-card-staging"], os.path.basename(source_path)
)
if document.changed(staging_path, source_path):
result_key = document.update_document(source_path, staging_path)
count[result_key] += 1
LOG.info("End: Update.")
document.log_state_counts(count, documents_type="property cards (staging)")
elapsed(start_time, LOG)
def tax_maps_staging_update():
"""Run update for RLID tax map staging repository."""
LOG.info("Start: Update tax map staging repository.")
start_time = datetime.datetime.now()
conn = credential.UNCPathCredential(
path.RLID_DATA_STAGING_SHARE, **credential.RLID_DATA_SHARE
)
with conn:
count = Counter()
for source_path in document.repository_file_paths(path.LANE_TAX_MAP_IMAGES):
staging_path = os.path.join(
REPO_PATH["tax-map-staging"],
# Tax maps have a one-deep bin.
os.path.split(os.path.dirname(source_path))[-1],
os.path.basename(source_path),
)
if document.changed(staging_path, source_path):
result_key = document.update_document(source_path, staging_path)
count[result_key] += 1
document.log_state_counts(count, documents_type="tax maps (staging)")
elapsed(start_time, LOG)
LOG.info("End: Update.")
def property_cards_update():
"""Run update for assessor property card RLID production repository."""
LOG.info("Start: Update RLID assessor property card repository.")
start_time = datetime.datetime.now()
staging_paths = document.repository_file_paths(
REPO_PATH["property-card-staging"], file_extensions=[".pdf"]
)
conn = credential.UNCPathCredential(
path.RLID_DATA_SHARE, **credential.RLID_DATA_SHARE
)
with conn:
count = Counter()
for staging_path in staging_paths:
rlid_path = rlid_document_path(
os.path.basename(staging_path), document_type="property-card"
)
if document.changed(rlid_path, staging_path):
result_key = document.update_document(staging_path, rlid_path)
count[result_key] += 1
LOG.info("End: Update.")
document.log_state_counts(count, documents_type="property cards")
elapsed(start_time, LOG)
def deeds_records_update():
"""Run update for deeds & records documents RLID repository."""
start_time = datetime.datetime.now()
PATH["logfile"] = os.path.join(
PATH["staging"], "Deeds_Records_Update_{}.log".format(start_time.year))
conn = credential.UNCPathCredential(PATH["staging"],
**credential.RLID_DATA_SHARE)
with conn:
# Attach logfile handler for staging logfile.
logfile = logging.FileHandler(PATH["logfile"])
logfile.setLevel(logging.INFO)
logfile.setFormatter(LOGFILE_FORMATTER)
LOG.addHandler(logfile)
LOG.info("START SCRIPT: Update RLID deeds & records repository.")
LOG.info(
"Start: Move deeds & records drop-files to staging directory.")
drop_extensions = [".exe", ".pdf", ".zip"
] + document.IMAGE_FILE_EXTENSIONS
for file_name in os.listdir(PATH["drop"]):
file_path = os.path.join(PATH["drop"], file_name)
file_extension = os.path.splitext(file_name)[-1].lower()
if all(
[os.path.isfile(file_path), file_extension
in drop_extensions]):
move_path = os.path.join(PATH["staging"], file_name)
shutil.move(file_path, move_path)
LOG.info("Moved %r to %r.", file_path, move_path)
LOG.info("End: Move.")
LOG.info("Start: Extract record archives.")
count = Counter()
for file_path in path.folder_file_paths(PATH["staging"]):
if os.path.splitext(file_path)[-1].lower() in [".exe", ".zip"]:
count[extract_records(file_path, archive_original=True)] += 1
document.log_state_counts(count, documents_type="archives")
# D&R archives include a few log & reference files; delete if present.
for file_path in path.folder_file_paths(PATH["staging"]):
for pattern in ["_logfile", "_xreffile"]:
if pattern.lower() in file_path.lower():
os.remove(file_path)
LOG.info("Start: Replace record images with PDFs.")
count = Counter()
for file_path in path.folder_file_paths(PATH["staging"]):
if (os.path.splitext(file_path)[-1].lower()
in document.IMAGE_FILE_EXTENSIONS):
count[convert_image(file_path, delete_original=True)] += 1
document.log_state_counts(count, documents_type="images")
LOG.info("Start: Place record PDFs in RLID repository.")
count = Counter()
for file_path in path.folder_file_paths(PATH["staging"]):
if os.path.splitext(file_path)[-1].lower() == ".pdf":
old_state = place_record_old(file_path)
new_state = place_record(
file_path, delete_original=(old_state == "placed"))
count.update([old_state, new_state])
document.log_state_counts(count, documents_type="records")
elapsed(start_time, LOG)
LOG.info("END SCRIPT")
def tax_maps_update():
"""Run update for RLID tax map repository."""
start_time = datetime.datetime.now()
conn = credential.UNCPathCredential(
path.RLID_DATA_SHARE, **credential.RLID_DATA_SHARE
)
with conn:
# Attach logfile handler for repository update logfile.
logfile = logging.FileHandler(
os.path.join(
REPO_PATH["tax-map"], "Tax_Map_Update_{}.log".format(start_time.year)
)
)
logfile.setLevel(logging.INFO)
logfile.setFormatter(LOGFILE_FORMATTER)
LOG.addHandler(logfile)
LOG.info("START SCRIPT: Update RLID tax map repository from staging.")
file_name_release_date = tax_map_file_name_release_map(
start_datetime=rlid_data_currency("Tax Maps")
)
count = Counter()
# Iterate through path/date map, adding, archiving & updating.
for file_name, release_date in file_name_release_date.items():
rlid_path = rlid_document_path(file_name, document_type="tax-map")
staging_path = rlid_document_path(
file_name, document_type="tax-map-staging"
)
result_key = update_tax_map(
staging_path, rlid_path, release_date, archive_previous=True
)
count[result_key] += 1
document.log_state_counts(count, documents_type="tax maps")
# Finally, update tax map repository currency date (if we placed any).
if count["updated"]:
rlid_data_currency_setter("Tax Maps", max(file_name_release_date.values()))
elapsed(start_time, LOG)
LOG.info("END SCRIPT: Update")