def UpdateAllMarcFiles(orig_deletion_list):
# Create a deletion list that consists of the original list from the
# BSZ as well as all the ID's from the files starting w/ "Diff":
util.Remove("augmented_deletion_list")
if orig_deletion_list is None: # Create empty file.
with open("augmented_deletion_list", "a") as _:
pass
else:
shutil.copyfile("../" + orig_deletion_list, "augmented_deletion_list")
EnsureFileIsEmptyOrEndsWithNewline("augmented_deletion_list")
extract_IDs_script_path = GetPathOrDie("extract_IDs_in_erase_format.sh")
for marc_file_name in glob.glob("*.mrc"):
if not marc_file_name.startswith("Diff"):
continue
if process_util.Exec(extract_IDs_script_path,
args=[marc_file_name, "augmented_deletion_list"],
timeout=100) != 0:
util.Error("failed to append ID's from \"" + marc_file_name
+ "\" to \"augmented_deletion_list\"!")
util.Info("Created an augmented deletion list.")
# Now delete ID's from the augmented deletion list from all MARC-21 files:
delete_ids_path = GetPathOrDie("delete_ids")
for marc_file_name in glob.glob("*.mrc"):
if marc_file_name.startswith("Diff"):
continue
trimmed_marc_file = marc_file_name[:-4] + "-trimmed.mrc"
if process_util.Exec(delete_ids_path, args=["augmented_deletion_list", marc_file_name, trimmed_marc_file],
timeout=200, new_stdout=util.GetLogDirectory() + "/trimmed_marc.log",
new_stderr=util.GetLogDirectory() + "/trimmed_marc.log") != 0:
util.Error("failed to create \"" + trimmed_marc_file + " from \"augmented_deletion_list\" and "
"\"" + marc_file_name + "\"!")
RemoveOrDie(marc_file_name)
RemoveOrDie("augmented_deletion_list")
util.Info("Deleted ID's from MARC files.")
# Now concatenate the changed MARC records with the trimmed data sets:
for marc_file_name in glob.glob("*-trimmed.mrc"):
root_name = marc_file_name[:-19]
diff_name = glob.glob("Diff" + root_name + "*.mrc")[0]
if not util.ConcatenateFiles([marc_file_name, diff_name], root_name + ".mrc"):
util.Error("We failed to concatenate \"" + marc_file_name + "\" and \"" + diff_name + "\"!")
RemoveOrDie(marc_file_name)
RemoveOrDie(diff_name)
util.Info("Created concatenated MARC files.")
# Rename files to include the current date and move them up a directory:
current_date_str = datetime.datetime.now().strftime("%y%m%d")
marc_files = glob.glob("*.mrc")
for marc_file_name in marc_files:
RenameOrDie(marc_file_name, "../" + marc_file_name[:-4] + "-" + current_date_str + ".mrc")
os.chdir("..")
util.Info("Renamed and moved files.")
# Create symlinks with "current" instead of "YYMMDD" in the orginal files:
for marc_file in marc_files:
new_name = marc_file[:-4] + "-" + current_date_str + ".mrc"
util.SafeSymlink(new_name, re.sub("\\d\\d\\d\\d\\d\\d", "current", new_name))
util.Info("Symlinked files.")
return ("GesamtTiteldaten-current.mrc", "Normdaten-current.mrc")
def DeleteMarcRecords(original_marc_file, deletion_list, processed_marc_file):
util.Remove(processed_marc_file)
if process_util.Exec("delete_ids", args=[deletion_list, original_marc_file, processed_marc_file],
timeout=200) != 0:
util.Error("failed to create \"" + processed_marc_file + "\" from \"" + deletion_list + "\" and \""
+ original_marc_file + "\"!")
util.Info("Successfully created \"" + processed_marc_file + "\".")
def AugmentDeletionList(orig_list, changed_marc_data, augmented_list):
util.Remove(augmented_list)
shutil.copyfile(orig_list, augmented_list)
if process_util.Exec("extract_IDs_in_erase_format.sh", args=[changed_marc_data, augmented_list],
timeout=100) != 0:
util.Error("failed to create \"" + augmented_list + "\" from \"" + changed_marc_data + "\"!")
util.Info("Successfully created \"" + augmented_list + "\".")
def ConcatenateFiles(files, target):
if files is None or len(files) == 0:
Error(
"\"files\" argument to util.ConcatenateFiles() is empty or None!")
if target is None or len(target) == 0:
Error(
"\"target\" argument to util.ConcatenateFiles() is empty or None!")
return process_util.Exec("/bin/cat", files, new_stdout=target) == 0
def ExecOrCleanShutdownAndDie(cmd_name, args, log_file_name=None):
if log_file_name is None:
log_file_name = "/proc/self/fd/2" # stderr
if not process_util.Exec(cmd_path=cmd_name, args=args, new_stdout=log_file_name,
new_stderr=log_file_name, append_stdout=True, append_stderr=True, setsid=False) == 0:
CleanUp(None, log_file_name)
util.SendEmail("util.ExecOrDie", "Failed to execute \"" + cmd_name + "\".\nSee logfile \"" + log_file_name
+ "\" for the reason.", priority=1)
sys.exit(-1)
#!/bin/python3
# -*- coding: utf-8 -*-
# Test program for testing the process_util.Exec function's ability to append stdout to an existing file.
import process_util
process_util.Exec("/bin/ls", ["-1"],
new_stdout="/tmp/test_stdout",
append_stdout=True)