def __apply_dictionary(dictionary, xml_path, output_path):
info("\tApplying dictionary to xml files.")
output_path = __prepare_xml_output_folder(output_path)
num_tei_files = 0
# Loop over each file in the folder
for folder_file in os.listdir(xml_path):
# Process it if it is an xml file
if folder_file.endswith(".xml"):
info("\t\tFound " + folder_file)
# Create an output file
write_file = open(output_path + folder_file[0:-4] + ".xml", "w", encoding='utf8')
# Open the file that will be read
with open(xml_path + folder_file, encoding='utf8') as fp:
line = fp.readline()
# Loop over every line (see line 62)
while line:
# Check if any of the keys in the dictionary occur in the line
for key, value in dictionary.items():
# If a key exists, then replace it
if key in line:
line = line.replace(key, "\n<" + value + ">" + key + "</" + value + ">\n")
write_file.write(line)
line = fp.readline()
# Write the new file
write_file.close()
ok("\t\tWrote new file " + output_path + folder_file[0:-4] + ".xml\n")
num_tei_files += 1
ok("\tFinished applying dictionary to all xml files.")
ok("\tThe processed files have been output to the folder \"%s\"." % output_path)
return output_path, num_tei_files
def __check_dictionary_folder(dictionary_folder):
file_count = 0
for folder_file in os.listdir(dictionary_folder):
if folder_file.endswith(".txt"): # Process the file if it is a txt file
file_count += 1
if file_count == 0:
warning("\tThe folder \"%s\" contains no txt files!" % dictionary_folder)
info("Nothing more to do. Exiting without applying dictionary files.")
sys.exit(0)
else:
info("\tFound %i txt dictionary files in the folder \"%s\".\n" % (file_count, dictionary_folder))
def __prepare_xml_output_folder(processed_folder):
processed_folder = processed_folder + "tei-prep/" if processed_folder[-1:] == "/" else processed_folder + "/tei-prep/"
if not os.path.isdir(processed_folder):
info("\t\tDirectory \"%s\" doesn't yet exist. Trying to create it." % processed_folder)
assert isinstance(processed_folder, str)
try:
os.mkdir(processed_folder)
except OSError:
error("\t\tCreation of the output directory \"%s\" failed" % processed_folder)
sys.exit(1)
else:
ok("\t\tSuccessfully created the output directory \"%s\".\n" % processed_folder)
return processed_folder
def main():
start_time = time.time()
info("Converting files...")
# Get Command line arguments
parser = argparse.ArgumentParser(description='Prepare docx files for usage as tei.')
parser.add_argument("-i", dest="unprocessed_folder", required=False, default="./input/",
help="the folder containing docx files for input; the default is \"./input/\".", metavar="INFOLDER",
type=lambda x: is_valid_input_folder(x))
parser.add_argument("-d", dest="dictionary", required=False, default="./dictionaries/",
help="the folder containing dictionary files for xml transformation; the default is \"./dictionaries/\".", metavar="DICTFOLDER",
type=lambda x: is_valid_dictionary_folder(x))
parser.add_argument("-o", dest="processed_folder", required=False, default="./output/",
help="the folder where the processed files will be placed; the default is \"./output/\".", metavar="OUTFOLDER",
type=lambda x: is_valid_output_folder(x))
args = parser.parse_args()
# Parse the docx files
xml_dir = docx_to_xml.convert_to_xml(unprocessed_folder=args.unprocessed_folder, processed_folder=args.processed_folder)
# Apply the dictionary to the xml files
num_dictionary_entries, tei_path, num_tei_files = apply_dict.process_dictionary(args.dictionary, xml_dir, args.processed_folder)
ok("Conversion and processing of files is complete.")
info("All tasks completed in in %f seconds." % (time.time() - start_time))
info("%i dictionary entries were applied to %i files." % (num_dictionary_entries, num_tei_files))
info("The fully processed files are available in \"%s\"." % tei_path)
def is_valid_output_folder(arg):
arg = arg + "/" if arg[-1:] != '/' else arg
if not os.path.isdir(arg):
info("\tThe output folder \"%s\" does not exist yet, trying to create it." % arg)
assert isinstance(arg, str)
try:
os.mkdir(arg)
except OSError:
error("\tCreation of the output folder \"%s\" failed" % arg)
sys.exit(1)
else:
ok("\tSuccessfully created the output folder \"%s\"." % arg)
return arg
def convert_to_xml(unprocessed_folder, processed_folder):
# type: (str, str) -> str
"""
Read every txt file in the folder and wrap any lines like f.1a
in a <pb> milestone. Otherwise wrap the whole line in an <ab>.
:param unprocessed_folder: The folder containing unprocessed docx files
:param processed_folder: The output folder for the processed xml files
:rtype: None
"""
info("\nConverting the docx files to xml.")
processed_folder = __prepare_xml_output_folder(processed_folder)
folio_designation = re.compile(r'.*[f|p].*\..*[0-9].*')
column_designation = re.compile(r'^.*[C|c]ol.*\..*[0-9].*$')
leave_alpha_num = re.compile('[^A-z0-9.]')
leave_num = re.compile('[^0-9]')
# Get all files in the folder
for folder_file in os.listdir(unprocessed_folder):
if folder_file.endswith(
".docx"): # Process the file if it is a txt file
info("\tFound " + folder_file)
# Create an output file for the updated file
write_file = open(processed_folder + folder_file[0:-5] + ".xml",
"w",
encoding='utf8')
# Open the file to be read
doc = Document(unprocessed_folder + folder_file)
for para in doc.paragraphs:
line = str(para.text.strip()) # Strip any whitespace
is_folio = folio_designation.match(line)
is_column = column_designation.match(line)
if not is_folio and not is_column: # if it isn't a folio or col break, then it is text. Use <ab></ab>
write_file.write("<ab>" + line + "</ab>\n")
elif is_folio: # Otherwise, if it matches the folio regex, wrap it in a <pb>
line = leave_alpha_num.sub('', line)
write_file.write("<pb n=\"" + line + "\"/>\n")
elif is_column: # Otherwise, if it matches column regex, wrap it in a <cb>
line = leave_num.sub('', line)
write_file.write("<cb n=\"" + line + "\"/>\n")
# Close the file
write_file.close()
ok("\tWrote new file " + processed_folder + folder_file[0:-5] +
".xml\n")
return processed_folder
def is_valid_input_folder(arg):
arg = arg + "/" if arg[-1:] != '/' else arg
if not os.path.isdir(arg):
assert isinstance(arg, str)
error("\tThe folder \"%s\" does not exist!" % arg)
sys.exit(1)
else:
file_count = 0
for folder_file in os.listdir(arg):
if folder_file.endswith(".docx"): # Process the file if it is a txt file
file_count += 1
if file_count == 0:
error("\tThe folder \"%s\" contains no docx files!" % arg)
sys.exit(1)
else:
info("\tFound %i docx files in the folder \"%s\".\n" % (file_count, arg))
return arg
def __build_dict(dict_folder):
info("\tBuilding complete dictionary.")
__check_dictionary_folder(dict_folder)
dictionary = {}
check_alpha_num = re.compile(r"[^A-z0-9]")
# Loop over each file in the folder
for folder_file in os.listdir(dict_folder):
# Process it if it is a txt file
if folder_file.endswith(".txt"):
# Start an entry count for this file
file_entries = 0
info("\t\tFound " + folder_file)
# Open the dictionary file
with open(dict_folder + folder_file, encoding='utf8') as fp:
line = fp.readline()
lineCount = 1
# Loop through each line in the file
while line:
line = line.strip() # Strip any accidental whitespace
entry = line.split(',') # Split on the ","
# Process the line if it has a valid entry with only two items, and the second is only alphanumeric
if len(entry) == 2 and not check_alpha_num.findall(entry[1].strip()):
dictionary[entry[0].strip()] = entry[1].strip() # Create the new entry in the dictionary (any old one is overwritten)
file_entries += 1
else: # Notify the user of the error
warning("\t\tThere is an error in line " + str(lineCount) + " of " + folder_file + ".")
error(line)
lineCount += 1
line = fp.readline()
ok("\t\tFinished parsing " + folder_file + ": " + str(file_entries) + " entries.\n")
ok("\tFinished loading dictionaries. " + str(len(dictionary)) + " entries in total.\n")
return dictionary
def process_dictionary(dictionary_path, xml_path, output_path):
info("Processing the dictionaries.")
full_dictionary = __build_dict(dictionary_path)
output_path, num_tei_files = __apply_dictionary(full_dictionary, xml_path, output_path)
return len(full_dictionary), output_path, num_tei_files