def create_contract_graph(directory, title):
"""
Generates a bar graph with the labels as the folders in the directory and the
bar heights as the number of documents in each folder
Params
directory: str where the folders are located
title: str title for bar graph
"""
ds = DirectoryAssistor()
folder_lst = ds.create_content_list(directory)
folder_dic = {}
for folder in folder_lst:
folder_dic[folder] = len(ds.create_content_list(directory + folder))
sort_folder_dic = sorted(folder_dic.items(),
key=lambda x: x[1],
reverse=True)
cols = []
labels = []
for i in sort_folder_dic:
labels.append(i[0])
cols.append(i[1])
fig, ax = plt.subplots(figsize=(10, 10))
# cols = list(new_dic.values())
# labels = list(new_dic.keys())
make_sns_bar_plot(ax, cols, labels, title=title)
class ContractPipeline():
def __init__(self, directory, stop_words):
"""
Instantiates a ContractPipeline Object
Params
directory: str for main directory where the folders for the documntes are stored
stop_words: list of words that will be removed from the documents
"""
self.directory = directory
self.stop_words = stop_words
self.ds = DirectoryAssistor()
self.individual_bag_o_words = {}
def get_list_of_docs(self):
"""
Creates
target_lst: list that has all of the types for each document in the
corresponding index of doc_text_lst
doc_text_lst: list of lowercased cleaned strings for the text in each document
"""
print('Converting to txt lists')
start_time = time.time()
folder_lst = self.ds.create_content_list(self.directory)
doc_lst = []
self.target_lst = []
self.doc_text_lst = []
for i in range(len(folder_lst)):
doc_lst.append(
self.ds.create_content_list(self.directory + folder_lst[i]))
self.individual_bag_o_words[folder_lst[i]] = []
for j in range(len(doc_lst[i])):
text = textract.process(self.directory + folder_lst[i] + '/' +
doc_lst[i][j])
# convert to str
text = text.decode('utf-8')
# lowercase all text
text = text.lower()
# remove all punctuation
text = re.sub(r'\W+', ' ', text)
# remove underscores
text = text.replace("_", "")
self.doc_text_lst.append(text)
self.target_lst.append(folder_lst[i])
self.individual_bag_o_words[folder_lst[i]].append(text.split())
lst = []
for val in self.target_lst:
lst.append(val.replace('_', ' '))
self.target_lst = lst
end_time = time.time()
print(f'This took {end_time-start_time:.2f} seconds')
def get_list_of_txts(self):
"""
Creates
target_lst: list that has all of the types for each document in the
corresponding index of doc_text_lst
doc_text_lst: list of lowercased cleaned strings for the text in each document
"""
print('Converting to txt lists')
start_time = time.time()
folder_lst = self.ds.create_content_list(self.directory)
doc_lst = []
self.target_lst = []
self.doc_text_lst = []
for i in range(len(folder_lst)):
doc_lst.append(
self.ds.create_content_list(self.directory + folder_lst[i]))
self.individual_bag_o_words[folder_lst[i]] = []
for j in range(len(doc_lst[i])):
# read in file as str
try:
with open(
self.directory + folder_lst[i] + '/' +
doc_lst[i][j], 'r') as f:
text = f.read().replace('\n', '')
except:
continue
# lowercase all text
text = text.lower()
# remove all punctuation
text = re.sub(r'\W+', ' ', text)
# remove underscores
text = text.replace("_", "")
self.doc_text_lst.append(text)
self.target_lst.append(folder_lst[i])
self.individual_bag_o_words[folder_lst[i]].append(text.split())
lst = []
for val in self.target_lst:
lst.append(val.replace('_', ' '))
self.target_lst = lst
end_time = time.time()
print(f'This took {end_time-start_time:.2f} seconds')
def bag_o_words(self):
print('Creating bag o words')
start_time = time.time()
for key in self.individual_bag_o_words.keys():
lst = []
for val in self.individual_bag_o_words[key]:
for word in val:
lst.append(word)
self.individual_bag_o_words[key] = Counter(lst)
total_word_lst = []
for i in self.doc_text_lst:
lst = i.split()
for j in lst:
total_word_lst.append(j)
self.total_bag_o_words = Counter(total_word_lst)
end_time = time.time()
print(f'This took {end_time-start_time:.2f} seconds')
def join_list_of_strings(self, lst):
"""
Joins the list into a string
Params
lst: list of words
"""
return [" ".join(x) for x in lst]
def remove_stop_words(self):
"""
Returns a new list of strings with stop words removed
stops_removed_str: list of strings with stop words removed
stops_removed_lst: list of lists containing words with stops removed
"""
print('Removing stop words')
start_time = time.time()
split_lst = [txt.split() for txt in self.doc_text_lst]
self.stops_removed_lst = []
for split in split_lst:
stops = [w for w in split if w not in self.stop_words]
stop_num = [
w for w in stops
if not (w.isdigit() or w[0] == '-' and w[1:].isdigit())
]
self.stops_removed_lst.append(stop_num)
self.stops_removed_str = self.join_list_of_strings(
self.stops_removed_lst)
end_time = time.time()
print(f'This took {end_time-start_time:.2f} seconds')
def word_condenser(self):
"""
Takes in a list of strings and lemmatizes or stems them depending on the
technique chosen
self.porter_str: list of strings with porter stem technique used
self.snowball_str: list of strings with snowball stem technique used
self.wordnet_str: list of strings with wordnet lemmatize technique used
"""
print('Condensing')
start_time = time.time()
porter = PorterStemmer()
snowball = SnowballStemmer('english')
wordnet = WordNetLemmatizer()
porter_lst = [[porter.stem(w) for w in words]
for words in self.stops_removed_lst]
snowball_lst = [[snowball.stem(w) for w in words]
for words in self.stops_removed_lst]
wordnet_lst = [[wordnet.lemmatize(w) for w in words]
for words in self.stops_removed_lst]
self.porter_str = self.join_list_of_strings(porter_lst)
self.snowball_str = self.join_list_of_strings(snowball_lst)
self.wordnet_str = self.join_list_of_strings(wordnet_lst)
end_time = time.time()
print(f'This took {end_time-start_time:.2f} seconds')
def count_vectorizer(self, max_features=None, ngram_range=(1, 1)):
"""
Sets up a word count matrix, a tfidf matrix, and a CountVectorizer for
the documents in the directory
Params
documents: list of strings to be vectorized
Returns
count_matrix: matrix with word counts
tfidf_matrix: a tfidf matrix of the documents
cv: CountVectorizer object for the documents
"""
print('Generating tfidf and count matrix')
start_time = time.time()
self.cv = CountVectorizer(max_features=max_features,
ngram_range=ngram_range)
self.tf_matrix = self.cv.fit_transform(self.stops_removed_str)
end_time = time.time()
print(f'This took {end_time-start_time:.2f} seconds')
def tf_vect(self, documents, max_features=None, ngram_range=(1, 1)):
"""
Returns tf-idf matrix from documents
Prams
documents: list of strings
"""
print('Generating tfidf')
start_time = time.time()
self.vect = TfidfVectorizer(max_features=max_features,
ngram_range=ngram_range)
self.tfidf = self.vect.fit_transform(documents)
end_time = time.time()
print(f'This took {end_time-start_time:.2f} seconds')
class ContractSifter():
def __init__(self, directory, stop_words):
self.directory = directory
self.ds = DirectoryAssistor()
self.stop_words = stop_words
self.porter_dic = {}
self.snowball_dic = {}
self.wordnet_dic = {}
self.combined_dic = {}
def create_dic(self):
"""
Returns a dictionary with folder names as the keys and an empty lst
as values
params
folder_names: list of folder names in the directory
Returns
Dictionary
"""
lst = self.ds.create_content_list(self.directory)
word_dic = {key: [] for key in lst}
return word_dic
def remove_stop_words(self, lst):
return [w for w in lst if w not in self.stop_words]
def add_words(self):
"""
Adds words from the files in the directories that are associated with
the keys in the self.word_dic
Returns
self.word_dic with a list of words with the following removed from each
file in the folder for that key:
1. Stop words
2. Punctuation
3. Underscores
"""
self.word_dic = self.create_dic()
for key in self.word_dic.keys():
lst = self.ds.create_content_list(self.directory+key)
for file in lst:
full_text = textract.process(self.directory+key+'/'+file)
str_full_text = full_text.decode('utf-8')
lower_full_text = str_full_text.lower()
edited_text = re.sub(r'\W+', ' ', lower_full_text)
edited_text = edited_text.replace("_","")
tokens = word_tokenize(edited_text)
stop_lst = self.remove_stop_words(tokens)
self.word_dic[key].append(stop_lst)
def combine_word_lists(self):
"""
Combine all of the lists for a key into one list from the Pipeline
word_dic attribute
"""
for key in self.word_dic.keys():
result = []
for lst in self.word_dic[key]:
result.extend(lst)
self.combined_dic[key] = result
def word_condenser(self):
porter = PorterStemmer()
snowball = SnowballStemmer('english')
wordnet = WordNetLemmatizer()
for key in self.combined_dic.keys():
porter_lst = []
snowball_lst = []
wordnet_lst = []
for word in self.combined_dic[key]:
porter_lst.append(porter.stem(word))
snowball_lst.append(snowball.stem(word))
wordnet_lst.append(wordnet.lemmatize(word))
self.porter_dic[key] = porter_lst
self.snowball_dic[key] = snowball_lst
self.wordnet_dic[key] = wordnet_lst
def word_count(self, dic):
"""
Returns the count of the words in each key of the dictionary
Params
dic = dict for which the words will be counted
Returns
new_dic: dict with word count for each key
"""
word_count_dic = {}
for key, val in dic.items():
word_count_dic[key] = Counter(val)
new_dic = dict(word_count_dic)
return new_dic
def word_cloud(self, dic):
"""
Generates a word cloud for each key in the dic
Params
dic: dict for which the word cloud will be generated
Returns
Plot with word cloud for each key in dic
"""
word_cloud_dic = {}
for key, val in dic.items():
word_cloud_dic[key] = ' '.join(val)
wc_lst = []
for val in word_cloud_dic.values():
wc = WordCloud(width=1000, height=1000, background_color='white',
min_font_size=9)
wc_lst.append(wc.generate(val))
fig, axs = plt.subplots(3,3, figsize=(15,12))
titles = list(dic.keys())
for cloud, title, ax in zip(wc_lst, titles, axs.flatten()):
chartBox = ax.get_position()
ax.set_position(pos=[chartBox.x0,chartBox.y0,chartBox.width*1.05,
chartBox.height*1.05])
ax.imshow(cloud)
ax.set_title(title, fontsize=16, weight='bold')
ax.axis("off")
axs[2,1].set_axis_off()
axs[2,2].set_axis_off()
chartBox = axs[2,0].get_position()
axs[2,0].set_position(pos=[chartBox.x0*2.8,chartBox.y0*.9,chartBox.width*1.05,
chartBox.height*1.05])
plt.show()
for img in image_list:
da.delete_files(img)
merge_pages(page_lst=page_lst, out_dir=out_dir, file_name=file_name)
for page in page_lst:
da.delete_files(out_dir+page)
if __name__ == '__main__':
directory='/Users/justinlansdale/Documents/Galvanize/Capstone3/EC2Data/\
changeFolder/pdfs/Original/'
out_dir = '/Users/justinlansdale/Documents/Galvanize/Capstone3/EC2Data/\
changeFolder/Amended/'
da = DirectoryAssistor()
doc_lst = da.create_content_list(directory)
print(doc_lst)
start = time.time()
for i in range(len(doc_lst)):
file_name = doc_lst[i]
try:
main(directory, file_name, out_dir)
except:
continue
end = time.time()
print(end-start)
class AmendmentDetector():
def __init__(self, original_doc_dir, amended_doc_dir, original_txt_dir,
amended_txt_dir):
"""
Instantiates the amendment detection class
Params:
original_doc_dir: str directory where the original document is located
amended_doc_dir: str directory where the amended document is located
original_txt_dir: str directory where the original txt file will be
storeed
amended_txt_dir: str directory where the amended txt file will be
stored
"""
self.original_doc_dir = original_doc_dir
self.amended_doc_dir = amended_doc_dir
self.original_txt_dir = original_txt_dir
self.amended_txt_dir = amended_txt_dir
self.da = DirectoryAssistor()
def convert_original_to_txt(self):
"""
Converts original doct to .txt file
"""
doc_lst = self.da.create_content_list(self.original_doc_dir)
for i in range(len(doc_lst)):
file_name = doc_lst[i]
try:
main(self.original_doc_dir, file_name, self.original_txt_dir)
except:
continue
def convert_amended_to_txt(self):
"""
Converts amended doc to .txt file
"""
doc_lst = self.da.create_content_list(self.amended_doc_dir)
for i in range(len(doc_lst)):
file_name = doc_lst[i]
try:
main(self.amended_doc_dir, file_name, self.amended_txt_dir)
except:
continue
def read_in_files(self):
"""
Reads in txt files
"""
original_lst = self.da.create_content_list(self.original_txt_dir)
amended_lst = self.da.create_content_list(self.amended_txt_dir)
for doc in original_lst:
try:
with open(self.original_txt_dir + doc, 'r') as f:
self.original = f.read().replace('\n', ' ')
except:
continue
for doc in amended_lst:
try:
with open(self.amended_txt_dir + doc, 'r') as f:
self.amended = f.read().replace('\n', ' ')
except:
continue
def print_changes(self):
"""
Prints changes made in the amended document
"""
original_lst = self.original.split()
amended_lst = self.amended.split()
original_value = []
amended_value = []
change_ref = []
for i in range(len(original_lst)):
if original_lst[i] != amended_lst[i]:
original_value.append(original_lst[i])
change_ref.append(list(original_lst[i - 40:i + 40]))
amended_value.append(amended_lst[i])
for i in range(len(original_value)):
print(f'\n Change # {i+1}: {original_value[i]} changed to \
{amended_value[i]} \n \n Reference text: {" ".join(change_ref[i])} \n')