def runflow(num_clusters, retokenize, recluster, dataset_path, minibatch):
if retokenize.lower() == 'y':
retokenize = "1"
else:
retokenize = "0"
if recluster.lower() == 'y':
recluster = "1"
else:
recluster = "0"
if minibatch.lower() == 'y':
minibatch = "1"
home_dir = path_utilities.remove_path_end(dataset_path)
corpusdir = os.path.join(
home_dir,
"converted-" + path_utilities.get_last_dir_from_path(dataset_path)
) #eg. /home/ljung/converted/
# record initial time that program started
t0 = time()
fr, all_cluster_words, distinct_cluster_labels = main_function(
num_clusters, retokenize, recluster, corpusdir, dataset_path, 10,
minibatch)
bar_clusters(fr, distinct_cluster_labels, num_clusters, home_dir,
dataset_path)
print_cluster_stats(fr, all_cluster_words, dataset_path, num_clusters)
# print total time taken to run program
print("\nTime taken: ", time() - t0, " seconds\n")
def get_document_contents(directory, dataset_path):
dataset_name, file_place = initialize_output_location(dataset_path)
# load in the extension index file from the output folder
ext_dict_file_loc = os.path.join(
file_place, "extension_index_" + dataset_name + ".npy")
ext_paths = np.load(ext_dict_file_loc).item()
# "filenames" list of the paths of files
# "data" list of the contents of files
filenames = []
data = []
i = 1
# get contents of txt files still in original dataset
txt_paths = ext_paths.get("txt")
print("Getting .txt contents from " + dataset_path)
for path in tqdm(txt_paths):
if os.path.isfile(path):
i = i + 1
# add the path of the file to "filenames"
filenames.append(path)
# read the contents of the file and remove newlines
fread = open(path, "r", errors='backslashreplace')
contents = fread.read()
fread.close()
contents = contents.replace("\n", "")
# add the string of the contents of the file to "data"
data.append(contents)
# get contents of converted files in the other directory
conv_folders = path_utilities.get_immediate_subdirectories(directory)
# for each folder in the directory (e.g. pdf/ doc/)
for folder in conv_folders:
filetype = path_utilities.get_last_dir_from_path(folder)
if filetype in ["pdf", "doc", "docx"]: #, "xml", "html"]:
print("Getting ." + folder + " contents")
for filename in tqdm(os.listdir(folder)):
cur_file = os.path.join(folder, filename)
if os.path.isfile(cur_file):
i = i + 1
# add the non-converted filename to "filenames"
new_name = path_utilities.str_decode(
path_utilities.remove_extension(filename))
filenames.append(new_name)
# read the contents of the file and remove newlines
fread = open(cur_file, "r", errors='backslashreplace')
contents = fread.read()
fread.close()
contents = contents.replace("\n", "")
# add the string of the file contents to "data"
data.append(contents)
print("Num total files: ", i)
print("All directory contents retrieved")
return filenames, data
def initialize_output_location(dataset_path):
# setup output path as "file_place" outside the repo
p = Path(Path(os.getcwd()).parent).parent
dataset_name = path_utilities.get_last_dir_from_path(dataset_path)
file_place = os.path.join(p, "cluster-datalake-outputs/",
dataset_name + "--output")
if not os.path.isdir(file_place):
os.mkdir(file_place)
return dataset_name, file_place
def count_and_sort_exts(extensions, num_slices, write_path, dataset_path):
# the name of the top-level directory of the dataset
dataset_name = path_utilities.get_last_dir_from_path(dataset_path)
# a dict mapping tokens to the count of how many times they appear
ext_count_dict = {}
# for each extension
for ext in extensions:
try:
if ext[2] != "._" and ext[-1] != "~" and ext[0] != "_":
# if the extension is already in our dict
if ext in ext_count_dict.keys():
# grab the old count
old_count = ext_count_dict.get(ext)
# increment and update the count in the dict
ext_count_dict.update({ext: old_count + 1})
else:
# otherwise, add a key,value pair with a count of 1
ext_count_dict.update({ext: 1})
except IndexError:
if ext.isalnum():
# if the extension is already in our dict
if ext in ext_count_dict.keys():
# grab the old count
old_count = ext_count_dict.get(ext)
# increment and update the count in the dict
ext_count_dict.update({ext: old_count + 1})
else:
# otherwise, add a key,value pair with a count of 1
ext_count_dict.update({ext: 1})
sorted_extensions = []
sorted_counts = []
fd = open(
os.path.join(write_path, "all_ext_counts_" + dataset_name + ".txt"),
"w")
# for each extension in the dict, iterating from largest to smallest count
for ext in sorted(ext_count_dict, key=ext_count_dict.get, reverse=True):
# add the extension to a sorted list of extensions
sorted_extensions.append(ext)
# add the corresponding count to a list of counts
sorted_counts.append(ext_count_dict[ext])
fd.write(ext + ": " + str(ext_count_dict[ext]) + "\n")
fd.close()
f = open(os.path.join(write_path, "top_exts_" + dataset_name + ".txt"),
'w')
if (len(sorted_extensions) < num_slices):
num_slices = len(sorted_extensions)
for i in range(num_slices):
f.write(sorted_extensions[i] + "\n")
f.close()
return sorted_extensions, sorted_counts
def runflow(dataset_path):
#===================================================================
#=#BLOCK#=#: Get read and write paths for cluster functions
#===================================================================
print("Getting read and write paths for cluster functions. ")
# check if the dataset location is a valid directory
check_valid_dir(dataset_path)
# get its absolute path
dataset_path = os.path.abspath(dataset_path)
# the name of the top-level directory of the dataset
dataset_name = get_last_dir_from_path(dataset_path)
# get converted file location and output location
out_dir = os.path.join(dataset_path,
"../" + "converted-" + dataset_name)
# define the write path for the entire program
write_path = "../../cluster-datalake-outputs/" + dataset_name + "--output/"
if not os.path.isdir(write_path):
os.system("mkdir " + write_path)
print("All results printing to " + write_path)
# get absolute paths
out_dir = os.path.abspath(out_dir)
write_path = os.path.abspath(write_path)
# get the location of the extension index file
print("Finding extension index file. ")
ext_dict_file_loc = os.path.join(write_path, "extension_index_"
+ dataset_name + ".npy")
# check if the above paths are valid
check_valid_dir(out_dir)
check_valid_file(ext_dict_file_loc)
# load the extension to path dict
ext_to_paths_dict = np.load(ext_dict_file_loc).item()
csv_path_list = []
txt_path_list = []
if "csv" in ext_to_paths_dict:
csv_path_list = ext_to_paths_dict["csv"]
if "txt" in ext_to_paths_dict:
txt_path_list = ext_to_paths_dict["txt"]
# location of files converted to csv format
csv_dir = os.path.join(out_dir, "csv/")
converted_csv_list = os.listdir(csv_dir)
doc2vec(dataset_path, dataset_name, write_path, csv_path_list)
return
def plot_extensions(dataset_path, num_extensions):
allpaths = DFS.DFS(dataset_path)
p = Path(os.getcwd()).parent
dataset_name = path_utilities.get_last_dir_from_path(dataset_path)
write_path = os.path.join(p, "outputs/", dataset_name + "--output/")
if not os.path.isdir(write_path):
os.mkdir(write_path)
# a list of all the file names (without the paths)
filenames = []
for path in allpaths:
filenames.append(path_utilities.get_fname_from_path(path))
filenames_no_ext, exts = remove_all_extensions(filenames)
plot_extension_pie(exts, num_extensions, write_path, dataset_path)
'''
def plot_extension_pie(extensions, num_slices, write_path, dataset_path):
sorted_tuple = count_and_sort_exts(extensions, num_slices, write_path,
dataset_path)
sorted_exts, sorted_counts = sorted_tuple
print("Number of unique extensions: ", len(sorted_exts))
dataset_name = path_utilities.get_last_dir_from_path(dataset_path)
labels = []
sizes = []
if (len(sorted_exts) < num_slices):
num_slices = len(sorted_exts)
for x in range(num_slices):
labels.append(sorted_exts[x])
sizes.append(sorted_counts[x])
plt.figure("pie")
plt.clf()
plt.pie(sizes, labels=labels)
plt.axis('equal')
plt.title(str(num_slices) + " Most Common Extensions in ")
pie_path = os.path.join(write_path,
"top_exts_pie_" + dataset_name + ".png")
plt.savefig(pie_path, dpi=300)
def extension_indexer(dataset_path, n, write_path):
allpaths = DFS(dataset_path)
# a list of all the filenames (without paths)
filenames = []
for path in allpaths:
filenames.append(path_utilities.get_fname_from_path(path))
filenames_no_ext, exts = extensions.remove_all_extensions(filenames)
sorted_tuple = extensions.count_and_sort_exts(exts, n, write_path,
dataset_path)
sorted_exts, sorted_counts = sorted_tuple
top_n_exts = sorted_exts #UNCOMMENT FOR ONLY CONVERTING TOP N EXTS [:n]
# makes a dictionary key for each of the top extensions
ext_locations = {}
for extension in top_n_exts:
ext_locations.update({extension: []})
# checks every file and saves the paths of those with the top extensions
# in a dict called "ext_locations"
for fp in allpaths:
fn = path_utilities.get_fname_from_path(fp)
if fn[:2] != "._":
ext = path_utilities.get_single_extension(fn)
if ext in top_n_exts:
ext_list = ext_locations.get(ext)
ext_list.append(fp)
ext_locations.update({ext: ext_list})
dataset_name = path_utilities.get_last_dir_from_path(dataset_path)
ext_write_path = os.path.join(write_path,
"extension_index_" + dataset_name + ".npy")
np.save(ext_write_path, ext_locations)
return ext_locations
def runflow(dataset_path, num_clusters, overwrite, overwrite_plot,
fill_threshold):
#===================================================================
#=#: Get read and write paths for cluster functions
#===================================================================
print("Getting read and write paths for cluster functions. ")
if overwrite == 'y' or overwrite == 'Y':
overwrite = "1"
if overwrite_plot == 'y' or overwrite_plot == 'Y':
overwrite_plot = "1"
# check if the dataset location is a valid directory
check_valid_dir(dataset_path)
# get its absolute path
dataset_path = os.path.abspath(dataset_path)
# the name of the top-level directory of the dataset
dataset_name = get_last_dir_from_path(dataset_path)
# get converted file location and output location
out_dir = os.path.join(dataset_path, "../" + "converted-" + dataset_name)
# define the write path for the entire program
write_path = "../../cluster-datalake-outputs/" + dataset_name + "--output/"
if not os.path.isdir(write_path):
os.system("mkdir " + write_path)
print("All results printing to " + write_path)
# get absolute paths
out_dir = os.path.abspath(out_dir)
write_path = os.path.abspath(write_path)
if not os.path.isdir(out_dir):
os.system("mkdir " + out_dir)
print("Converting to " + out_dir)
# get the location of the extension index file
print("Finding extension index file. ")
ext_dict_file_loc = os.path.join(
write_path, "extension_index_" + dataset_name + ".npy")
# check if the above paths are valid
check_valid_dir(out_dir)
check_valid_file(ext_dict_file_loc)
# load the extension to path dict
ext_to_paths_dict = np.load(ext_dict_file_loc).item()
csv_path_list = []
if "csv" in ext_to_paths_dict:
csv_path_list = ext_to_paths_dict["csv"]
# location of files converted to csv format
csv_dir = os.path.join(out_dir, "csv/")
#===================================================================
#=#: Generates the files needed for clustering, clusters,
# and prints various results.
#===================================================================
# if csvs have less than fill_threshold*100% nonempty cells in
# every row then we throw them out of our clustering.
# we have two dicts, one made up of files which were converted to
# csv format, and the other made up of files that were in csv
# format originally. we concatenate both dicts into "header_dict".
# Get the combined header dict
header_dict_converted = get_header_dict(csv_dir, [], fill_threshold, True)
header_dict_csv = get_header_dict("", csv_path_list, fill_threshold, False)
header_dict = dict(header_dict_converted)
header_dict.update(header_dict_csv)
# Get the file/header array, distance matrix
dist_tuple = dist_mat_generator(header_dict, write_path, overwrite,
dataset_name)
jacc_matrix, filename_header_pairs = dist_tuple
# cluster, generate labels
labels = agglomerative(jacc_matrix, num_clusters, filename_header_pairs,
overwrite_plot, write_path, dataset_name)
# plot in 3D
print("Plotting clusters in R^3. ")
plot_clusters(jacc_matrix, labels, write_path, overwrite_plot,
dataset_name, num_clusters)
# generate results in pdf and text files
print("Generating results. ")
list_cluster_lists, scores = generate_results(filename_header_pairs,
labels, num_clusters,
dataset_path, write_path,
dataset_name)
# get a table of the most common attributes in each cluster
print("Getting cluster attributes. ")
clust_attr_lists = get_cluster_attributes(filename_header_pairs, labels,
num_clusters, write_path,
dataset_name)
return scores
def convert(dataset_path, num_top_exts, num_processes):
check_valid_dir(dataset_path)
# the name of the top-level directory of the dataset
dataset_name = path_utilities.get_last_dir_from_path(dataset_path)
# get its absolute path
dataset_path = os.path.abspath(dataset_path)
#dest = os.path.join(dataset_path, "../finalconverted-" + dataset_name + "/")
dest = os.path.join(
Path(dataset_path).parent, "converted-" + dataset_name + "/")
if not os.path.isdir(dest):
os.system("mkdir " + dest)
check_valid_dir(dest)
# get the script output location
write_path = os.path.join("../../cluster-datalake-outputs/" +
dataset_name + "--output/")
# get its absolute path
write_path = os.path.abspath(write_path)
if not os.path.isdir(write_path):
os.system("mkdir ../../cluster-datalake-outputs/")
os.system("mkdir " + write_path)
check_valid_dir(write_path)
# create the destination directories for converted files.
csv_dest = os.path.join(dest, "csv/")
print("csv_dest: ", csv_dest)
if not os.path.isdir(csv_dest):
os.system("mkdir " + csv_dest)
# get a dictionary which maps extension names of the form "csv"
# to lists of the full paths of files with those extensions in the
# dataset.
# CREATES "extension_index_<dataset_name>.npy"
ext_locations = DFS.extension_indexer(dataset_path, num_top_exts,
write_path)
# if we have extensions with the following names, performs
# conversion.
'''
comp_paths = []
if "z" in ext_locations:
comp_paths = ext_locations.get("z")
comp_paths.extend(ext_locations.get("gz"))
comp_paths.extend(ext_locations.get("zip"))
handle_compressed(comp_paths, dest)
'''
'''
if "pdf" in ext_locations:
pdf_paths = ext_locations.get("pdf")
convert_pdfs(pdf_paths, dest, num_processes)
if "doc" in ext_locations:
doc_paths = ext_locations.get("doc")
convert_doc(doc_paths, dest, num_processes)
if "docx" in ext_locations:
docx_paths = ext_locations.get("docx")
convert_docx(docx_paths, dest, num_processes)
if "html" in ext_locations:
html_paths = ext_locations.get("html")
html_paths.extend(ext_locations.get("htm"))
convert_html(html_paths, dest, num_processes)
if "xml" in ext_locations:
xml_paths = ext_locations.get("xml")
convert_xml(xml_paths, dest, num_processes)
'''
if "xls" in ext_locations:
xls_paths = ext_locations.get("xls")
valid_xls = get_valid_filenames_struct(xls_paths)
convert_tabular(valid_xls, csv_dest, num_processes)
if "xlsx" in ext_locations:
xlsx_paths = ext_locations.get("xlsx")
valid_xlsx = get_valid_filenames_struct(xlsx_paths)
convert_tabular(valid_xlsx, csv_dest, num_processes)
if "tsv" in ext_locations:
tsv_paths = ext_locations.get("tsv")
valid_tsv = get_valid_filenames_struct(tsv_paths)
convert_tsv(valid_tsv, csv_dest, num_processes)
def main():
print("ARGUMENTS: ")
args = load_arguments()
print("Arguments loaded. ")
dataset_path = args.dataset_path
dest = os.path.join(dataset_path, "../")
num_clusters = args.num_clusters
num_top_exts = args.num_extensions
num_processes = args.num_processes
overwrite_dist = 'y'
overwrite_plot = 'y'
fill_threshold = 0.4
# check if destination is valid, get its absolute path
check_valid_dir(dest)
dest = os.path.abspath(dest)
# check if dataset is valid, get its absolute path
check_valid_dir(dataset_path)
dataset_path = os.path.abspath(dataset_path)
# the name of the top-level directory of the dataset
dataset_name = get_last_dir_from_path(dataset_path)
# define the write path for the entire program
write_path = "../../cluster-datalake-outputs/" + dataset_name + "--output/"
if not os.path.isdir(write_path):
os.system("mkdir " + write_path)
print("All results printing to " + write_path)
# get absolute path
write_path = os.path.abspath(write_path)
# write results to a text file
f = open(os.path.join(write_path, 'shuffle_test_' + dataset_name + '.txt'),
'w')
f.write("shuffle_ratio" + "," + "freqdrop_score" + "," +
"silhouette_score" + "," + "naive_tree_dist_score" + "," + "\n")
#===================================================================
#=#: Shuffle and cluster, recording the ensemble score.
#===================================================================
shuffle_tracker = []
# get a list of the paths to every file in the dataset
# rooted at "dataset_path"
filepaths = DFS.DFS(dataset_path)
# generate path to the new root of our test dataset
shuffled_dataset_name = "shuffled_" + dataset_name
shuffled_dataset_path = os.path.join(dest, shuffled_dataset_name)
print("clustering: ", shuffled_dataset_path)
# copy dataset to this new location
os.system("cp -r " + dataset_path + " " + shuffled_dataset_path)
# we gradually increase the proportion of the test dataset
# which is shuffled
shuffle_ratio = 0.0
while shuffle_ratio <= 1.0:
# define the write path for the entire program
write_path = "../../cluster-datalake-outputs/" + shuffled_dataset_name + "--output/"
# get converted file location and output location
out_dir = os.path.join(shuffled_dataset_path,
"../" + "converted-" + shuffled_dataset_name)
if not os.path.isdir(write_path):
os.system("mkdir " + write_path)
if not os.path.isdir(out_dir):
os.system("mkdir " + out_dir)
csv_path = os.path.join(out_dir, "csv/")
if not os.path.isdir(csv_path):
os.system("mkdir " + csv_path)
txt_path = os.path.join(out_dir, "txt/")
if not os.path.isdir(txt_path):
os.system("mkdir " + txt_path)
# shuffle and convert the test dataset
shuffle_tracker = shuffle(shuffled_dataset_path, shuffle_ratio, False,
shuffle_tracker, filepaths)
DFS.extension_indexer(shuffled_dataset_path, num_top_exts, write_path)
# cluster the shuffled test dataset
scores = schema_clustering.runflow(shuffled_dataset_path, num_clusters,
overwrite_dist, overwrite_plot,
fill_threshold)
# print results
print("Shuffle ratio: ", shuffle_ratio, "Freqdrop score: ", scores[0],
"Silhouette score: ", scores[1], "Naive score: ", scores[2])
f.write(
format(shuffle_ratio, '.3f') + "," + format(scores[0], '.3f') +
"," + format(scores[1], '.3f') + "," + format(scores[2], '.3f') +
"," + '\n')
# delete the shuffled dataset, outputs, and converted files
os.system("rm -r " + write_path)
os.system("rm -r " + out_dir)
shuffle_ratio += args.step
f.close()
return