def main(): # construct the argument parser and parse the arguments ap = argparse.ArgumentParser() ap.add_argument("-i",type=str, required=True, help="path to inputfile for the first task") ap.add_argument("-s", type=str, default="stopword_list.txt", help="The path to the stopword list used for the task") args = vars(ap.parse_args()) s=open(args["s"], "r") if s.mode == 'r': stopwords =s.readlines() for j in range(len(stopwords)): stopwords[j] = stopwords[j].split("\n")[0] stopwords[j] = stopwords[j].split("\t")[0] else: return "error reading stopwords file" p = stemmer.PorterStemmer() i=open(args["i"], "r") if i.mode == 'r': contents =i.readlines() for line in contents: document = line words = document.split(" ") for j in range(len(words)): words[j] = words[j].split(".")[0] words[j] = words[j].split(",")[0] words[j] = words[j].split("!")[0] words[j] = words[j].split(":")[0] words[j] = words[j].split("?")[0] words[j] = words[j].split(";")[0] words[j] = words[j].split("/")[0] words[j] = words[j].split("\n")[0] words[j] = words[j].lower() word_list = [] for word in words : if word not in stopwords : if not word.isnumeric(): word_list.append(word) for j in range(len(word_list)): word_list[j] = p.stem(word_list[j], 0,len(word_list[j])-1) print(word_list) else : return "error reading input file"
def stemWords(wordlist): """ receives a list of words and returns the list with stemmed words""" #using Porter Stemmer for a start # lancStemmer = nltk.stem.lancaster.LancasterStemmer() porterStemmer = porterstemmer.PorterStemmer() i = 0 endreached = False while endreached != True: #stem word at position i wordlist[i] = porterStemmer.stem(wordlist[i], 0, len(wordlist[i]) - 1) i += 1 if i == len(wordlist): endreached = True return wordlist
# -*- coding: utf-8 -*- """ Created on Tue Nov 5 17:56:19 2019 @author: Peter """ import math import pandas as pd #This may need to be changed if you do not have porterstemmer imported locally #It will be included in my zip folder import porterstemmer p = porterstemmer.PorterStemmer() '''Method to turn a sentence string into a list of tokens''' def tokenize(anArray): splitWordsArray = [] for _ in range(0, len(sentences)): temp = [] mystr = sentences[_].lower() for c in mystr[::1]: if (not c.isalpha() and c != " "): mystr = mystr.replace(c, "") for x in nums: mystr = mystr.replace(x, "") temp = mystr.split() splitWordsArray.append(temp) return splitWordsArray
def main(): # construct the argument parser and parse the arguments (input, output path and stopwords) ap = argparse.ArgumentParser() ap.add_argument("-i", type=str, required=True, help="path to inputfile for the first task") ap.add_argument( "-o", type=str, required=True, help="path to output file if it exists, if not, it will be created") ap.add_argument("-s", type=str, default="stopword_list.txt", help="The path to the stopword list used for the task") args = vars(ap.parse_args()) #opening the stopwords file and preparing the stopwords inside s = open(args["s"], "r") if s.mode == 'r': stopwords = s.readlines() for j in range(len(stopwords)): stopwords[j] = stopwords[j].split("\n")[0] stopwords[j] = stopwords[j].split("\t")[0] else: return "error reading stopwords file" # Initializing the stemmer tool p = stemmer.PorterStemmer() # Opening the input file i = open(args["i"], "r") if i.mode == 'r': # Reading the lines in the input file contents = i.readlines() #Initializing the output variables titles = [] output_doc = [] #Parsing the lines and preprocessing the content for line in contents: title, document = line.split("\t") titles.append(title) words = document.split(" ") for j in range(len(words)): words[j] = words[j].split(".")[0] words[j] = words[j].split(",")[0] words[j] = words[j].split("!")[0] words[j] = words[j].split(":")[0] words[j] = words[j].split("?")[0] words[j] = words[j].split(";")[0] words[j] = words[j].split("/")[0] temp = words[j].split('"') if len(temp) > 1: words[j] = temp[1] else: words[j] = temp[0] words[j] = words[j].split("\n")[0] words[j] = words[j].lower() word_list = [] # Eliminating stopwords and numerical values for word in words: if word not in stopwords: if not word.isnumeric(): word_list.append(word) # Stemming the content for j in range(len(word_list)): word_list[j] = p.stem(word_list[j], 0, len(word_list[j]) - 1) output_doc.append(word_list) else: return "error reading input file" # Opening the output file ( or creating it if it doesnt exist) o = open(args["o"], "w+") # organizing output lines for (title, doc) in zip(titles, output_doc): output_line = title + "\t" for j in range(len(doc)): output_line += doc[j] if j < len(doc) - 1: output_line += " " # Writing the outputs in the output file o.write(output_line + "\n") # Closing the file readers o.close() i.close() s.close()
def stem(sword): ps = porterstemmer.PorterStemmer() return ps.stem(sword, 0, len(sword) - 1)