def create_keywordProcessor(list_of_terms, remove_stopwords=True, custom_stopword_list=[""]): """ Creates a new flashtext KeywordProcessor and optionally does some lightweight text cleaning to remove stopwords, including any provided by the user. """ # create a KeywordProcessor keyword_processor = KeywordProcessor() keyword_processor.add_keywords_from_list(list_of_terms) # remove English stopwords if requested if remove_stopwords == True: keyword_processor.remove_keywords_from_list(stopwords.words('english')) # remove custom stopwords keyword_processor.remove_keywords_from_list(custom_stopword_list) return (keyword_processor)
import requests from flashtext.keyword import KeywordProcessor from nltk.corpus import stopwords # let's read in a couple of forum posts forum_posts = pd.read_csv("../input/ForumMessages.csv") # get a smaller sub-set for playing around with sample_posts = forum_posts.Message[0:3] # get data from list of top 5000 pypi packages (last 30 days) url = 'https://hugovk.github.io/top-pypi-packages/top-pypi-packages-30-days.json' data = requests.get(url).json() # get just the list of package names list_of_packages = [data_item['project'] for data_item in data['rows']] # create a KeywordProcess keyword_processor = KeywordProcessor() keyword_processor.add_keywords_from_list(list_of_packages) # remove english stopwords keyword_processor.remove_keywords_from_list(stopwords.words('english')) # remove custom stopwords keyword_processor.remove_keywords_from_list(['http','kaggle']) # test our keyword processor for post in sample_posts: keywords_found = keyword_processor.extract_keywords(post, span_info=True) print(keywords_found)
mt = MosesTokenizer(lang=f'{lang}') for domain in domains: sourcedir = f"{workdrive}/coding/Text2TCS/git/ACTER/{lang}/{domain}/annotations/" list_of_terms = [] with open(f"{sourcedir}/{domain}_{lang}_terms_nes.ann", "r", encoding="utf-8") as f: for line in f.readlines(): s = str(line).replace("OOD_Term", "").replace("Common_Term", "").replace("Specific_Term", "").replace( "Named_Entity", "").strip("\n").strip("\t") list_of_terms.append(s) kwp.add_keywords_from_list(list_of_terms) # Remove unwanted terms from list (single letters, prepositions, stop-words etc.) abc_list = list(string.ascii_uppercase + string.ascii_lowercase) kwp.remove_keywords_from_list(abc_list) kwp.remove_keywords_from_list(word_boundary_list) for i in stopwords(f"{lang}"): kwp.remove_keyword(i) kwp.remove_keyword(i.capitalize()) # Extract the terms with open(f"{outdir}/{lang}{suffix1}/{domain}{suffix2}full_tok.txt", "r", encoding="utf-8") as f: sentences = f.readlines() # print(sentences[-10:]) results =[] for line in tqdm(sentences): s = kwp.extract_keywords(line.rstrip()) results.append(s) print(results[-10:])