def __init__(self,sentence): f = open('data/training_data', 'r') train_text=f.read() #data=open('data2','r') #test_data=data.read() custom_sent_tokenizer = PunktSentenceTokenizer(train_text) self.tokenized = custom_sent_tokenizer.tokenize(sentence)
class Tokenizer(object): def __init__(self, language, normalize=False, train_text_gen=None): """ A tokenizer using NLTK Penn Treebank tokenizer, and the Punkt sentence tokenizer. Params: language: Language to tokenize (currently doesn't do anything) train_text_gen: A generator of training text for the sentence tokenizer. """ self.language = language self.train_text_gen = train_text_gen self.normalize = normalize if train_text_gen: self.sent_tokenizer = self._train_sentence_tokenizer() else: self.sent_tokenizer = PunktSentenceTokenizer() def _train_sentence_tokenizer(self): return PunktSentenceTokenizer(train_text="\n".join(self.train_text_gen)) def tokenize(self, text): tokenized = [] for sentence in self.sent_tokenizer.tokenize(text): tokenized_sentence = [] for word in word_tokenize(sentence): if self.normalize: word = word.lower() tokenized_sentence.append(word) tokenized.append(tokenized_sentence) return tokenized
def POS_tagging(corpus): train_text = state_union.raw("2005-GWBush.txt") sample_text = corpus #print(train_text) custom_sentence_tokenizer = PunktSentenceTokenizer(train_text) # textfile = open("POS_tagged",'w') # textfile.write(train_text) # textfile.write("\n\n\n\n\n\n\n\n\n\n") # print(custom_sentence_tokenizer) tokenized = custom_sentence_tokenizer.tokenize(sample_text) tuples_list = [] def process_content(): try: for i in tokenized: words = nltk.word_tokenize(i) tagged = nltk.pos_tag(words) for w in tagged: tuples_list.append(w) except Exception as e: c=0 # print(str(e)) process_content() return tuples_list
def extractNounPhrases(sentence): nounPhrases = [] try: tokenizer = PunktSentenceTokenizer(sentence) tokenized = tokenizer.tokenize(sentence) words = nltk.word_tokenize(tokenized[0]) tagged = nltk.pos_tag(words) firstNN = False for tag in tagged: pos = tag[1] if "NN" in pos: if firstNN: nounPhrase = firstNoun + " " + tag[0] nounPhrases.append(nounPhrase) firstNN = False continue else: firstNoun = tag[0] firstNN = True continue firstNN = False except Exception as e: print(str(e)) return nounPhrases
def get_sentences(self, remove_url=True): ''' generator :param remove_url --> replace URLs in sentences with one space char ; :return: tuple of sentences for each mime-part ; ''' tokenizer = PunktSentenceTokenizer() for raw_line, mime_type, lang in tuple(self.get_text_mime_part()): if 'html' in mime_type: soup = BeautifulSoup(raw_line) if not soup.body: continue # cause exactly sentences are needed, soup.body.strings returns lines+0d0a lines = tuple(soup.body.strings) raw_line = ''.join(lines) try: sents = tuple(tokenizer.tokenize(raw_line)) except Exception as err: sents = tuple(raw_line) if remove_url: sents = tuple(map(lambda sent: self.__URLINTEXT_PAT.sub(' ', sent.lower()), sents)) sents = (s.strip().lower() for s in sents) sents = tuple(s for s in tuple(sents) if s) if len(sents) == 0: continue yield sents
def normalize(text): p = PunktSentenceTokenizer() bullet1 = '\xe2\x80\xa2'.decode('utf-8') bullet2 = '\xc2\xb7'.decode('utf-8') usable = '' for sentence in p.tokenize(text): if len(sentence) < 500: if bullet1 not in sentence and bullet2 not in sentence: usable += '%s ' % sentence return usable
def tokenize_english_document(input_text): """ This is a crude tokenizer for input conversations in English. :param input_text: :return: """ end_list = [] block_tokenizer = BlanklineTokenizer() sentence_tokenizer = PunktSentenceTokenizer() word_tokenizer = WhitespaceTokenizer() # using the 38 characters in one line rule from ITV subtitle guidelines characters_per_line = 38 lines_per_subtitle = 2 blocks = block_tokenizer.tokenize(input_text) for block in blocks: # We have one speaker sentences = sentence_tokenizer.tokenize(block) # We have the sentences for sentence in sentences: words = word_tokenizer.tokenize(sentence) reverse_words = words[::-1] lines = [] current_line = '' line_full = False while reverse_words: word = reverse_words.pop() longer_line = ' '.join([current_line, word]).strip() if len(longer_line) > characters_per_line and len(current_line): # The longer line is overreaching boundaries reverse_words.append(word) line_full = True elif len(word) >= characters_per_line: # Very long words current_line = longer_line line_full = True else: current_line = longer_line if line_full: lines.append(current_line) current_line = '' line_full = False if len(lines) >= lines_per_subtitle: end_list.append(lines) lines = [] if current_line: lines.append(current_line) if lines: end_list.append(lines) return end_list
def tag(sentence): try: tokenizer = PunktSentenceTokenizer(sentence) tokenized = tokenizer.tokenize(sentence) words = nltk.word_tokenize(tokenized[0]) tagged = nltk.pos_tag(words) return tagged except Exception as e: print(str(e))
def aristo_get_named_entities(self, text): """ Parses the texts to obtain named entities :param text: The text to parse :return:returns a named entity treexw """ custom_sent_tokenizer = PunktSentenceTokenizer(text) tokenized = custom_sent_tokenizer.tokenize(text) for i in tokenized[5:]: words = nltk.word_tokenize(i) tagged = nltk.pos_tag(words) namedEnt = nltk.ne_chunk(tagged, binary=False) return ((namedEnt))
def name_ent_recog(post): train_text = state_union.raw("2005-GWBush.txt") sample_text = post custom_sent_tokenizer = PunktSentenceTokenizer(train_text) tokenized = custom_sent_tokenizer.tokenize(sample_text) namedEnt = [] try: for i in tokenized: words = nltk.word_tokenize(i) tagged = nltk.pos_tag(words) namedEnt.append(nltk.ne_chunk(tagged)) except Exception as e: print(str(e)) return namedEnt
def sentenceTagging(text, trainingText): csTokenizer = PunktSentenceTokenizer(trainingText) tokenized = csTokenizer.tokenize(text) taggedSentence = [] try: for i in tokenized: words = nltk.word_tokenize(i) tagged = nltk.pos_tag(words) taggedSentence.append(tagged) #chinkingWords(tagged).draw() namedEntityRecog(tagged) except Exception as e: print(str(e)) return taggedSentence
def pos(self, paragraph): wordsdict = collections.OrderedDict() sent_tokenizer = PunktSentenceTokenizer() for sentence in self.sent_detector.tokenize(paragraph): tokens = sent_tokenizer.tokenize(sentence) for token in tokens: words = nltk.word_tokenize(token) tagged = nltk.pos_tag(words) for word in tagged: if word[1] in self.tagdict: wordsdict[word[0]] = self.tagdict[word[1]][0] return wordsdict
def main(): training_text = state_union.raw('2005-GWBush.txt') sample_text = state_union.raw('2006-GWBush.txt') custom_sent_tokenizer = PunktSentenceTokenizer(training_text) tokenized = custom_sent_tokenizer.tokenize(sample_text) choice = 0 while choice < 5: choice = input("1 for named_chunks. This provides some information about proper nouns.\n, 2 for process_chunks. This tells you if a noun phrase followed by n adverb occurs., \n3 for proccess content, this just prints stuff, 4 for...") if choice == 1: named_chunks(text_trained_tokenized(sample_text, training_text)) elif choice == 2: process_chunks(text_trained_tokenized(sample_text, training_text)) elif choice == 3: process_content(text_trained_tokenized(sample_text, training_text)) elif choice == 4: print "try again, bitch!"
def get_sentence_occurrences(document, terms, doc_term=None): terms_present = get_document_occurrences(document, terms) # Use a Tokenizer from NLTK to build a sentence list tokenizer = Tokenizer(document) sentences = tokenizer.tokenize(document) # Create a list of lists containing the collection of terms which cooccurr # in a sentence occurrences = [] for sentence in sentences: sentence_occurrences = set() for term in terms_present: if term != doc_term: if re.search(' %s ' % term.label, sentence): sentence_occurrences.add(term) if len(sentence_occurrences) > 0: sentence_occurrences = list(sentence_occurrences) to_remove = set() for inside in sentence_occurrences: for term in sentence_occurrences: if term != inside and\ term.label.find(inside.label) != -1: to_remove.add(inside) if to_remove: print "removing", to_remove for term in to_remove: sentence_occurrences.remove(term) if doc_term: sentence_occurrences.append(doc_term) occurrences.append(sentence_occurrences) return occurrences
def name_ent_recog(post): train_text = state_union.raw("2005-GWBush.txt") sample_text = post custom_sent_tokenizer = PunktSentenceTokenizer(train_text) tokenized = custom_sent_tokenizer.tokenize(sample_text) namedEnt = [] try: for i in tokenized: words = nltk.word_tokenize(i) tagged = nltk.pos_tag(words) namedEnt.append(nltk.ne_chunk(tagged)) # chunkGram = r"""Chunk: {<RB.?>*<VB.?>*<NNP.?>*<NN>?}""" # # chunkGram = r"""Chunk: {<.*>+} # # }<VB.?|IN|DT>+{""" # chunkParser = nltk.RegexpParser(chunkGram) # chunked = chunkParser.parse(tagged) # print(chunked) # #print(tagged) except Exception as e: print(str(e)) return namedEnt
class NER: """docstring for ClassName""" def __init__(self, query): self.original_query = query conf = shelve.open('conf') self.train_text = conf['train_text'] self.custom_sent_tokenizer = PunktSentenceTokenizer(self.train_text) self.tokenized = self.custom_sent_tokenizer.tokenize(self.original_query) def processContent(self): try: for i in self.tokenized: words = nltk.word_tokenize(i) tagged = nltk.pos_tag(words) namedEnt = nltk.ne_chunk(tagged, binary=True) #print(namedEnt) #namedEnt.draw() return namedEnt except Exception as e: print(str(e)) # Parse named entities from tree def structureNamedEntities(self): ne = [] for subtree in self.named_entity_tree: if type(subtree) == Tree: # If subtree is a noun chunk, i.e. NE != "O" ne_label = subtree.label() ne_string = " ".join([token for token, pos in subtree.leaves()]) ne.append((ne_string, ne_label)) return ne def performNER(self): self.named_entity_tree = self.processContent() #print(type(self.named_entity_tree)) self.named_entity_tuple = self.structureNamedEntities() #print(ne) names = [element[0] for element in self.named_entity_tuple] return names
def extract_features(self): """ All approach of extracting features from raw data implemented here """ custom_tokenizer = PunktSentenceTokenizer() regex_tokenizer = RegexpTokenizer(r'[a-zA-Z]+') ps = PorterStemmer() tokenized = [] with open(self.file_path, 'r') as current_document: for each_line in current_document: tokenized.extend(custom_tokenizer.tokenize(each_line)) # tokenizing words line by line feature_list = [] try: for each_sentence in tokenized: # words = nltk.word_tokenize(each_sentence) words = regex_tokenizer.tokenize(each_sentence) tagged = nltk.pos_tag(words) feature_list.extend([ps.stem(pos[0].lower()) for pos in tagged if pos[1] == 'NN']) # listing the nouns in a list except Exception as E: print(str(E)) feature_dictionary = Counter(feature_list) # converts an iterable object(in this case, LIST) to dictionary return feature_dictionary
def tokenize_sentence(input_text: str) -> List[str]: """ Converts a text into a list of sentence tokens """ if input_text is None or len(input_text) == 0: return [] tokenizer = PunktSentenceTokenizer() return tokenizer.tokenize(input_text)
def tokenizeText(text): text = text.replace("?", "?,") custom_sent_tokenizer = PunktSentenceTokenizer(text) tokenize = custom_sent_tokenizer.tokenize(text) return tokenize
import nltk from nltk.corpus import state_union from nltk.tokenize import PunktSentenceTokenizer train_text = state_union.raw("2005-GWBush.txt") sample_text = state_union.raw("2006-GWBush.txt") custom_sent_tokenizer = PunktSentenceTokenizer(train_text) ## train tokenizer tokenized = custom_sent_tokenizer.tokenize(sample_text) def process_content(): try: for i in tokenized: words = nltk.word_tokenize(i) tagged = nltk.pos_tag(words) chunkGram = r"""Chunk: {<RB.?>*<VB.?>*<NNP>+<NN>?}""" chunkParser = nltk.RegexpParser(chunkGram) chunked = chunkParser.parse(tagged) chunked.draw() ##print(chunked) except Exception as e: print(str(e)) process_content()
import nltk from nltk.corpus import state_union from nltk.tokenize import PunktSentenceTokenizer trainText = state_union.raw("2005-GWBush.txt") sampleText = state_union.raw("2006-GWBush.txt") # train tokenizer: customSentTokenizer = PunktSentenceTokenizer(trainText) tokenized = customSentTokenizer.tokenize(sampleText) def processContent(): try: for i in tokenized: words = nltk.word_tokenize(i) tagged = nltk.pos_tag(words) print(tagged) except Exception as e: print(str(e)) processContent()
import nltk from nltk.tokenize import PunktSentenceTokenizer from nltk.corpus import state_union train_text = state_union.raw('2005-GWBush.txt') sample_text = state_union.raw('2006-GWBush.txt') cst = PunktSentenceTokenizer(train_text) tknd = cst.tokenize(sample_text) try: for i in tknd[5:]: word = nltk.word_tokenize(i) # print(word) tgd = nltk.pos_tag(word) nER = nltk.ne_chunk(tgd) print(nER) except Exception as e: print(str(e))
import nltk from nltk.corpus import state_union from nltk.tokenize import PunktSentenceTokenizer train_txt = state_union.raw("2005-GWBush.txt") sample_txt = state_union.raw("2006-GWBush.txt") custom_sent_tokenize = PunktSentenceTokenizer(train_txt) tokenized = custom_sent_tokenize.tokenize(sample_txt) def process_content(): try: for w in tokenized: words = nltk.word_tokenize(w) tagged = nltk.pos_tag(words) print(tagged) except Exception as e: print(str(e)) process_content()
from nltk.tokenize import word_tokenize from nltk.tokenize import PunktSentenceTokenizer from nltk.corpus import state_union import nltk import json import re train_text = state_union.raw('2005-GWBush.txt') inputFile = 'data.json' outputFile = 'sample_text.txt' custom_sent_tokenizer = PunktSentenceTokenizer(train_text) tokenized = custom_sent_tokenizer.tokenize(outputFile) def createInput(): with open(inputFile, 'r') as i: with open(outputFile, 'w') as o: for line in i: try: tweet = json.loads(line) text = tweet['text'] o.write(' '.join(re.sub("(RT)|(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)"," ",text).split())) except BaseException as e: continue def tagInput(): try: with open(outputFile, 'r') as f: for line in f: words = nltk.word_tokenize(line)
print(ps.stem(w)) # Part of speach Tagging tag_list = [nltk.pos_tag(w) for w in words] print("Part of speech Tagg:", tag_list) for i in words: words = nltk.word_tokenize(i) tagged = nltk.pos_tag(words) print(tagged) # Chunking custom_sent_tokenizer = PunktSentenceTokenizer(text) tokenized = custom_sent_tokenizer.tokenize(text) def process_content(): try: for i in tokenized: words = nltk.word_tokenize(i) tagged = nltk.pos_tag(words) chunkGram = r"""Chunk: {<RB.?>*<VB.?>*<NNP>+<NN>?}""" chunkParser = nltk.RegexpParser(chunkGram) chunked = chunkParser.parse(tagged) chunked.draw() for i in tokenized: words = nltk.word_tokenize(i) tagged = nltk.pos_tag(words)
import nltk from nltk.corpus import state_union from nltk.tokenize import PunktSentenceTokenizer train_raw = state_union.raw("2005-GWBush.txt") sample_raw = state_union.raw("2006-GWBush.txt") tokenizer = PunktSentenceTokenizer(train_raw) sentences = tokenizer.tokenize(sample_raw) def process_data(): try: for sentence in sentences: words = nltk.word_tokenize(sentence) tagged = nltk.pos_tag(words) namedEnt = nltk.ne_chunk(tagged, binary=True) namedEnt.draw() except Exception as e: print(str(e)) process_data()
import nltk from nltk.tokenize import word_tokenize, PunktSentenceTokenizer text_file = open(".\login2.txt", "r") text = text_file.read() word = nltk.word_tokenize(text) custom_sen = PunktSentenceTokenizer(text) tokenized = custom_sen.tokenize("hello. how can i login. where is otp?") def process(): try: for w in tokenized: word = nltk.word_tokenize(w) tagged = nltk.pos_tag(word) namedEnt = nltk.ne_chunk(tagged) namedEnt.draw() print(tagged) except Exception as e: print(str(e)) process()
import nltk import wikipedia from nltk.tokenize import PunktSentenceTokenizer from nltk.corpus import state_union train_text = state_union.raw("2005-GWBush.txt") _text = input("Enter a text:") sample_text = wikipedia.summary(_text, sentences=1) custom_SentTok = PunktSentenceTokenizer(train_text) tokenized = custom_SentTok.tokenize(sample_text) def process_content(): try: for i in tokenized: words = nltk.word_tokenize(i) tagged = nltk.pos_tag(words) chunkGram = r"""Chunk: {<.*>+} }<VB.?|IN|DT|TO>+{""" chunkParser = nltk.RegexpParser(chunkGram) chunked = chunkParser.parse(tagged) chunked.draw() except Exception as e: print(str(e)) process_content()
m = textid_re.search(line) if m: sys.stdout.write(u".\n<{{{%s}}}!!!>\n" % m.group(1)) continue if line == "\n": continue if normquotes == 1: line = line.replace(u"«", " ' ") line = line.replace(u"»", " ' ") line = line.replace(u"“", " ' ") line = line.replace(u"”", " ' ") line = line.replace(u"\"", " ' ") sentences = st.tokenize(line) for s in sentences: if wptokenizer == 1: for w1 in wtt.tokenize(s): for w in wtw.tokenize(w1): sys.stdout.write(w.encode("utf-8")) sys.stdout.write("\n") else: for w in wtt.tokenize(s): sys.stdout.write(w.encode("utf-8")) sys.stdout.write("\n") # sys.stdout.write(".\n") sys.stdout.write("\n") sys.stdout.write("\n")
""" import nltk from nltk.corpus import state_union from nltk.tokenize import PunktSentenceTokenizer # The Tokenizer which will be used. '''This tokenizer is capable of unsupervised machine learning --> PuckSentenceTokenizer ''' '''o you can actually train it on any body of text that you use. First, let's get some imports out of the way that we're going to use:''' def process_content(): try: for i in tokenized[:5]: # here we are applying sentence limit so we can use this one for the processing the sentences. words = nltk.word_tokenize(i) # Tokenizes all the word , using the word tokenize! tagged = nltk.pos_tag(words) # Tags the specific words with the Natural language . print(tagged) # Prints the words with the Tags in the form of the tupple .! except Exception as e: print(str(e)) # if there is an exception then this prints out the exception if __name__ == '__main__': train_text = state_union.raw("2005-GWBUSH.txt") # This is the train text which will be used to tokenize the sample Test(unsupervised learning) sample_text = state_union.raw("2006-GWBUSH.txt") # This is the sample text which will be tokenized later onward print(type(sample_text)) custom_sent_tokenizer = PunktSentenceTokenizer( train_text) # This is the Train Text in the form of sentence being tokenized using the unsupervised learning.! #tokenized = custom_sent_tokenizer.tokenize(sample_text) # Tokenizing he Custom sentence tokenize tokenized = custom_sent_tokenizer.tokenize("Hi! my name is Shafay. I am 20 years old. I love playing games.") #print(tokenized) # this is just for the Debugging purposes! process_content() # Calling the process content function!
# -*- coding: utf-8 -*- """ Created on Tue Nov 24 06:45:40 2020 @author: Dell """ import nltk from nltk.corpus import state_union from nltk.tokenize import PunktSentenceTokenizer trainText = "A malapropism also called a malaprop, acyrologia, or Dogberryism is the mistaken use of an incorrect word in place of a word with a similar sound, resulting in a nonsensical, sometimes humorous utterance. An example is the statement by baseball player Yogi Berra, Texas has a lot of electrical votes, rather than electoral votes. Malapropisms often occur as errors in natural speech and are sometimes the subject of media attention, especially when made by politicians or other prominent individuals. Philosopher Donald Davidson has said that malapropisms show the comple process through which the brain translates thoughts into language.Humorous malapropisms are the type that attract the most attention and commentary, but bland malapropisms are common in speech and writing." customSentTokenizer = PunktSentenceTokenizer(trainText) tokenized = customSentTokenizer.tokenize(trainText) def processContent(): try: for i in tokenized: words = nltk.word_tokenize(i) tagged = nltk.pos_tag(words) #chunkGram = """Chunk: {<RB}""" print(tagged) except Exception as e: print(str(e)) processContent()
VBP verb, sing. present, non-3d take VBZ verb, 3rd person sing. present takes WDT wh-determiner which WP wh-pronoun who, what WP$ possessive wh-pronoun whose WRB wh-abverb where, when """ # retrieving the corpus train_text = state_union.raw('2005-GWBush.txt') text = state_union.raw('2006-GWBush.txt') # training the sentence tokenizer (unsupervised) tokenizer = PunktSentenceTokenizer(train_text) sentence = tokenizer.tokenize(text) # tagging the tokens by word tokenizing the sentence and the using regular exp to chunk the tokens try: for s in sentence: token = word_tokenize(s) pos = pos_tag(token) print(pos) chunkreg = r"""Chunk: {<RB.?>*<VB.?>*<NNP>+<NN>?}""" chunkParser = nltk.RegexpParser(chunkreg) chunked = chunkParser.parse(pos) chunked.draw() except Exception as e: print(str(e))
PERSON - Eddy Bonte, President Obama LOCATION - Murray River, Mount Everest DATE - June, 2008-06-29 TIME - two fifty a m, 1:30 p.m. MONEY - 175 million Canadian Dollars, GBP 10.40 PERCENT - twenty pct, 18.75 % FACILITY - Washington Monument, Stonehenge GPE - South East Asia, Midlothian ''' train_text = state_union.raw("2005-GWBush.txt") sample_text = state_union.raw("2006-GWBush.txt") sent_tokenizer = PunktSentenceTokenizer(train_text) tokenized = sent_tokenizer.tokenize(sample_text) def processContent(): try: for i in tokenized: words = word_tokenize(i) tagged = nltk.pos_tag(words) # print(tagged) # namedEnt = nltk.ne_chunk(tagged) namedEnt = nltk.ne_chunk( tagged, binary=True ) ## When type of the named entity is not important(puts all named entities together. # namedEnt.draw() print(namedEnt)
def get_sentence_occurrences(document, terms, terms_present=None, remove_overlap=False, remove_duplicates=False): """ Returns a list of lists representing the terms occuring in each sentence. Semantically equivalent to: [[term for term in terms if term in sent] for sent in document] Order of optional operations is: remove duplicates, remove overlap, add doc terms, remove duplicate doc terms """ # get list of terms in the document to narrow sentence-level search if terms_present is None: terms_present = set(get_document_occurrences(document, terms)) # Use a Tokenizer from NLTK to build a sentence list tokenizer = Tokenizer(document) sentences = tokenizer.tokenize(document) logging.info("scanning %d sentences for %d terms" % (len(sentences), len(terms))) # Create a list of lists containing the collection of terms which cooccurr # in a sentence occurrences = [] for sentence in sentences: sentence_occurrences = [] for term in terms_present: # build list of search patterns starting with label patterns = ['\b%s\b' % term.label] patterns.extend(term.searchpatterns) for pattern in patterns: try: # search for any occurrence of term, stop when found if re.search(pattern, sentence, flags=re.IGNORECASE): sentence_occurrences.append(term) break except re.error: logging.warning('Term %d (%s) pattern "%s" failed' % (term.ID, term.label, pattern)) term.searchpatterns.remove(pattern) # remove duplicates if remove_duplicates: sentence_occurrences = list(set(sentence_occurrences)) # remove overlapping elements if remove_overlap: to_remove = set() # build set of terms to remove for inside in sentence_occurrences: for term in sentence_occurrences: if term != inside and\ inside.label.find(term.label) != -1: to_remove.add(term) # remove terms for term in to_remove: sentence_occurrences.remove(term) # add to list of sentences if any terms are found if sentence_occurrences: occurrences.append(sentence_occurrences) return occurrences
#01 Segmentation sentences = brown.sents(categories=category) tokens = brown.words(categories=category) new_token = [] for w in tokens: word = re.sub(r'[-[_\],`!?():{}&$#@%*+;/\'"\t\n\b0-9]', r'', w.lower()) if word != '' and word not in stop_Words: new_token.append(word) row_text = ' '.join(new_token) #unsupervised learning ML alogrithm to detect end of sentences (EOS) custom_sent_tokenizer = PunktSentenceTokenizer(row_text) tokenized = custom_sent_tokenizer.tokenize(row_text) last_text = ' '.join(tokenized) #prediction Algorithm def markov_chain(text): words = text.split(' ') myDict = defaultdict(list) for currentWord, nextWord in zip(words[0:-1], words[1:]): myDict[currentWord].append(nextWord) myDict = dict(myDict) return myDict markov_return = markov_chain(last_text) numOfKeys = len(markov_return)
import nltk from nltk.corpus import state_union from nltk.tokenize import PunktSentenceTokenizer traintext = state_union.raw( "/home/varun/PycharmProjects/untitled/speechtrain.txt") sampletext = state_union.raw( "/home/varun/PycharmProjects/untitled/speechsample.txt") costum_sent_tokenizer = PunktSentenceTokenizer(traintext) tokenized = costum_sent_tokenizer.tokenize(sampletext) def process_content(): try: for i in tokenized: words = nltk.word_tokenize(i) tagged = nltk.pos_tag(words) #chunking chunkgram = r"""Chunk: {<RB.?>*<VB.?>*<NNP><NN>?}""" #chunking chunkgram = r"""Chunk: {<.*>+} # }<VB.?|IN|DT|TO>+{""" nameEnt = nltk.ne_chunk(tagged, binary=true) print(nameEnt) # chunkParser = nltk.RegexpParser(chunkgram) # chunked = chunkParser.parse(tagged) # print(chunked) except Exception as e: print(str(e))
import nltk from nltk.corpus import state_union from nltk.tokenize import PunktSentenceTokenizer train_text = state_union.raw("2005-GWBush.txt") #sample_text = state_union.raw("2006-GWBush.txt") custom_sent_tokenizer = PunktSentenceTokenizer(train_text) tokenized = custom_sent_tokenizer.tokenize( "Please save my ass my no is 98437598237459, 47598437") def process_content(): try: for i in tokenized: words = nltk.word_tokenize(i) tagged = nltk.pos_tag(words) chunkGram = r"""Chunk: {<CD>+}""" chunkParser = nltk.RegexpParser(chunkGram) chunked = chunkParser.parse(tagged) #print(chunked.label()) for subtree in chunked.subtrees(): if subtree.label() == 'Chunk': print(subtree[0][0]) except Exception as e: print(str(e))
import nltk from nltk.tokenize import PunktSentenceTokenizer sentence1 = """The group arrived at two o'clock on Monday afternoon to start class.""" sentence2 = """The Little Mermaid (Danish: Den lille havfrue) is a fairy tale written by the Danish author Hans Christian Andersen about a young mermaid who is willing to give up her life in the sea and her identity as a mermaid to gain a human soul.""" #Chunking custom_sent_tokenizer = PunktSentenceTokenizer(sentence1) tokenized = custom_sent_tokenizer.tokenize(sentence2) def process_content(): for i in tokenized: words = nltk.word_tokenize(i) tagged = nltk.pos_tag(words) chunkGram = r"""Chunk: {<RB.?>*<VB.?>*<NNP>+<NN>?}""" chunkParser = nltk.RegexpParser(chunkGram) chunked = chunkParser.parse(tagged) # chunked.draw() print chunked process_content() # many named nouns # chunking: chunk: 'noun phrases' be anoun, and modifiers around that noun. # descriptive group of words surrounding that noun. downside: only can use
import nltk from nltk.corpus import state_union from nltk.tokenize import PunktSentenceTokenizer,word_tokenize sample_test= state_union.raw("2006-GWBush.txt") sample_train=state_union.raw("2005-GWBush.txt") custom_sent_tokenizer=PunktSentenceTokenizer(sample_train) tokenize=custom_sent_tokenizer.tokenize(sample_test) #words =word_tokenize(tokenizerr) #print(words) for w in tokenize: words=word_tokenize(w) tagged=nltk.pos_tag(words) print(tagged) chunkgram= r"""Chunk: {<.*>+} }<NNP.?>+{""" chunkParser =nltk.RegexpParser(chunkgram) chunked=chunkParser.parse(tagged) print(chunked)
words = word_tokenize(mlk) filtered_mlk = [] for w in words: if w not in stop_words: filtered_mlk.append(w) # Fancy One Liner # filtered_mlk = [w in words if w not in stop_words] # pp(filtered_mlk) train_text = state_union.raw('2005-GWBush.txt') custom_sent_tokenizer = PunktSentenceTokenizer(train_text) custom_tokenized = custom_sent_tokenizer.tokenize(mlk) def process_content(): for i in custom_tokenized[5:]: words = word_tokenize(i) tagged = nltk.pos_tag(words) namedEnt = nltk.ne_chunk(tagged); print(namedEnt) # chunkGram = r"""Chunk: {<.*>+} # }<VB.?|IN|DT|TO>+{""" # chunkParser = nltk.RegexpParser(chunkGram) # chunked = chunkParser.parse(tagged) # chunked.draw() # except Exception as e:
Girl: Hola amigo... Hobo: his is all theatrical. 我说: "U.S.A 你好啊". U.S.A is the abbreviation of United States. To use statistical parameters such as mean and standard deviation reliably, you need to have a good estimator for them. The maximum likelihood estimates (MLEs) provide one such estimator. However, an MLE might be biased, which means that its expected value of the parameter might not equal the parameter being estimated.""" sentences = sent_tokenize(article) for sentence in sentences: tokens = word_tokenize(sentence) #print(sentence) text = webtext.raw('overheard.txt') print(text) sent_tokenizer = PunktSentenceTokenizer(text) sents1 = sent_tokenizer.tokenize(text) sents2 = sent_tokenize(text) sents1_article = sent_tokenizer.tokenize(article) sents2_article = sent_tokenize(article) print(sents1[0]) print(sents2[0]) print() print(sents1[677]) print(sents2[677]) print() print(sents1[678]) print(sents2[678]) print() print(sents1[679])
def natural_sentence(string): pst = PunktSentenceTokenizer(string) t = pst.tokenize(string) word = nltk.word_tokenize(t[0]) #here we chunking sentance into word tagged = nltk.pos_tag(word) #here each word is tagged means it is noud, pronoun, etc... is recognized print tagged chunkGram = r"""WRB:{<WRB.?>*<WP>*<WDT>?}""" #REGEXP for detecting wh question chunkParser = nltk.RegexpParser(chunkGram) #differentiate wh question chunked = chunkParser.parse(tagged) #getting each word this will gives the output in tree form for subtree in chunked.subtrees(): if subtree.label() == 'WRB': # for only wh question for j in subtree.leaves(): f = 0 final = "" final += j[0] chunk = r"""VB: {<VBZ>*<VBP>?}""" #here we are detecting type of wording and arranging it to proper place cp = nltk.RegexpParser(chunk) word = nltk.word_tokenize(t[0]) tagged = nltk.pos_tag(word) ch = cp.parse(tagged) flg = 0 for subtree in ch.subtrees(): if subtree.label() == 'VB': for j in subtree.leaves(): final += " "+j[0] flg = 1 break if flg == 0: final += " is" chunk = r"""PRP: {<PRP.?>?}""" cp = nltk.RegexpParser(chunk) ch = cp.parse(tagged) for subtree in ch.subtrees(): if subtree.label() == 'PRP': for j in subtree.leaves(): final += " "+j[0] chunk = r"""PRP: {<JJ.?>?}""" cp = nltk.RegexpParser(chunk) ch = cp.parse(tagged) for subtree in ch.subtrees(): if subtree.label() == 'PRP': for j in subtree.leaves(): final += " "+j[0] chunk = r"""PRP: {<RB.?>?}""" cp = nltk.RegexpParser(chunk) ch = cp.parse(tagged) for subtree in ch.subtrees(): if subtree.label() == 'PRP': for j in subtree.leaves(): final += " "+j[0] chunk = r"""PRP: {<VB.?>?}""" cp = nltk.RegexpParser(chunk) ch = cp.parse(tagged) for subtree in ch.subtrees(): if subtree.label() == 'PRP': for j in subtree.leaves(): final += " "+j[0] chunk = r"""NN: {<NN.?>?}""" cp = nltk.RegexpParser(chunk) ch = cp.parse(tagged) for subtree in ch.subtrees(): if subtree.label() == 'NN': for j in subtree.leaves(): if f == 0: final += " "+j[0] f = 1 else: final += " of "+j[0] f = 0 print final final_string = grammar(final) #sending generated sentance to ginger grammer for correcting grammar print final_string ws.send(final_string.upper()) #sending final sentance to board return chunkGram = r"""NN:{<PRP.?>*<NN.?>?}""" #same thing like wh question is here for simple present tence sentance chunkParser = nltk.RegexpParser(chunkGram) chunked = chunkParser.parse(tagged) for subtree in chunked.subtrees(): if subtree.label() == 'NN': for j in subtree.leaves(): f = 0 w = nltk.word_tokenize(string) w.remove(j[0]) final = "" final += " "+j[0] chunk = r"""VB: {<VBP>*<VBZ>*<VB>*<VB.?>*<MD.?>?}""" cp = nltk.RegexpParser(chunk) word = nltk.word_tokenize(t[0]) tagged = nltk.pos_tag(word) ch = cp.parse(tagged) flg = 0 for subtree in ch.subtrees(): if subtree.label() == 'VB': for j in subtree.leaves(): w.remove(j[0]) final += " "+j[0] flg = 1 break if flg == 0: final += " is" chunk = r"""PRP: {<PRP.?>?}""" cp = nltk.RegexpParser(chunk) ch = cp.parse(nltk.pos_tag(w)) for subtree in ch.subtrees(): if subtree.label() == 'PRP': for j in subtree.leaves(): final += " "+j[0] w.remove(j[0]) chunk = r"""NN: {<NN.?>?}""" cp = nltk.RegexpParser(chunk) ch = cp.parse(nltk.pos_tag(w)) for subtree in ch.subtrees(): if subtree.label() == 'NN': for j in subtree.leaves(): if f == 0: final += " "+j[0] f = 1 else: final += " of "+j[0] w.remove(j[0]) f = 0 for wrd in w: final += " "+wrd print final final_string = grammar(final) print final_string ws.send(final_string.upper()) return
def create_sentence_list(text): from nltk.tokenize import PunktSentenceTokenizer p = PunktSentenceTokenizer() return [sentence for sentence in p.tokenize(text)]
31. VBP Verb, non-3rd person singular present 32. VBZ Verb, 3rd person singular present 33. WDT Wh-determiner 34. WP Wh-pronoun 35. WP$ Possessive wh-pronoun 36. WRB Wh-adverb ''' train = state_union.raw("2005-GWBush.txt") # text = state_union.raw("2006-GWBush.txt") text = "George W Bush is the president of United States. Sky is blue and so are you." # PunktSentenceTokenizer is a unsupervised ML tokenizer training = PunktSentenceTokenizer(train) tokenized_text = training.tokenize(text) def process_content(): try: for i in tokenized_text: words = word_tokenize(i) tagged = nltk.pos_tag(words) #print tagged chunk_gram = r"""Chunk: {<RB.?>*<VB.?>*<NNP.?>+<NN>?}""" chunkParser = nltk.RegexpParser(chunk_gram) chunked = chunkParser.parse(tagged) chunked.draw() except Exception as e:
group_seq = Sequence(1) for group in os.listdir(input_dir): group_id = group_seq.id_for(group) sys.stderr.write("%s\t%s\n" % (group, group_id)) for article_path in os.listdir("%s/%s" % (input_dir, group)): # stitch article into a single string article = "" for line in open("%s/%s/%s" % (input_dir, group, article_path)): line = line.strip() if len(line) > 0 and not line.startswith(">"): article += " " article += line # tokenise sentences then some simple normalisation, collect just binary features all_tokens = set() for sentence in sent_tokenizer.tokenize(article): tokens = token_tokenizer.tokenize(sentence) tokens = map(lambda t: t.lower().replace(":","").replace("|",""), tokens) # : and | reserved for vw format tokens = filter(lambda t: len(t) > 3, tokens) all_tokens.update(tokens) # write out token features ; weight of 0.5 sys.stdout.write("1 1 '%s_%s |tokens" % (group, article_path)) # 1 weight=1 label for token in all_tokens: sys.stdout.write(" %s:0.5" % token) # followed by single group feature (with weighting = magic 10, therefore quadratic features have weight 5) sys.stdout.write(" |group %s:10" % group) sys.stdout.write("\n")
import nltk from nltk.corpus import state_union from nltk.tokenize import PunktSentenceTokenizer from nltk.corpus import stopwords #PunktSentenceTokenizer #is the abstract class for the default sentence tokenizer, #i.e. sent_tokenize() train_text = state_union.raw("2005-GWBush.txt") sample_text = state_union.raw("2006-GWBush.txt") custom_sentence_tokenizer = PunktSentenceTokenizer( train_text) #training our module, which is optional tokenized = custom_sentence_tokenizer.tokenize( sample_text ) # you can also do, tokenized = PunktSentenceTokenizer().tokenize(sample_text) def process_content(): try: example_sent = "This is merely an example sentence, which shows the use of stopwords" stop_words = set(stopwords.words("english")) #we can also add our own stop_words stop_words.add("Hiiii") # stopwords are those words which have no meaning, thus removing them is quite appreciated word_tokens = word_tokenize(example_sent) filtered_sentence = [w for w in word_tokens if not w in stop_words] #List-Complrehension # OR
+ = match 1 or more ? = match 0 or 1 repetitions. * = match 0 or MORE repetitions . = Any character except a new line ''' from nltk.tag import pos_tag from nltk.tokenize import PunktSentenceTokenizer, word_tokenize from nltk import RegexpParser sample_text_file = open("../sample.txt", "r") text = sample_text_file.read() pst = PunktSentenceTokenizer() tokenized = pst.tokenize(text) def process_content(): try: for s in tokenized: words = word_tokenize(s) tagged = pos_tag(words) chunkGram = r"""Chunk: {<VB.?>*<NNP>+<NN>?}""" # look for any verb, atleast one proper noun and zero or one noun chunkParser = RegexpParser(chunkGram) chunked = chunkParser.parse(tagged) chunked.draw() # print(chunked) except Exception as e:
Chinking is basically alot of chunking, since the process of Chunking of another chunk is termed as Chinking! Removing of a Chunk from a Chunk. You just need to denote }{ this after the Chunking sequence , so that these are explicit out from the Data! ''' import nltk from nltk.corpus import state_union from nltk.tokenize import PunktSentenceTokenizer train_text = state_union.raw("2005-GWBush.txt") # Adding the Raw text using the state_unio from the Text file sample_text = state_union.raw("2006-GWBush.txt") # Adding the Sample Text With the same process using the state union custom_sent_tokenizer = PunktSentenceTokenizer( train_text) # using the Custom sentence Tokenizer which uses the Punksentence tokenizer for the training of the text tokenized = custom_sent_tokenizer.tokenize( sample_text) # Using the Custom text Tokenizer for the Tokenizing of the sample Text def process_content (): try: for i in tokenized[0:5]: words = nltk.word_tokenize(i) tagged = nltk.pos_tag(words) chunkGram = r"""Chunk: {<.*>+} }<VB.?|IN|DT|TO>+{""" chunkParser = nltk.RegexpParser(chunkGram) # using the regular expression to Regexly parse the Chuncked Parser chunked = chunkParser.parse(tagged) # Using the Parser and adding the Parsing to the Tags which have been formulated
import nltk from nltk.corpus import state_union # An unsupervised machine learning tokenizer(comes pre-trained.) from nltk.tokenize import PunktSentenceTokenizer, word_tokenize # We train the PunktSentenceTokenizer to a clinton speech in 1993 TRAIN_TEXT = state_union.raw("1993-Clinton.txt") # We train the PunktSentenceTokenizer to a clinton speech in 1994 SAMPLE_TEXT = state_union.raw("1994-Clinton.txt") # Actual training of the PunktSentenceTokenizer SENTENCE_TOKENIZER = PunktSentenceTokenizer(TRAIN_TEXT) # Tokenizing using the trained model. TOKENIZED = SENTENCE_TOKENIZER.tokenize(SAMPLE_TEXT) # Processing function. def process_content(): try: for i in TOKENIZED: words = nltk.word_tokenize(i) tagged = nltk.pos_tag(words) print "POS of tagged words", (tagged) except Exception as e: print(str(e)) process_content()
ps = PosterStemmer() example_words = ["python","pythoner","pythoning","pythoned","pythonly"] for w in example_words: print(ps.stem(w)) ##Part of Speech Tagging from nltk.corpus import state_union from nltk.tokenize import PunktSentenceTokenizer #Unsupervised machine learning tokenizer -> PunktSentenceTokenizer train_text = state_union.raw("2005-GWBush.txt") sample_text = state_union.raw("2006-GWBush.txt") custom_sent_tokenizer = PunktSentenceTokenizer(train_text) #training on train_text tokenized = custom_sent_tokenizer.tokenize(sample_text) #applying model to sample_text #this will generate sentences def process_content(): try: for i in tokenized: words= nltk.word_tokenize(i) tagged = nltk.pos_tag(words) print(tagged) except: Exception as e: print(str(e)) process_content() #POS tag list """
#read from txt file # inputText = state_union.raw(os.path.abspath(os.path.join(os.getcwd(),"..\Dataset\RabindranathTagore.txt"))) inputText = state_union.raw( os.path.abspath(os.getcwd() + "\Dataset\RabindranathTagore.txt")) experimentText = state_union.raw( "I:\Information\WorkSpace\AdiRepo\MachineLearning\DataSet\SubhasChandraBose.txt" ) #train tokenizer, if require. trainedTokenizer = PunktSentenceTokenizer() # trainedTokenizer = PunktSentenceTokenizer(inputText) #tokenizing experimentText sentences = trainedTokenizer.tokenize(experimentText) def partOfSpeechTaggig(): for sentence in sentences: words = nltk.word_tokenize(sentence) # pos_tag() take list of words or sentence as input and tag part of speech taggedWords = nltk.pos_tag(words) #region Chunking grammer = R"""Chunk: {<RB.?>*<VB.?>*<NNP>+<NN>*}""" chunckingParser = nltk.RegexpParser(grammer) chunk = chunckingParser.parse(taggedWords) #for tWord in taggedWords:
pa = parser.parse_args() lang = pa.lang filePath = pa.file outputPath = filePath + '.sent' if __name__ == "__main__": file = open(filePath, 'r') output = open(outputPath, 'w') sst = None if lang == 'EN': sst = nltk.data.load('nltk:tokenizers/punkt/english.pickle') elif lang == 'ES': sst = nltk.data.load('nltk:tokenizers/punkt/spanish.pickle') else: sst = PunktSentenceTokenizer() for line in file: if line == "\n": sys.stdout.write(line) continue line = line.replace("«", "'") line = line.replace("»", "'") line = line.replace("“", "'") line = line.replace("”", "'") line = line.replace("\"", "'") sentences = sst.tokenize(line.decode("utf-8")) for s in sentences: output.write((s+'\n').encode('utf-8')) file.close() output.close()
) #------------------------------------------------------------------------------------ Preprocessing # Stop words stop_words = set(stopwords.words('english')) # Initializes the lemmatizer normalizer = WordNetLemmatizer() # Creates an empty list of processed speeches preprocessed_speeches = [] # ---------------------- Preprocessing loop for speech in speeches: # ------------------ Tokenizing # Initializes sentence tokenizer sentence_tokenizer = PunktSentenceTokenizer() # Tokenizes speech into sentences sentence_tokenized_speech = sentence_tokenizer.tokenize(speech) # ------------------ Normalizing loop # Creates an empty sentences list word_sentences = [] for sentence in sentence_tokenized_speech: # ----------- Removes noise from sentence and tokenizes the sentence into words word_tokenized_sentence = [re.sub('[^a-zA-Z0-9]+', '', word.lower()) \ for word in sentence.replace(",", "").replace("-", " ").replace(":", "").split()] # ---------------- Removes stopwords from sentences sentence_no_stopwords = [ word for word in word_tokenized_sentence if word not in stop_words ] # ---------------- Before lemmatizing, adds a 's' to the word 'us' word_sentence_us = [ 'uss' if word == 'us' else word for word in sentence_no_stopwords ]
#!/usr/bin/env python import nltk from nltk.corpus import state_union from nltk.tokenize import PunktSentenceTokenizer #unsupervised tokenizer train_text = state_union.raw('2005-GWBush.txt') #print train_text test_text = state_union.raw('2006-GWBush.txt') custom_sent_token = PunktSentenceTokenizer(train_text) tokenized = custom_sent_token.tokenize(test_text) #print tokenized #print type(tokenized) def chunk(): try: for i in tokenized: words = nltk.word_tokenize(i) tagged = nltk.pos_tag(words) regexp = r"""Chunk: {<RB.?>*<VB.?>*<NNP>+<NN>?} }<VB.?|IN|DT|TO>+{""" parser = nltk.RegexpParser(regexp)
[] = quant[ia]tative = will find either quantitative, or quantatative. [a-z] = return any lowercase letter a-z [1-5a-qA-Z] = return all numbers 1-5, lowercase letters a-q and uppercase A-Z""" # In[3]: train = state_union.raw("2005-GWBush.txt") test = state_union.raw("2006-GWBush.txt") # In[4]: pst = PunktSentenceTokenizer(train) #training the tokenizer # In[5]: tokenised = pst.tokenize(test) # In[7]: for i in tokenised: words = word_tokenize(i) tokenise = nltk.pos_tag(words) chunkgram = r"""chunk :{<RB.?>*<VB.?>*<NNP>+<NN>?}""" chunkprase = nltk.RegexpParser(chunkgram) chunkd = chunkprase.parse(tokenise) print(chunkd) # In[11]:
## file myself or do I feed it straight into NLTK? ## option 1 content = file.read() #print content match = re.findall(r"msgstr\s\"([^\"\\]*)\"", content) match = " ".join(match) match = match.decode('utf8') #print match #print(sent_tokenize(match)) train_text = state_union.raw("2005-GWBush.txt") sample_text = state_union.raw("2006-GWBush.txt") custom_sent_tokenizer = PunktSentenceTokenizer(train_text) tokenized = custom_sent_tokenizer.tokenize(match) def process_content(): try: for i in tokenized: words = nltk.word_tokenize(i) tagged = nltk.pos_tag(words) print(tagged) except Exception as e: print (str(e)) processed_content = process_content() print(processed_content)
#labeling part of speech import nltk from nltk.corpus import state_union from nltk.tokenize import PunktSentenceTokenizer #unsupervised [we can re train] #train_text = state_union.raw("2005-GWBush.txt") #sample_text = state_union.raw("2006-GWBush.txt") train_text = "This is a training text, which consists of manya place name like: India, America and USA. Since Chinahas corona virus in the contry. Donald trump refuse to add them in the meeting." sample_text = "This is a sample text which goes under the program and test out that either it working or not. Contry : India , China, USA and Name : Donald Trump, Modi, Varun better etc." custom_sent_tokenizer = PunktSentenceTokenizer(train_text) #training the text tokenized = custom_sent_tokenizer.tokenize( sample_text) #after training we use it to different data #Give the result about who all are verbs, adjective and all the things. def process_content(): try: for i in tokenized: words = nltk.word_tokenize(i) tagged = nltk.pos_tag(words) print(tagged) except Exception as e: print(str(e)) process_content()
""" Created on Thu Nov 19 09:15:11 2015 @author: nilakant """ import nltk from nltk.corpus import state_union from nltk.tokenize import PunktSentenceTokenizer #unsupervised tokenizer train_text = state_union.raw("2006-GWBush.txt") sample_text = state_union.raw("2006-GWBush.txt") custom_sent_tokenizer = PunktSentenceTokenizer(train_text) tokenized = custom_sent_tokenizer.tokenize(sample_text) def process_content(): try: for i in tokenized: words = nltk.word_tokenize(i) tagged = nltk.pos_tag(words) chunkGram = r"""Chunk: {<.*>+} }<VB.?|IN|DT|TO>+{""" chunkParser = nltk.RegexpParser(chunkGram) chunked = chunkParser.parse(tagged) chunked.draw() #print(tagged)
"--train", type=str, nargs="*", metavar="language(s)", default=None, help="Train a model of given language(s)") # parse the arguments from standard input args = parser.parse_args() if args.intra is not None: if len(args.intra) == 0: sentence = input('Enter a sentence for POS tagging: ') print('sent for POS tagging: {}'.format(sentence)) punktok = PunktSentenceTokenizer() tokenized = punktok.tokenize(text=sentence) POSGenerator = pos.POSGenerator(method='nltk') POSGenerator.process_content(tokenized_text=tokenized) else: # file name input print('file path for POS tagging: {}'.format(args.intra[0])) try: f = open(args.intra[0]) print('file read') except Exception as e: print(e) if args.train is not None: if len(args.train) == 0: print('Please specify a language') else: