def clean_preprocess(text, do_farasa_tokenization, farasa): text = " ".join( bt._run_split_on_punc( preprocess(text, do_farasa_tokenization=do_farasa_tokenization, farasa=farasa))) text = " ".join(text.split()) #removes extra whitespaces return text
def transform(self, sentences_list, extract_and_paste_emojies=False): """ transforming data and applying all pre-processing steps over it. In case 'fit' is required, it will yiled an error in case data is not fitted yet Parameters ---------- :param sentences_list: list (of arabic sentences) list of sentences to apply the function on. Each sentence is treated independently :param extract_and_paste_emojies: boolean. Default: False whether to handle emojies is a special way. Currently we only know how to handle emojies in a very specific way (extract them and then paste them at the end of the sentence - not ideal) TODO: handle empjies in a better way (do not convert them to ?? and "leave" them as is in the sentence) :return: list list of transformed arabic sentences. Same input list, but after the transform function has been applied over all of them """ farasa_segmenter = FarasaSegmenter(interactive=True) new_sentences_list = list() # looping over each sentence for cur_text in sentences_list: # in case we decided to use the farase preprocess if self.use_default_farsa_preprocess: preprocessed_text = preprocess(cur_text, do_farasa_tokenization=True, farasa=farasa_segmenter, use_farasapy=True) preprocessed_text_as_list = preprocessed_text.split(" ") # removal of punctuation (e.g., '?', '!?!') preprocessed_text_as_list = [ cur_word for cur_word in preprocessed_text_as_list if not all(j in string.punctuation for j in cur_word) ] if extract_and_paste_emojies: emojies_found = self.extract_emojis(text=cur_text) preprocessed_text_as_list.extend(emojies_found) new_sentences_list.append(' '.join(preprocessed_text_as_list)) # currently not doing anything in such case, only supports the default case else: new_sentences_list.append(cur_text) return new_sentences_list