def word2vec(corpus_list):
'''
convert the text into vectors after deleting the stop-words
'''
stopwords = [
"a", "able", "about", "above", "according", "accordingly", "across",
"actually", "after", "afterwards", "again", "against", "ain't", "aint",
"ain´t", "ain’t", "all", "allow", "allows", "almost", "alone", "along",
"already", "also", "although", "always", "am", "among", "amongst",
"an", "and", "another", "any", "anybody", "anyhow", "anyone",
"anything", "anyway", "anyways", "anywhere", "apart", "appear",
"appreciate", "appropriate", "are", "aren't", "arent", "aren´t",
"aren’t", "around", "as", "aside", "ask", "asking", "associated", "at",
"available", "away", "awfully", "b", "be", "became", "because",
"become", "becomes", "becoming", "been", "before", "beforehand",
"behind", "being", "believe", "below", "beside", "besides", "best",
"better", "between", "beyond", "both", "brief", "but", "by", "c",
"came", "can", "can't", "cannot", "cant", "can´t", "can’t", "cause",
"causes", "certain", "certainly", "clearly", "co", "com", "come",
"comes", "concerning", "consequently", "consider", "considering",
"containing", "contains", "corresponding", "could", "couldn't",
"couldnt", "couldn´t", "couldn’t", "course", "currently", "d",
"definitely", "despite", "did", "didn't", "didnt", "didn´t", "didn’t",
"do", "does", "doesn't ", "doesnt ", "doesn´t ", "doesn’t ", "doing",
"don't", "done", "dont", "don´t", "don’t", "down", "downwards",
"during", "e", "each", "edu", "eg", "eight", "either", "else",
"elsewhere", "enough", "entirely", "especially", "et", "etc", "even",
"ever", "every", "everybody", "everyone", "everything", "everywhere",
"ex", "exactly", "example", "except", "f", "far", "few", "fifth",
"first", "five", "followed", "following", "follows", "for", "former",
"formerly", "forth", "four", "from", "further", "furthermore", "g",
"get", "gets", "getting", "given", "gives", "go", "goes", "going",
"gone", "got", "gotten", "greetings", "h", "had", "hadn't", "hadnt",
"hadn´t", "hadn’t", "happens", "hardly", "has", "hasn't", "hasnt",
"hasn´t", "hasn’t", "have", "haven't", "havent", "haven´t", "haven’t",
"having", "he", "he'd", "he'll", "he's", "hed", "hell", "hence", "her",
"here", "here's", "hereafter", "hereby", "herein", "heres", "hereupon",
"here´s", "here’s", "hers", "herself", "hes", "he´d", "he´ll", "he´s",
"he’d", "he’ll", "he’s", "hi", "him", "himself", "his", "hither",
"hopefully", "how's", "howbeit", "however", "hows", "how´s", "how’s",
"i", "i'd", "i'll", "i'm", "i've", "id", "ie", "if", "ignored", "ill",
"im", "immediate", "in", "inasmuch", "inc", "indeed", "indicate",
"indicated", "indicates", "inner", "insofar", "instead", "into",
"inward", "is", "isn't", "isnt", "isn´t", "isn’t", "it", "it's", "its",
"itself", "it´s", "it’s", "ive", "i´d", "i´ll", "i´m", "i´ve", "i’d",
"i’ll", "i’m", "i’ve", "j", "just", "k", "keep", "keeps", "kept",
"known", "knows", "l", "last", "lately", "later", "latter", "latterly",
"least", "less", "lest", "let", "let's", "lets", "let´s", "let’s",
"like", "liked", "likely", "little", "look", "looking", "looks", "ltd",
"m", "mainly", "many", "may", "maybe", "me", "mean", "meanwhile",
"merely", "might", "more", "moreover", "most", "mostly", "much",
"must", "mustn't", "mustnt", "mustn´t", "mustn’t", "my", "myself", "n",
"namely", "nd", "near", "nearly", "necessary", "need", "needs",
"neither", "never", "nevertheless", "nine", "no", "nobody", "non",
"none", "noone", "nor", "normally", "not", "nothing", "novel", "now",
"nowhere", "o", "obviously", "of", "off", "often", "oh", "on", "once",
"one", "ones", "only", "onto", "or", "other", "others", "otherwise",
"ought", "our", "ours", "ourselves", "out", "outside", "over",
"overall", "own", "p", "particular", "particularly", "per", "perhaps",
"placed", "please", "plus", "possible", "presumably", "probably",
"provides", "q", "que", "quite", "qv", "r", "rather", "rd", "re",
"really", "reasonably", "regarding", "regardless", "regards",
"relatively", "respectively", "right", "s", "said", "same", "saw",
"say", "saying", "says", "second", "secondly", "see", "seeing", "seem",
"seemed", "seeming", "seems", "seen", "self", "selves", "sensible",
"sent", "serious", "seriously", "seven", "several", "shall", "shan't",
"shant", "shan´t", "shan’t", "she", "she'd", "she'll", "she's", "shed",
"shell", "shes", "she´d", "she´ll", "she´s", "she’d", "she’ll",
"she’s", "should", "shouldn't", "shouldnt", "shouldn´t", "shouldn’t",
"since", "six", "so", "some", "somebody", "somehow", "someone",
"something", "sometime", "sometimes", "somewhat", "somewhere", "soon",
"specified", "specify", "specifying", "still", "sub", "such", "sup",
"sure", "t", "take", "taken", "tell", "tends", "th", "than", "that",
"that's", "thats", "that´s", "that’s", "the", "their", "theirs",
"them", "themselves", "then", "thence", "there", "there's",
"thereafter", "thereby", "therefore", "therein", "theres", "thereupon",
"there´s", "there’s", "these", "they", "they'd", "they'll", "they're",
"they've", "theyd", "theyll", "theyre", "theyve", "they´d", "they´ll",
"they´re", "they´ve", "they’d", "they’ll", "they’re", "they’ve",
"think", "third", "this", "thorough", "thoroughly", "those", "though",
"three", "through", "throughout", "thru", "thus", "to", "together",
"too", "took", "toward", "towards", "tried", "tries", "truly", "try",
"trying", "twice", "two", "u", "un", "under", "unfortunately",
"unless", "unlikely", "until", "unto", "up", "upon", "us", "use",
"used", "useful", "uses", "using", "usually", "uucp", "v", "various",
"very", "via", "viz", "vs", "w", "want", "wants", "was", "wasn't",
"wasnt", "wasn´t", "wasn’t", "way", "we", "we'd", "we'll", "we're",
"we've", "wed", "welcome", "well", "went", "were", "weren't", "werent",
"weren´t", "weren’t", "weve", "we´d", "we´ll", "we´re", "we´ve",
"we’d", "we’ll", "we’re", "we’ve", "whatever", "whereafter", "whereas",
"whereby", "wherein", "whereupon", "wherever", "whither", "whoever",
"whole", "will", "willing", "wish", "with", "within", "without",
"won't", "wonder", "wont", "won´t", "won’t", "would", "wouldn't",
"wouldnt", "wouldn´t", "wouldn’t", "x", "y", "yet", "you", "you'd",
"you'll", "you're", "you've", "youd", "youll", "your", "youre",
"yours", "yourself", "yourselves", "youve", "you´d", "you´ll",
"you´re", "you´ve", "you’d", "you’ll", "you’re", "you’ve", "z", "zero"
] + [
'ain', 'aren', 'couldn', 'didn', 'doesn', 'doesnt', 'don', 'hadn',
'hasn', 'haven', 'isn', 'll', 'mustn', 'shan', 'shouldn', 've', 'wasn',
'weren', 'won', 'wouldn'
]
# stopwords = ENGLISH_STOP_WORDS.union(stopwords)
stopwords = stopwords
# texts is the list of all the questions, and each of the question is a str of selected words
# texts = [' '.join([word for word in doc.lower().split() if word not in stopwords]) for doc in corpus_list]
# tf-idf
vect = CountVectorizer(stop_words=stopwords).fit(corpus_list)
feature_names = vect.get_feature_names() # words
# print(feature_names)
trans_vect = vect.transform(corpus_list)
tfidf = TfidfTransformer().fit_transform(trans_vect)
max_value = tfidf.max(axis=0).toarray().ravel()
sorted_by_tfidf = max_value.argsort()
# print([feature_names[i] for i in sorted_by_tfidf[-40:]])
# print(tfidf.toarray())
return tfidf
