def make_140_scores_lexicon(): polarities = {} for line in util.lines( constants.LEXICONS + "Sentiment140-Lexicon-v0.1/unigrams-pmilexicon.txt"): info = line.split() polarities[info[0]] = float(info[1]) util.write_json(polarities, constants.PROCESSED_LEXICONS + '140-scores.json')
def make_kuperman_scores_lexicon(): polarities = {} for i, line in enumerate( util.lines(constants.LEXICONS + "kuperman/raw_ratings.csv")): if i == 0: continue info = line.split(",") if len(info[1].split()) == 1: polarities[info[1]] = float(info[2]) util.write_json(polarities, constants.PROCESSED_LEXICONS + 'kuperman.json')
def make_qwn_scores_lexicon(): polarities = collections.defaultdict(float) for line in util.lines( constants.LEXICONS + "qwn/turneyLittman_propSyn_08_mcr30-noAntGloss.dict"): info = line.split("\t") mod = float(info[3]) for word in info[2].split(", "): if "_" not in word: polarities[word.split("#")[0]] += mod util.write_json(polarities, constants.PROCESSED_LEXICONS + 'qwn-scores.json')
def make_twitter_scores_lexicon(): polarities = {} for line in util.lines( constants.LEXICONS + "twitter/MaxDiff-Twitter-Lexicon/Maxdiff-Twitter-Lexicon_-1to1.txt" ): info = line.split() if len(info[1].split()) > 1: continue polarities[info[1]] = float(info[0]) util.write_json(polarities, constants.PROCESSED_LEXICONS + 'twitter-scores.json')
def make_mpqa_lexicon(): polarities = {} for line in util.lines(constants.LEXICONS + 'mpqa_subjectivity.txt'): split = line.strip().split() w = split[2].split("=")[1] polarity = split[-1].split("=")[1] if polarity == 'neutral': polarities[w] = 0 elif polarity == 'positive': polarities[w] = 1 else: polarities[w] = -1 util.write_json(polarities, constants.PROCESSED_LEXICONS + 'mpqa.json')
def make_bingliu_lexicon(): polarities = {} for polarity in ['positive', 'negative']: for line in util.lines( constants.LEXICONS + 'bl_opinion_lexicon/{:}-words.txt'.format(polarity)): try: line = line.strip().encode('ascii', 'ignore') if len(line) == 0 or line[0] == ';': continue polarities[line] = 1 if polarity == 'positive' else -1 except: print("skipping", line) util.write_json(polarities, constants.PROCESSED_LEXICONS + 'bingliu.json')
def make_finance_lexicon(): fp = open(constants.LEXICONS + "finance.csv") fp.readline() polarities = {} for line in fp: info = line.split(",") word = info[0].lower() if info[7] != '0': polarities[word] = -1 elif info[8] != '0': polarities[word] = 1 else: polarities[word] = 0 util.write_json(polarities, constants.PROCESSED_LEXICONS + "finance.json")
def make_concreteness_lexicon(top=75, bottom=25): raw_scores = {} fp = open(constants.LEXICONS + "concreteness/raw_ratings.csv") fp.readline() for line in fp: info = line.split(",") if len(info[0].split()) > 1: continue raw_scores[info[0]] = float(info[2]) pos_thresh = np.percentile(raw_scores.values(), top) neg_thresh = np.percentile(raw_scores.values(), bottom) polarities = {} label_func = lambda s: 1 if s > pos_thresh else -1 if s < neg_thresh else 0 for word, score in raw_scores.items(): polarities[word] = label_func(score) util.write_json(polarities, constants.PROCESSED_LEXICONS + "concreteness.json")
def make_qwn_lexicon(): polarities = collections.defaultdict(float) for line in util.lines( constants.LEXICONS + "qwn/turneyLittman_propSyn_08_mcr30-noAntGloss.dict"): info = line.split("\t") if info[1] == "neg": mod = -1 else: mod = 1 for word in info[2].split(", "): if "_" not in word: polarities[word.split("#")[0]] += mod polarities = { word: np.sign(val) for word, val in polarities.items() if val != 0 } util.write_json(polarities, constants.PROCESSED_LEXICONS + 'qwn.json')
def make_inquirer_lexicon(): polarities = {} for line in util.lines(constants.LEXICONS + 'inquirerbasic.csv'): for l in line.strip().split('\r'): split = l.split(",") w = split[0].lower() if "#" in w: if w.split("#")[1] != "1": continue w = w.split("#")[0] polarity_neg = split[-1] polarity_pos = split[-2] if polarity_neg == 'Negativ' and polarity_pos == 'Positiv': continue elif polarity_neg == 'Negativ': polarities[w] = -1 elif polarity_pos == 'Positiv': polarities[w] = 1 else: polarities[w] = 0 util.write_json(polarities, constants.PROCESSED_LEXICONS + 'inquirer.json')