def count_sentiment_for_list_test(self): sentiment = Sentiment(output_results='') sentiment_after = sentiment.count_sentiment_for_list( document_tokens=['this', 'is', 'my', 'string'], lexicon={'this': 1, 'is': 1, 'my': 1, 'string': 1, 'this is': -2, 'is my': -5, 'nice evening': -99, 'is m': -999}) self.assertEqual(4, sentiment_after)
def count_sentiment_for_list_bigrams(self): sentiment = Sentiment(output_results='') document_tokens = ['not so good', 'bad', 'very good', 'very', 'really nice'] lexicon = {'good': 3, 'awesome': 2, 'neutral': 0, 'very good': 4, 'not good': -2, 'bad': -2} sentiment_before = 2 sentiment_after = sentiment.count_sentiment_for_list(document_tokens, lexicon) self.assertEqual(sentiment_before, sentiment_after)
def count_sentiment_for_list_unigrams(self): sentiment = Sentiment(output_results='') document_tokens = ['good', 'good', 'bad', 'tri gram test'] lexicon = {'good': 3, 'bad': -3, 'awesome': 2, 'neutral': 0, 'very good': 4, 'tri gram test': 7 } sentiment_before = 10 sentiment_after = sentiment.count_sentiment_for_list(document_tokens, lexicon) self.assertEqual(sentiment_before, sentiment_after)
def sentiment_ngrams_selection(dataset, max_features=[None], classifiers=None, features_ngrams=None, bins=10): logging.info('Starting with %s' % dataset) ml_predictions = {} sentiment = Sentiment() # logging.basicConfig(filename='processing.log', level=logging.DEBUG, # format='%(asctime)s - sentiment_ngrams.py - ' # '%(levelname)s - %(message)s') # dataset='Movies_&_TV1200.csv' -> feature_space_size 1200 x 116495 logging.info('# max features = %s' % max_features) if max_features is None: max_features = [None] else: max_features = max_features thresholds = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1] max_features = [math.floor(x * max_features) for x in thresholds] for mf in max_features: logging.info('Starting with %s and %s features' % (dataset, mf)) for n_gram_name, n_grams_range in features_ngrams.iteritems(): logging.info('Starting with %s' % n_gram_name) # print 'CountVectorizer' f_name = n_gram_name + '_CountVectorizer' classes, ml_prediction, results_ml = sentiment.supervised_sentiment( dataset=dataset, # worksheet_name='Arkusz1', n_gram_range=n_grams_range, n_folds=10, classifiers={'LinearSVC': LinearSVC()}, # classifiers=None, # all classifier available in sentiment class # classifiers=classifiers, amazon=True, lowercase=True, stop_words='english', max_df=1.0, min_df=0.0, max_features=mf, f_name_results=f_name, vectorizer='CountVectorizer', # tokenizer=document_preprocessor.tokenizer_with_stemming ) ml_predictions.update(ml_prediction) results_to_pickle(dataset, '%s-%s-%s' % (n_gram_name, f_name, str(mf)), results_ml)
# -*- coding: utf-8 -*- __author__ = 'Łukasz Augustyniak' import pandas as pd from textlytics.sentiment.sentiment import Sentiment from textlytics.sentiment.document_preprocessing import DocumentPreprocessor df = pd.read_csv('C:\Users\Dell\Documents\GitHub\word2vec\d2v-vs-bow\Automotive9600.csv') dp = DocumentPreprocessor() df, _ = dp.star_score_to_sentiment(df, score_column='Stars', star_mean_score=3) s = Sentiment() df_lex, lexicon_prediction, lexicon_result, classes = \ s.lex_sent_batch( df=df, lexs_files=['amazon_automotive_25_w2v_all.txt', 'amazon_automotive_25.txt'], words_stem=False, dataset_name='word_vectorization') print lexicon_result
def get_dataset_with_kfolds_indexes(base_path, output_folder, dataset_filter, n_reviews=2000, n_cv=10, vectorizer_type='CountVectorizer', stars=None, model=None): """ Main function for getting data and all necessary setting to start up supervised learning approach for sentiment analysis based on Amazon data with predefined cross-validation folds. Parameters ---------- base_path : string Path to all folders and files needed in analysis, e.g, csv files with amazon data. output_folder : string Path to the directory where all outcomes of the experiment will be stored. dataset_filter : string Filter files nas for dataset that will be used in the experiment. n_reviews : int, 2000 by default Number of reviews from each dataset to use in analysis. n_cv : int, 10 by default Number of Cross-Validation folds that will be used in experiment. vectorizer_type : object, as default - CounterVectorizer (Scikit-Learn). Type of vectorizer that will be used to build feature vector. stars : list List of stars that will be mapped into sentiment. model : gensim.Doc2Vec Model that will convert list of documents into list of document's vectors. """ datasets = glob.glob( path.join(base_path, '*%s*.csv' % dataset_filter)) log.info('Datasets will be used in experiment: {}'.format(datasets)) if not exists(output_folder): makedirs(output_folder) log.info('New directory has been created in: {}'.format(output_folder)) for dataset in datasets: dataset_name = path.basename(dataset).split('.')[0] log.info('Dataset name: %s' % dataset_name) dp = DocumentPreprocessor() df = pd.DataFrame.from_csv(dataset, sep=';', index_col=False) # filter stars in reviews if stars is not None: df = df[df['review/score'].isin(stars)] df, _ = dp.star_score_to_sentiment(df, score_column='review/score') # extract only Document and Sentiment columns df['Document'] = df['review/text'] df = df[['Sentiment', 'Document']] indexes_all = set(df.index) log.info('All indexes: {}'.format(len(indexes_all))) predictions = [] results = [] log.info('Vectorizer type processed: {}'.format(vectorizer_type)) f_name = 'Supervised-learning-{}-{}-{}'.format( dataset_name, vectorizer_type, list_to_str(stars)) s = Sentiment(dataset_name=dataset_name) classes, ml_prediction, results_ml = s.supervised_sentiment( docs=df['Document'], y=np.array(df['Sentiment']), classifiers=ALL_CLASSIFIERS, f_name_results=f_name, vectorizer=vectorizer_type, n_folds=n_cv, model=model, ) results.append(results_ml) predictions.append(ml_prediction) to_pickle(p=output_folder, dataset=dataset_name, f_name=f_name, obj=results)
def imdb_sentiment(n_cv=10, vectorizer_type='CountVectorizer'): """ Main function for getting data and all necessary setting to start up supervised learning approach for sentiment analysis based on IMDB data. Parameters ---------- n_cv : int, 10 by default Number of Cross-Validation folds that will be used in experiment. vectorizer_type : object, as default - CounterVectorizer (Scikit-Learn). Type of vectorizer that will be used to build feature vector. """ dataset_name = 'IMDB' dataset = Dataset() df = dataset.load_several_files() features_ngrams = { # 'unigrams': (1, 1), 'n_grams_1_2': (1, 2), # 'n_grams_1_3': (1, 3), } max_features = 3000 clfs = { 'BernoulliNB': BernoulliNB(), 'LogisticRegression': LogisticRegression(), 'LinearSVC': LinearSVC(), } predictions = [] results = [] for n_gram_name, n_grams_range in features_ngrams.iteritems(): log.info('Ngram type processed: {}'.format(n_gram_name)) log.info('Vectorizer type processed: {}'.format(vectorizer_type)) f_name = 'Supervised-learning-{}-{}-{}-{}'.format( n_gram_name, max_features, vectorizer_type, dataset_name) s = Sentiment(dataset_name=dataset_name) classes, ml_prediction, results_ml = s.supervised_sentiment( docs=df['Document'], y=df['Sentiment'], n_gram_range=n_grams_range, classifiers=clfs, lowercase=True, stop_words='english', max_df=1.0, min_df=0.0, max_features=max_features, f_name_results=f_name, vectorizer=vectorizer_type, n_folds=n_cv, ) results.append(results_ml) predictions.append(ml_prediction) results_to_pickle(dataset=dataset_name, f_name='predictions-%s' % f_name, obj=ml_prediction) results_to_pickle(dataset=dataset_name, f_name=f_name, obj=results) results_to_pickle(dataset=dataset_name, f_name='predictions-%s' % f_name, obj=predictions)
def sentiment_lexicons_imdb(lexs_names=None, lex_path=None, output_folder=None, evaluate=True): """ Counting sentiment analysis tasks with lexicon for IMDB Dataset with predefined Cross-Validation split. Parameters ---------- lexs_names : list List of path/file names for lexicons loading. lex_path: str Path to the directory with lexicon's files. output_folder : str Path where we want to save our results. evaluate : bool, True by default If true the metrics for analysis will be counted, otherwise only prediction will be saved. Returns ---------- Nothing, all necessary files will be saved automatically. """ dataset_name = 'IMDB' results = [] predictions = [] predictions_directory = join(output_folder, 'predictions') if not exists(output_folder): makedirs(output_folder) log.info('New directory has been created in: {}'.format(output_folder)) if not exists(predictions_directory): makedirs(predictions_directory) log.info('Directory for predictions has been created: {}'.format( predictions_directory)) dataset = Dataset() df = dataset.load_several_files() log.info('Pre-processing phase starts!') dp = DocumentPreprocessor() df.Document = [dp.remove_numbers(doc) for doc in df.Document] sent_lex = SentimentLexicons(stemmed=False, lexicons_path=lex_path) lexicons = sent_lex.load_lexicons(lexicons_file_names=lexs_names) s = Sentiment(n_jobs=len(lexs_names), output_results=output_folder) df_lex, lexicon_prediction, lexicon_result, classes = \ s.lex_sent_batch( df=df, dataset_name=dataset_name, lexicons=lexicons) results.append(lexicon_result) predictions.append(lexicon_prediction) to_pickle(f_path=join(output_folder, '{}-{}.pkl'.format(dataset_name, 'results')), obj=results)
def sentiment_lexicons_amazon_cv(datasets_path='', dataset_filter=None, lexs_names=None, n_reviews=2000, train=False, norm_freq=False, lex_path=None, f_name='', n_cv=10, stars=None, frequentiment_lexicons_path='', output_folder=None, evaluate=True): """ Counting sentiment analysis tasks with lexicon for Amazon Dataset with predefined Cross-Validation split. Parameters ---------- frequentiment_lexicons_path : str Path to frequentiment lexicons (csv files with comma as separator). datasets_path: str Path to the datasets directory, it must contain folder/train_test_subsets with cross-validation information - data frame indexes for each fold. Datasets are in csv files - converted from Web Amazon dataset structure. dataset_filter : list List of substring for choosing datasets. lexs_names : list List of path/file names for lexicons loading. n_reviews: int, 2000 by default. number of reviews from each star score. train : bool, False by default. If True you are counting sentiment for train subsets, otherwise counting sentiment for testing subsets. norm_freq : tuple with floats Tuple, i.e., (-1, 1) fot threshold cutting, lower than first value of tuples will be negative, between -1 and 1 will be neutral more than 1 will be positive. lex_path: str Path to the directory with lexicon's files. f_name : str Additional part of output files name (results, predictions). n_cv : int, 10 by default Number of Cross-Validation's folds to performed. stars : list Star scores that will be used in experiment, as default all. output_folder : str Path where we want to save our results. evaluate : bool, True by default If true the metrics for analysis will be counted, otherwise only prediction will be saved. Returns ---------- Nothing, all necessary files will be saved automatically. """ results = {} predictions = {} datasets = [] predictions_directory = join(output_folder, 'predictions') if not exists(output_folder): makedirs(output_folder) log.info('New directory has been created in: {}'.format(output_folder)) if not exists(predictions_directory): makedirs(predictions_directory) log.info('Directory for predictions has been created: {}'.format( predictions_directory)) # lexs_names = [x.split('.')[0] for x in lexs_files] train_test_path = join(datasets_path, 'train_test_subsets') # get all datasets and cv if dataset_filter is not None: for df in dataset_filter: datasets.extend(glob(join(datasets_path, '{}.csv'.format(df)))) else: datasets = glob(join(datasets_path, '*.txt.gz.csv')) log.debug('Datasets to process: {}'.format(datasets)) if frequentiment_lexicons_path: log.debug('Freq lexs path: {}'.format( join(frequentiment_lexicons_path, '*.csv'))) freq_lexs = glob(join(frequentiment_lexicons_path, '*.csv')) else: freq_lexs = [] for dataset in datasets: # load Amazon data dataset_file_name = basename(dataset) dataset_name = dataset_file_name.split('.')[0] log.info('Dataset name: %s' % dataset_name) dp = DocumentPreprocessor() df = pd.DataFrame.from_csv(dataset, sep=';', index_col=False) if stars is not None: df = df[df['review/score'].isin(stars)] df, _ = dp.star_score_to_sentiment(df, score_column='review/score') # extract only Document and Sentiment columns df['Document'] = df['review/text'] # df['Document'] = df['review/summary'] df = df[['Document', 'Sentiment']] log.info('Pre-processing phase starts!') df.Document = [dp.remove_numbers(doc) for doc in df.Document] try: # load train/test sets folds f_path = join(train_test_path, 'train-test-%s-%s.pkl' % (n_reviews, dataset_name)) with open(f_path, 'rb') as f: train_test_indexes = pickle.load(f) log.info('Pickle has been loaded, %s' % f_path) results[dataset_name] = [] predictions[dataset_name] = [] # iterate over all cross-validation subsets for cv_idx, cv in enumerate(train_test_indexes[:n_cv]): log.info('Start for {}: CV: {}/{} '.format( dataset_name, cv_idx + 1, n_cv)) freq_lexs_ = [ basename(x) for x in freq_lexs if '{}-{}'.format(dataset_name, cv_idx) in x ] log.info( 'Dataset: {}, CV: {} => frequentiment lexicons: {}'.format( dataset_name, cv_idx, freq_lexs_)) lexs_names.extend(freq_lexs_) if stars is not None: cv = (set(cv[0]).intersection(df.index.values), set(cv[1]).intersection(df.index.values)) sent_lex = SentimentLexicons(stemmed=False, lexicons_file_names=lexs_names, lexicons_path=lex_path) lexicons = sent_lex.load_lexicons( lexicons_file_names=lexs_names, lexicons_path=lex_path) if train: ind = cv[0] f_name = 'train-%s-fold-%s' % (dataset_name, cv_idx) else: ind = cv[1] f_name = 'test-%s-fold-%s' % (dataset_name, cv_idx) s = Sentiment() df_lex, lexicon_prediction, lexicon_result, classes = \ s.lex_sent_batch( df=df.ix[ind], dataset_name=dataset_name, lexicons=lexicons, evaluate=evaluate) results[dataset_name].append(lexicon_result) predictions[dataset_name].append(lexicon_prediction) to_pickle(p=output_folder, dataset='', f_name=f_name, obj=lexicon_prediction) # df_lex.to_excel(join(output_folder, 'predictions', 'predictions-%s.xls' % f_name)) df_lex.to_pickle( join(output_folder, 'predictions', 'predictions-%s.pkl' % f_name)) # save predictions # results_to_pickle(dataset=dataset_name, # f_name='Part-predictions-%s' % f_name, # obj=lexicon_prediction) except IOError as err: log.error('%s not loaded' % dataset_name) raise IOError(str(err)) to_pickle(p=output_folder, dataset='', f_name='Results', obj=results) to_pickle(p=output_folder, dataset='', f_name='Predictions', obj=predictions)
def ensemble_lex_clf(stars, lex_test_path=None, lex_train_path=None, datasets=None, new_lex_path='', unigrams_path=None, clfs=[], lexs=[], supervised=False, res_name='all', n_folds=10, csv_path='/datasets/amazon-data/csv/', binary=False, freq=None): """ Counting sentiment orientation for lexicon and/or classifier's predictions with ensemble classification. """ if datasets is None: datasets = [ 'Automotive', 'Books', 'Clothing_&_Accessories', 'Electronics', 'Health', 'Movies_&_TV', 'Music', 'Sports_&_Outdoors', 'Toys_&_Games', 'Video_Games' ] log.info('Experiments will be conducted for these datasets: {}'.format( datasets)) f_unigrams = glob(path.join(unigrams_path, '*.pkl')) log.debug('f_unigrams: %s' % f_unigrams[0]) for dataset in datasets: log.info('%s dataset is starting' % dataset) results = [] predictions = [] feature_list = [] f_unigrams_ = [x for x in f_unigrams if dataset in x] unigrams_pred = pd.read_pickle(f_unigrams_[0]) # CV folds for i in xrange(10): start = datetime.now() log.info('#%s CV is started' % i) f_lex_test = 'predictions-test-%s-fold-%s.pkl' % (dataset, i) f_lex_train = 'predictions-train-%s-fold-%s.pkl' % (dataset, i) # get indexes with proper star values g = glr.GenerateLexicons(csv_path=csv_path) g.get_reviews(filter_str=dataset, file_type='csv') review = g.reviews[dataset] log.debug('Columns in review df: {}'.format(review.columns)) # log.debug('Review Score: {}'.format(review['review/score'])) idx_scores = review[review['review/score'].isin(stars)].index u = review[review['review/score'].isin(stars)]['review/score'] log.debug('Unique stars in DF: {}'.format(set(u))) log.debug('#scores: {}'.format(len(u))) # ######################## lexicons-based approach ################ df_test = pd.read_pickle(path.join(lex_test_path, f_lex_test)) df_train = pd.read_pickle(path.join(lex_train_path, f_lex_train)) # df_frequentiment_lex = pd.read_csv(path.join(new_lex_path, # '%s-%s.csv' # % (dataset, i)), # index_col=0, # names=['unigrams', 'bigrams', # 'trigrams'], # skiprows=1) if new_lex_path: df_frequentiment_lex = merge_frequentiment_predictions( cv=i, dataset_name=dataset, lex_generated_path=new_lex_path, freq=freq) df_test = pd.merge(df_test, df_frequentiment_lex, left_index=True, right_index=True, how='left') log.debug('Frequentiment lex: {}'.format(df_test.columns)) # ######################## supervised learning ################### if supervised: # get sentiment labels/classes from dataset (evaluation and fitting) df_uni_test = pd.DataFrame.from_dict({ k: v for k, v in unigrams_pred[i].iteritems() if 'train' not in k }) df_uni_train = pd.DataFrame.from_dict({ k: v for k, v in unigrams_pred[i].iteritems() if 'train' in k }) df_train = pd.merge(df_train, df_uni_train, left_index=True, right_index=True, how='left') df_test = pd.merge(df_test, df_uni_test, left_index=True, right_index=True, how='left') # only lexicons or supervised too? if stars is not None: idx_test = list(set(df_test.index).intersection(idx_scores)) log.debug('idx_scores test: {}, \nlen: {}'.format( sorted(idx_test[:10]), len(idx_test))) # log.debug('!!! {}'.format(len(idx_test))) # log.debug('DF: {}'.format(len(df_test.index))) classes_test = np.asarray(df_test.ix[idx_test].Sentiment) df_test = df_test.ix[idx_test] # idx_train = df_train.Sentiment.isin(stars) idx_train = list( set(df_train.index).intersection(set(idx_scores))) log.debug('idx_scores train: {}'.format(len(idx_train))) classes_train = np.asarray(df_train.ix[idx_train].Sentiment) df_train = df_train.ix[idx_train] else: classes_test = np.asarray(df_test.Sentiment) classes_train = np.asarray(df_train.Sentiment) l = list(df_train.columns) for j, x in enumerate(l): if 'cv' in x: l[j] = x.split('-')[-1] if '-train' in x: l[j] = x.split('-')[0] log.info('Lexicons: %s' % l) df_train.columns = l # TODO poprawić, bo bardzo nie ładnie napisane lexs = [ x for x in df_train.columns if x not in ['Document', 'Sentiment'] ] log.info('As feature set this columns have been chosen: {}'.format( lexs)) df_test = df_test[clfs + lexs] df_train = df_train[clfs + lexs] df_test = df_test[clfs + lexs] df_train = df_train[clfs + lexs] df_train = df_train.sort(axis=1) df_test = df_test.sort(axis=1) log.debug('Unique classes Test: {}'.format(set(classes_test))) log.debug('Unique classes Train: {}'.format(set(classes_train))) log.debug('DF train shape: {}'.format(df_train.shape)) log.debug('DF test shape: {}'.format(df_test.shape)) s = Sentiment() pred = s.sentiment_classification( X=df_train.as_matrix().astype(np.int), y=classes_train, X_test=df_test.as_matrix().astype(np.int), y_test=classes_test, n_folds=None, classifiers=ALL_CLASSIFIERS, kfolds_indexes=[(df_train.index, df_test.index)], save_clf=False, cv_normal=False) results_ml = s.results # add information about chosen lexicons and classifiers results_ml['lexs-clfs'] = lexs + clfs log.info('Results: %s' % results_ml) results.append(results_ml) predictions.append(pred) log.info('Flow for dataset %s, #%s CV end' % (dataset, i)) stop = datetime.now() log.info('It took %s seconds ' % (stop - start).seconds) results_to_pickle(dataset, 'ensemble-%s' % res_name, results)
def amazon_supervised(base_path, output_folder, dataset_filter, n_cv=10, vectorizer_type='CountVectorizer', stars=None, stars_dist=None): """ Main function for getting data and all necessary setting to start up supervised learning approach for sentiment analysis based on Amazon data with predefined cross-validation folds. Parameters ---------- base_path : string Path to all folders and files needed in analysis, e.g, csv files with amazon data. output_folder : string Path to the directory where all outcomes of the experiment will be stored. dataset_filter : string Filter files nas for dataset that will be used in the experiment. n_reviews : int, 2000 by default Number of reviews from each dataset to use in analysis. n_cv : int, 10 by default Number of Cross-Validation folds that will be used in experiment. vectorizer_type : object, as default - CounterVectorizer (Scikit-Learn). Type of vectorizer that will be used to build feature vector. stars : list List of stars that will be mapped into sentiment. """ datasets = glob.glob(path.join(base_path, '*%s*.csv' % dataset_filter)) log.info('Datasets will be used in experiment: {}'.format(datasets)) if not exists(output_folder): makedirs(output_folder) log.info('New directory has been created in: {}'.format(output_folder)) for dataset in datasets: dataset_name = path.basename(dataset).split('.')[0] log.info('Dataset name: %s' % dataset_name) dp = DocumentPreprocessor() df = pd.DataFrame.from_csv(dataset, sep=';', index_col=False) # filter stars in reviews if stars is not None: dp = DocumentPreprocessor() if stars_dist is not None: df = df.ix[dp.get_reviews(df, 'review/score', stars_dist)] df = df[df['review/score'].isin(stars)] df, _ = dp.star_score_to_sentiment(df, score_column='review/score') # extract only Document and Sentiment columns df['Document'] = df['review/text'] df = df[['Sentiment', 'Document']] log.info('All indexes: {}'.format(len(set(df.index)))) try: features_ngrams = { 'unigrams': (1, 1), 'n_grams_1_2': (1, 2), # 'n_grams_1_3': (1, 3), } log.info('Feature ngrams: {}'.format(features_ngrams)) results = [] for n_gram_name, n_grams_range in features_ngrams.iteritems(): log.info('Ngram type processed: {}'.format(n_gram_name)) log.info( 'Vectorizer type processed: {}'.format(vectorizer_type)) f_name = 'Supervised-learning-{}-{}-{}-n_reviews-{}'.format( vectorizer_type, n_gram_name, '-'.join([str(s) for s in stars]), min(stars_dist.values())) s = Sentiment(dataset_name=dataset_name, output_results=output_folder) log.info('Chosen dataframe subset is %s x %s' % df.shape) classes, ml_prediction, results_ml = s.supervised_sentiment( docs=df['Document'], y=np.array(df['Sentiment']), n_gram_range=n_grams_range, classifiers=ALL_CLASSIFIERS, lowercase=True, stop_words='english', # max_df=1.0, # min_df=0.0, max_features=50000, f_name_results=f_name, vectorizer=vectorizer_type, n_folds=n_cv, ) results.append(results_ml) except IOError as err: log.error('%s not loaded' % dataset_name) raise IOError(str(err)) to_pickle(f_path=join(output_folder, 'resutls-super-example.pkl'), obj=results)
def sentiment_doc2vec_amazon_cv(base_path, output_folder, dataset_filter, n_reviews=2000, n_cv=10, vectorizer_type='doc2vec', stars=None, model=None, n_max_unsupervised=None, d2v_size=100, save_model=None): """ Main function for getting data and all necessary setting to start up supervised learning approach for sentiment analysis based on Amazon data with predefined cross-validation folds. Parameters ---------- base_path : string Path to all folders and files needed in analysis, e.g, csv files with amazon data. output_folder : string Path where to save results. dataset_filter : string Filter files nas for dataset that will be used in the experiment. n_reviews : int, 2000 by default Number of reviews from each dataset to use in analysis. n_cv : int, 10 by default Number of Cross-Validation folds that will be used in experiment. vectorizer_type : object, as default - CounterVectorizer (Scikit-Learn). Type of vectorizer that will be used to build feature vector. stars : list List of stars that will be mapped into sentiment. model : gensim.Doc2Vec Model that will convert list of documents into list of document's n_max_unsupervised : int How many document will be used during doc-2-vec approach for training as unsupervised examples. save_model : string, by default None - without saving Path where models doc-2-vec should be saved. d2v_size : int Length of the doc-2-vec vectors. """ train_test_path = path.join(base_path, 'train_test_subsets') datasets = glob.glob( path.join(base_path, '*{}*.csv'.format(dataset_filter))) log.info('Datasets will be used in experiment: {}'.format(datasets)) if not exists(output_folder): makedirs(output_folder) log.info('New directory has been created in: {}'.format(output_folder)) for dataset in datasets: dataset_name = path.basename(dataset).split('.')[0] log.info('Dataset name: {}'.format(dataset_name)) dp = DocumentPreprocessor() df = pd.DataFrame.from_csv(dataset, sep=';', index_col=False) # filter stars in reviews if stars is not None: df = df[df['review/score'].isin(stars)] # TODO remove comment below # df, _ = dp.star_score_to_sentiment(df, score_column='review/score') # extract only Document and Sentiment columns df['Document'] = df['review/text'] df['Sentiment'] = df['review/score'] df = df[['Sentiment', 'Document']] indexes_all = set(df.index) log.info('All indexes: {}'.format(len(indexes_all))) # try: # load train/test sets folds f_path = path.join( train_test_path, 'train-test-{}-{}.pkl'.format(n_reviews, dataset_name)) with open(f_path, 'rb') as f: train_test_indexes = pickle.load(f) log.info('Pickle has been loaded: {}'.format(f_path)) predictions = [] results = [] for i, cv in enumerate(train_test_indexes[:n_cv]): log.info('%s fold from division has been started!' % i) if stars is not None: cv = (list(set(cv[0]).intersection(df.index)), list(set(cv[1]).intersection(df.index))) log.info('Vectorizer type processed: {}'.format(vectorizer_type)) f_name = 'Supervised-learning-{}-folds-unsup-{}-stars-{}-d2v-size-{}'.format( vectorizer_type, n_max_unsupervised, list_to_str(stars), d2v_size) # s = Sentiment(dataset_name='{}-cv-{}'.format(dataset_name, i), s = Sentiment(dataset_name=dataset_name, save_model=save_model) log.info('Length train: {}\n Length test: {}'.format( len(cv[0]), len(cv[1]))) # df_ = df.ix[cv[0] + cv[1]] # W2V specific # unsup_docs = df.loc[~df.index.isin(df_.index)]['Document'][ # :n_max_unsupervised] # log.debug('Unsup_docs {}'.format(len(unsup_docs))) # # log.info( # 'Chosen dataframe subset is {} x {}'.format(df_.shape[0], # df_.shape[1])) model_path = path.join( save_model, '{}-doc-2-vec.model-{}.pkl'.format(f_name, dataset_name)) if exists(model_path): log.info('Doc-2-Vec will be loaded: {}'.format(model_path)) model = pd.read_pickle(model_path) docs = s.labelize_tokenize_docs(docs=df['Document'], label_type=s.w2v_label) X = s.get_doc_2_vec_vectors(model=model, corpus=docs) else: log.info('Doc-2-Vec will be trained!') X, model = s.build_doc2vec(df['Document'], [], model) to_pickle(save_model, dataset_name, '{}-doc-2-vec.model'.format(f_name), model, set_time=False) df_csv = pd.DataFrame() df_csv['class'] = df['Sentiment'] df_csv = pd.merge(df_csv, pd.DataFrame(X), left_index=True, right_index=True) log.debug( 'Data Frame with labels and features, shape: {}x{}'.format( df_csv.shape[0], df_csv.shape[1])) df_csv.to_csv(path.join(save_model, '{}-{}.csv'.format(dataset_name, d2v_size)), header=False, index=False)