def run_all_domains(model,
base_path,
size,
save_model='/models/doc2vec/domains'):
log.info('Build vocabulary!')
model.build_vocab(get_amazon_docs(base_path))
model.train(get_amazon_docs(base_path))
to_pickle(save_model,
'all-domains',
'{}-doc-2-vec.model'.format(size),
model,
set_time=False)
def get_dataset_with_kfolds_indexes(base_path, output_folder, dataset_filter,
n_reviews=2000, n_cv=10,
vectorizer_type='CountVectorizer',
stars=None, model=None):
"""
Main function for getting data and all necessary setting to start up
supervised learning approach for sentiment analysis based on Amazon data
with predefined cross-validation folds.
Parameters
----------
base_path : string
Path to all folders and files needed in analysis, e.g, csv files with
amazon data.
output_folder : string
Path to the directory where all outcomes of the experiment will
be stored.
dataset_filter : string
Filter files nas for dataset that will be used in the experiment.
n_reviews : int, 2000 by default
Number of reviews from each dataset to use in analysis.
n_cv : int, 10 by default
Number of Cross-Validation folds that will be used in experiment.
vectorizer_type : object, as default - CounterVectorizer (Scikit-Learn).
Type of vectorizer that will be used to build feature vector.
stars : list
List of stars that will be mapped into sentiment.
model : gensim.Doc2Vec
Model that will convert list of documents into list of document's
vectors.
"""
datasets = glob.glob(
path.join(base_path, '*%s*.csv' % dataset_filter))
log.info('Datasets will be used in experiment: {}'.format(datasets))
if not exists(output_folder):
makedirs(output_folder)
log.info('New directory has been created in: {}'.format(output_folder))
for dataset in datasets:
dataset_name = path.basename(dataset).split('.')[0]
log.info('Dataset name: %s' % dataset_name)
dp = DocumentPreprocessor()
df = pd.DataFrame.from_csv(dataset, sep=';', index_col=False)
# filter stars in reviews
if stars is not None:
df = df[df['review/score'].isin(stars)]
df, _ = dp.star_score_to_sentiment(df, score_column='review/score')
# extract only Document and Sentiment columns
df['Document'] = df['review/text']
df = df[['Sentiment', 'Document']]
indexes_all = set(df.index)
log.info('All indexes: {}'.format(len(indexes_all)))
predictions = []
results = []
log.info('Vectorizer type processed: {}'.format(vectorizer_type))
f_name = 'Supervised-learning-{}-{}-{}'.format(
dataset_name, vectorizer_type, list_to_str(stars))
s = Sentiment(dataset_name=dataset_name)
classes, ml_prediction, results_ml = s.supervised_sentiment(
docs=df['Document'],
y=np.array(df['Sentiment']),
classifiers=ALL_CLASSIFIERS,
f_name_results=f_name,
vectorizer=vectorizer_type,
n_folds=n_cv,
model=model,
)
results.append(results_ml)
predictions.append(ml_prediction)
to_pickle(p=output_folder, dataset=dataset_name, f_name=f_name,
obj=results)
def sentiment_lexicons_imdb(lexs_names=None,
lex_path=None,
output_folder=None,
evaluate=True):
"""
Counting sentiment analysis tasks with lexicon for IMDB Dataset with
predefined Cross-Validation split.
Parameters
----------
lexs_names : list
List of path/file names for lexicons loading.
lex_path: str
Path to the directory with lexicon's files.
output_folder : str
Path where we want to save our results.
evaluate : bool, True by default
If true the metrics for analysis will be counted, otherwise only
prediction will be saved.
Returns
----------
Nothing, all necessary files will be saved automatically.
"""
dataset_name = 'IMDB'
results = []
predictions = []
predictions_directory = join(output_folder, 'predictions')
if not exists(output_folder):
makedirs(output_folder)
log.info('New directory has been created in: {}'.format(output_folder))
if not exists(predictions_directory):
makedirs(predictions_directory)
log.info('Directory for predictions has been created: {}'.format(
predictions_directory))
dataset = Dataset()
df = dataset.load_several_files()
log.info('Pre-processing phase starts!')
dp = DocumentPreprocessor()
df.Document = [dp.remove_numbers(doc) for doc in df.Document]
sent_lex = SentimentLexicons(stemmed=False, lexicons_path=lex_path)
lexicons = sent_lex.load_lexicons(lexicons_file_names=lexs_names)
s = Sentiment(n_jobs=len(lexs_names), output_results=output_folder)
df_lex, lexicon_prediction, lexicon_result, classes = \
s.lex_sent_batch(
df=df,
dataset_name=dataset_name,
lexicons=lexicons)
results.append(lexicon_result)
predictions.append(lexicon_prediction)
to_pickle(f_path=join(output_folder,
'{}-{}.pkl'.format(dataset_name, 'results')),
obj=results)
def sentiment_lexicons_amazon_cv(datasets_path='',
dataset_filter=None,
lexs_names=None,
n_reviews=2000,
train=False,
norm_freq=False,
lex_path=None,
f_name='',
n_cv=10,
stars=None,
frequentiment_lexicons_path='',
output_folder=None,
evaluate=True):
"""
Counting sentiment analysis tasks with lexicon for Amazon Dataset with
predefined Cross-Validation split.
Parameters
----------
frequentiment_lexicons_path : str
Path to frequentiment lexicons (csv files with comma as separator).
datasets_path: str
Path to the datasets directory, it must contain
folder/train_test_subsets with cross-validation information - data
frame indexes for each fold. Datasets are in csv files - converted from
Web Amazon dataset structure.
dataset_filter : list
List of substring for choosing datasets.
lexs_names : list
List of path/file names for lexicons loading.
n_reviews: int, 2000 by default.
number of reviews from each star score.
train : bool, False by default.
If True you are counting sentiment for train subsets,
otherwise counting sentiment for testing subsets.
norm_freq : tuple with floats
Tuple, i.e., (-1, 1) fot threshold cutting, lower than
first value of tuples will be negative, between -1 and 1 will be neutral
more than 1 will be positive.
lex_path: str
Path to the directory with lexicon's files.
f_name : str
Additional part of output files name (results, predictions).
n_cv : int, 10 by default
Number of Cross-Validation's folds to performed.
stars : list
Star scores that will be used in experiment, as default all.
output_folder : str
Path where we want to save our results.
evaluate : bool, True by default
If true the metrics for analysis will be counted, otherwise only
prediction will be saved.
Returns
----------
Nothing, all necessary files will be saved automatically.
"""
results = {}
predictions = {}
datasets = []
predictions_directory = join(output_folder, 'predictions')
if not exists(output_folder):
makedirs(output_folder)
log.info('New directory has been created in: {}'.format(output_folder))
if not exists(predictions_directory):
makedirs(predictions_directory)
log.info('Directory for predictions has been created: {}'.format(
predictions_directory))
# lexs_names = [x.split('.')[0] for x in lexs_files]
train_test_path = join(datasets_path, 'train_test_subsets')
# get all datasets and cv
if dataset_filter is not None:
for df in dataset_filter:
datasets.extend(glob(join(datasets_path, '{}.csv'.format(df))))
else:
datasets = glob(join(datasets_path, '*.txt.gz.csv'))
log.debug('Datasets to process: {}'.format(datasets))
if frequentiment_lexicons_path:
log.debug('Freq lexs path: {}'.format(
join(frequentiment_lexicons_path, '*.csv')))
freq_lexs = glob(join(frequentiment_lexicons_path, '*.csv'))
else:
freq_lexs = []
for dataset in datasets:
# load Amazon data
dataset_file_name = basename(dataset)
dataset_name = dataset_file_name.split('.')[0]
log.info('Dataset name: %s' % dataset_name)
dp = DocumentPreprocessor()
df = pd.DataFrame.from_csv(dataset, sep=';', index_col=False)
if stars is not None:
df = df[df['review/score'].isin(stars)]
df, _ = dp.star_score_to_sentiment(df, score_column='review/score')
# extract only Document and Sentiment columns
df['Document'] = df['review/text']
# df['Document'] = df['review/summary']
df = df[['Document', 'Sentiment']]
log.info('Pre-processing phase starts!')
df.Document = [dp.remove_numbers(doc) for doc in df.Document]
try:
# load train/test sets folds
f_path = join(train_test_path,
'train-test-%s-%s.pkl' % (n_reviews, dataset_name))
with open(f_path, 'rb') as f:
train_test_indexes = pickle.load(f)
log.info('Pickle has been loaded, %s' % f_path)
results[dataset_name] = []
predictions[dataset_name] = []
# iterate over all cross-validation subsets
for cv_idx, cv in enumerate(train_test_indexes[:n_cv]):
log.info('Start for {}: CV: {}/{} '.format(
dataset_name, cv_idx + 1, n_cv))
freq_lexs_ = [
basename(x) for x in freq_lexs
if '{}-{}'.format(dataset_name, cv_idx) in x
]
log.info(
'Dataset: {}, CV: {} => frequentiment lexicons: {}'.format(
dataset_name, cv_idx, freq_lexs_))
lexs_names.extend(freq_lexs_)
if stars is not None:
cv = (set(cv[0]).intersection(df.index.values),
set(cv[1]).intersection(df.index.values))
sent_lex = SentimentLexicons(stemmed=False,
lexicons_file_names=lexs_names,
lexicons_path=lex_path)
lexicons = sent_lex.load_lexicons(
lexicons_file_names=lexs_names, lexicons_path=lex_path)
if train:
ind = cv[0]
f_name = 'train-%s-fold-%s' % (dataset_name, cv_idx)
else:
ind = cv[1]
f_name = 'test-%s-fold-%s' % (dataset_name, cv_idx)
s = Sentiment()
df_lex, lexicon_prediction, lexicon_result, classes = \
s.lex_sent_batch(
df=df.ix[ind],
dataset_name=dataset_name,
lexicons=lexicons,
evaluate=evaluate)
results[dataset_name].append(lexicon_result)
predictions[dataset_name].append(lexicon_prediction)
to_pickle(p=output_folder,
dataset='',
f_name=f_name,
obj=lexicon_prediction)
# df_lex.to_excel(join(output_folder, 'predictions', 'predictions-%s.xls' % f_name))
df_lex.to_pickle(
join(output_folder, 'predictions',
'predictions-%s.pkl' % f_name))
# save predictions
# results_to_pickle(dataset=dataset_name,
# f_name='Part-predictions-%s' % f_name,
# obj=lexicon_prediction)
except IOError as err:
log.error('%s not loaded' % dataset_name)
raise IOError(str(err))
to_pickle(p=output_folder, dataset='', f_name='Results', obj=results)
to_pickle(p=output_folder,
dataset='',
f_name='Predictions',
obj=predictions)
def sentiment_doc2vec_amazon_cv(base_path,
output_folder,
dataset_filter,
n_cv=10,
vectorizer_type='gensim',
stars=None,
model=None,
n_max_unsupervised=None,
d2v_size=100,
save_model=None):
"""
Main function for getting data and all necessary setting to start up
supervised learning approach for sentiment analysis based on Amazon data
with predefined cross-validation folds.
Parameters
----------
base_path : string
Path to all folders and files needed in analysis, e.g, csv files with
amazon data.
output_folder : string
Path where to save results.
dataset_filter : string
Filter files nas for dataset that will be used in the experiment.
n_cv : int, 10 by default
Number of Cross-Validation folds that will be used in experiment.
vectorizer_type : object, as default - CounterVectorizer (Scikit-Learn).
Type of vectorizer that will be used to build feature vector.
stars : list
List of stars that will be mapped into sentiment.
model : gensim.Doc2Vec
Model that will convert list of documents into list of document's
n_max_unsupervised : int
How many document will be used during doc-2-vec approach for training as
unsupervised examples.
save_model : string, by default None - without saving
Path where models doc-2-vec should be saved.
d2v_size : int
Length of the doc-2-vec vectors.
"""
datasets = glob.glob(
path.join(base_path, '*{}*.csv'.format(dataset_filter)))
log.info('Datasets will be used in experiment: {}'.format(datasets))
if not exists(output_folder):
makedirs(output_folder)
log.info('New directory has been created in: {}'.format(output_folder))
for dataset in datasets:
dataset_name = path.basename(dataset).split('.')[0]
log.info('Dataset name: {}'.format(dataset_name))
df = pd.DataFrame.from_csv(dataset, sep=';', index_col=False)
# filter stars in reviews
if stars is not None:
df = df[df['review/score'].isin(stars)]
# TODO remove comment below
# df, _ = dp.star_score_to_sentiment(df, score_column='review/score')
# extract only Document and Sentiment columns
df['Document'] = df['review/text']
df['Sentiment'] = df['review/score']
df = df[['Sentiment', 'Document']]
indexes_all = set(df.index)
log.info('All indexes: {}'.format(len(indexes_all)))
predictions = []
results = []
log.info('Vectorizer type processed: {}'.format(vectorizer_type))
f_name = 'Supervised-learning-{}-folds-unsup-{}-stars-{}-d2v-size-{}'.format(
vectorizer_type, n_max_unsupervised, list_to_str(stars), d2v_size)
s = Sentiment(dataset_name=dataset_name, save_model=save_model)
model_path = path.join(
save_model, '{}-gensim.model-{}.pkl'.format(f_name, dataset_name))
if exists(model_path):
log.info('Gensim will be loaded: {}'.format(model_path))
model = pd.read_pickle(model_path)
docs = s.labelize_tokenize_docs(docs=df['Document'],
label_type=s.w2v_label)
X = s.get_doc_2_vec_vectors(model=model, corpus=docs)
else:
log.info('Gensim will be trained!')
docs = df['Document']
log.info('#docs: {}'.format(len(docs)))
X, model = s.build_gensim(docs, model=model)
to_pickle(save_model,
dataset_name,
'{}-gensim.model'.format(f_name),
model,
set_time=False)
df_csv = pd.DataFrame()
df_csv['class'] = df['Sentiment']
df_csv = pd.merge(df_csv,
pd.DataFrame(X),
left_index=True,
right_index=True)
log.debug('Data Frame with labels and features, shape: {}x{}'.format(
df_csv.shape[0], df_csv.shape[1]))
df_csv.to_csv(path.join(save_model,
'{}-{}.csv'.format(dataset_name, d2v_size)),
header=False,
index=False)
def amazon_supervised(base_path,
output_folder,
dataset_filter,
n_cv=10,
vectorizer_type='CountVectorizer',
stars=None,
stars_dist=None):
"""
Main function for getting data and all necessary setting to start up
supervised learning approach for sentiment analysis based on Amazon data
with predefined cross-validation folds.
Parameters
----------
base_path : string
Path to all folders and files needed in analysis, e.g, csv files with
amazon data.
output_folder : string
Path to the directory where all outcomes of the experiment will
be stored.
dataset_filter : string
Filter files nas for dataset that will be used in the experiment.
n_reviews : int, 2000 by default
Number of reviews from each dataset to use in analysis.
n_cv : int, 10 by default
Number of Cross-Validation folds that will be used in experiment.
vectorizer_type : object, as default - CounterVectorizer (Scikit-Learn).
Type of vectorizer that will be used to build feature vector.
stars : list
List of stars that will be mapped into sentiment.
"""
datasets = glob.glob(path.join(base_path, '*%s*.csv' % dataset_filter))
log.info('Datasets will be used in experiment: {}'.format(datasets))
if not exists(output_folder):
makedirs(output_folder)
log.info('New directory has been created in: {}'.format(output_folder))
for dataset in datasets:
dataset_name = path.basename(dataset).split('.')[0]
log.info('Dataset name: %s' % dataset_name)
dp = DocumentPreprocessor()
df = pd.DataFrame.from_csv(dataset, sep=';', index_col=False)
# filter stars in reviews
if stars is not None:
dp = DocumentPreprocessor()
if stars_dist is not None:
df = df.ix[dp.get_reviews(df, 'review/score', stars_dist)]
df = df[df['review/score'].isin(stars)]
df, _ = dp.star_score_to_sentiment(df, score_column='review/score')
# extract only Document and Sentiment columns
df['Document'] = df['review/text']
df = df[['Sentiment', 'Document']]
log.info('All indexes: {}'.format(len(set(df.index))))
try:
features_ngrams = {
'unigrams': (1, 1),
'n_grams_1_2': (1, 2),
# 'n_grams_1_3': (1, 3),
}
log.info('Feature ngrams: {}'.format(features_ngrams))
results = []
for n_gram_name, n_grams_range in features_ngrams.iteritems():
log.info('Ngram type processed: {}'.format(n_gram_name))
log.info(
'Vectorizer type processed: {}'.format(vectorizer_type))
f_name = 'Supervised-learning-{}-{}-{}-n_reviews-{}'.format(
vectorizer_type,
n_gram_name, '-'.join([str(s) for s in stars]),
min(stars_dist.values()))
s = Sentiment(dataset_name=dataset_name,
output_results=output_folder)
log.info('Chosen dataframe subset is %s x %s' % df.shape)
classes, ml_prediction, results_ml = s.supervised_sentiment(
docs=df['Document'],
y=np.array(df['Sentiment']),
n_gram_range=n_grams_range,
classifiers=ALL_CLASSIFIERS,
lowercase=True,
stop_words='english',
# max_df=1.0,
# min_df=0.0,
max_features=50000,
f_name_results=f_name,
vectorizer=vectorizer_type,
n_folds=n_cv,
)
results.append(results_ml)
except IOError as err:
log.error('%s not loaded' % dataset_name)
raise IOError(str(err))
to_pickle(f_path=join(output_folder, 'resutls-super-example.pkl'),
obj=results)
def sentiment_doc2vec_amazon_cv(base_path,
output_folder,
dataset_filter,
n_reviews=2000,
n_cv=10,
vectorizer_type='doc2vec',
stars=None,
model=None,
n_max_unsupervised=None,
d2v_size=100,
save_model=None):
"""
Main function for getting data and all necessary setting to start up
supervised learning approach for sentiment analysis based on Amazon data
with predefined cross-validation folds.
Parameters
----------
base_path : string
Path to all folders and files needed in analysis, e.g, csv files with
amazon data.
output_folder : string
Path where to save results.
dataset_filter : string
Filter files nas for dataset that will be used in the experiment.
n_reviews : int, 2000 by default
Number of reviews from each dataset to use in analysis.
n_cv : int, 10 by default
Number of Cross-Validation folds that will be used in experiment.
vectorizer_type : object, as default - CounterVectorizer (Scikit-Learn).
Type of vectorizer that will be used to build feature vector.
stars : list
List of stars that will be mapped into sentiment.
model : gensim.Doc2Vec
Model that will convert list of documents into list of document's
n_max_unsupervised : int
How many document will be used during doc-2-vec approach for training as
unsupervised examples.
save_model : string, by default None - without saving
Path where models doc-2-vec should be saved.
d2v_size : int
Length of the doc-2-vec vectors.
"""
train_test_path = path.join(base_path, 'train_test_subsets')
datasets = glob.glob(
path.join(base_path, '*{}*.csv'.format(dataset_filter)))
log.info('Datasets will be used in experiment: {}'.format(datasets))
if not exists(output_folder):
makedirs(output_folder)
log.info('New directory has been created in: {}'.format(output_folder))
for dataset in datasets:
dataset_name = path.basename(dataset).split('.')[0]
log.info('Dataset name: {}'.format(dataset_name))
dp = DocumentPreprocessor()
df = pd.DataFrame.from_csv(dataset, sep=';', index_col=False)
# filter stars in reviews
if stars is not None:
df = df[df['review/score'].isin(stars)]
# TODO remove comment below
# df, _ = dp.star_score_to_sentiment(df, score_column='review/score')
# extract only Document and Sentiment columns
df['Document'] = df['review/text']
df['Sentiment'] = df['review/score']
df = df[['Sentiment', 'Document']]
indexes_all = set(df.index)
log.info('All indexes: {}'.format(len(indexes_all)))
# try:
# load train/test sets folds
f_path = path.join(
train_test_path,
'train-test-{}-{}.pkl'.format(n_reviews, dataset_name))
with open(f_path, 'rb') as f:
train_test_indexes = pickle.load(f)
log.info('Pickle has been loaded: {}'.format(f_path))
predictions = []
results = []
for i, cv in enumerate(train_test_indexes[:n_cv]):
log.info('%s fold from division has been started!' % i)
if stars is not None:
cv = (list(set(cv[0]).intersection(df.index)),
list(set(cv[1]).intersection(df.index)))
log.info('Vectorizer type processed: {}'.format(vectorizer_type))
f_name = 'Supervised-learning-{}-folds-unsup-{}-stars-{}-d2v-size-{}'.format(
vectorizer_type, n_max_unsupervised, list_to_str(stars),
d2v_size)
# s = Sentiment(dataset_name='{}-cv-{}'.format(dataset_name, i),
s = Sentiment(dataset_name=dataset_name, save_model=save_model)
log.info('Length train: {}\n Length test: {}'.format(
len(cv[0]), len(cv[1])))
# df_ = df.ix[cv[0] + cv[1]]
# W2V specific
# unsup_docs = df.loc[~df.index.isin(df_.index)]['Document'][
# :n_max_unsupervised]
# log.debug('Unsup_docs {}'.format(len(unsup_docs)))
#
# log.info(
# 'Chosen dataframe subset is {} x {}'.format(df_.shape[0],
# df_.shape[1]))
model_path = path.join(
save_model,
'{}-doc-2-vec.model-{}.pkl'.format(f_name, dataset_name))
if exists(model_path):
log.info('Doc-2-Vec will be loaded: {}'.format(model_path))
model = pd.read_pickle(model_path)
docs = s.labelize_tokenize_docs(docs=df['Document'],
label_type=s.w2v_label)
X = s.get_doc_2_vec_vectors(model=model, corpus=docs)
else:
log.info('Doc-2-Vec will be trained!')
X, model = s.build_doc2vec(df['Document'], [], model)
to_pickle(save_model,
dataset_name,
'{}-doc-2-vec.model'.format(f_name),
model,
set_time=False)
df_csv = pd.DataFrame()
df_csv['class'] = df['Sentiment']
df_csv = pd.merge(df_csv,
pd.DataFrame(X),
left_index=True,
right_index=True)
log.debug(
'Data Frame with labels and features, shape: {}x{}'.format(
df_csv.shape[0], df_csv.shape[1]))
df_csv.to_csv(path.join(save_model,
'{}-{}.csv'.format(dataset_name,
d2v_size)),
header=False,
index=False)