def data():
# get current directory
path = os.getcwd()
# get one directory up
path = os.path.dirname(path)
WORD_VECTORS = "../embeddings/word2vec.txt"
WORD_VECTORS_DIMS = 300
TRAIN_DATA = path + "/datasets/ABSA16_Restaurants_Train_SB1_v2.xml"
VAL_DATA = path + "/datasets/EN_REST_SB1_TEST.xml"
max_length = 80
# load word embeddings
print("loading word embeddings...")
word2idx, idx2word, embeddings = load_word_vectors(WORD_VECTORS,
WORD_VECTORS_DIMS)
print("loading categories")
entity_attribute_pairs = createCategories2()
# load raw data
print("loading datasets...")
train_review, train_ent_attrib = \
read_xml2_train3(entity_attribute_pairs, TRAIN_DATA)
gold_review, gold_ent_attrib = \
read_xml2_train3(entity_attribute_pairs, VAL_DATA)
y_train = train_ent_attrib
y_test = gold_ent_attrib
print("Tokenizing...")
# nltk tokenizer
X_train = [
casual_tokenize(x,
preserve_case=False,
reduce_len=True,
strip_handles=False) for x in train_review
]
X_test = [
casual_tokenize(x,
preserve_case=False,
reduce_len=True,
strip_handles=False) for x in gold_review
]
print("Vectorizing...")
X_train = numpy.array(
[vectorize(x, word2idx, max_length) for x in X_train])
X_test = numpy.array([vectorize(x, word2idx, max_length) for x in X_test])
print("Turning test and train data to numpy arrays")
X_train = numpy.array(X_train)
y_train = numpy.array(y_train)
X_test = numpy.array(X_test)
y_test = numpy.array(y_test)
label_encoder = LabelBinarizer()
y_train_res = label_encoder.fit_transform(y_train)
y_test = label_encoder.fit_transform(y_test)
# Everything to numpy
X_train = numpy.array(X_train)
y_train = numpy.array(y_train_res)
y_test = numpy.array(y_test)
return embeddings, X_train, X_test, y_train, y_test, max_length
def create_aspect_emb():
''' This a method that creates a vector to initialize the aspect embeddings'''
# get current directory
path = os.getcwd()
# get one directory up
path = os.path.dirname(path)
WORD_VECTORS = "../embeddings/glove.6B.300d.txt"
WORD_VECTORS_DIMS = 300
print("loading word embeddings...")
word2idx, idx2word, embeddings = load_word_vectors(WORD_VECTORS,
WORD_VECTORS_DIMS)
entity_attribute_pairs = createCategories2()
TRAIN_DATA = path + "/datasets/ABSA16_Restaurants_Train_SB1_v2.xml"
d = get_words_for_each_category(entity_attribute_pairs, TRAIN_DATA)
for i in range(0, 12):
cat = d[i]
cat = tokenize(cat)
cat = set(cat)
sum_emb = np.zeros(shape=(1, 300))
word_count = 0
for word in cat:
if word in word2idx.keys():
sum_emb = sum_emb + embeddings[word2idx[word]]
word_count += 1
sum_emb = sum_emb / word_count
with open(path + "/embeddings/aspect_embeddings.txt", "a") as the_file:
for e in sum_emb:
for num in e:
the_file.write(str(num) + ' ')
the_file.write('\n')
sum_emb = np.array(sum_emb)
return sum_emb
}
# keep the same seed for random variables
np.random.seed(seed)
torch.manual_seed(seed)
# torch.backends.cudnn.enabled = False
if torch.cuda.is_available():
torch.cuda.manual_seed(seed)
########################################################
# PREPARE FOR DATA
########################################################
# load word embeddings
print("loading word embeddings...")
word2idx, idx2word, embeddings = load_word_vectors(WORD_VECTORS,
WORD_VECTORS_DIMS)
tword2idx, idx2tword, topic_embeddings = load_word_vectors(WORD_VECTORS,
WORD_VECTORS_DIMS)
train_set = SentimentDataset(file=TRAIN_DATA, word2idx=word2idx, tword2idx=tword2idx,
max_length=0, max_topic_length=0, topic_bs=True)
print("Batching...")
train_sampler, validation_sampler = split_train_set(train_set, contiguous=True, split_rate=0.1)
loader_train = DataLoader(train_set, batch_size=BATCH_SIZE, sampler=train_sampler,
shuffle=False, num_workers=4)
loader_val = DataLoader(train_set, batch_size=BATCH_SIZE, sampler=validation_sampler,
def data():
# get current directory
path = os.getcwd()
# get one directory up
path = os.path.dirname(path)
WORD_VECTORS = "../embeddings/glove.6B.300d.txt"
TRAIN_DATA = path + "/datasets/ABSA16_Restaurants_Train_SB1_v2.xml"
VAL_DATA = path + "/ABSA/datasets/EN_REST_SB1_TEST.xml"
BATCH_SIZE = 128
EPOCHS = 50
WORD_VECTORS_DIMS = 300
MAX_LENGTH = 80
max_length = 80
_hparams = {
"rnn_size": 100,
"bidirectional": True,
"noise": 0.2,
"dropout_words": 0.2,
"dropout_rnn": 0.5,
}
# load word embeddings
print("loading word embeddings...")
word2idx, idx2word, embeddings = load_word_vectors(WORD_VECTORS,
WORD_VECTORS_DIMS)
print("loading categories")
entity_attribute_pairs = createCategories2()
polarity_labels = create_polarities()
# load raw data
print("loading datasets...")
train_review, train_ent_attrib, train_polarity,train_aux = \
read_xml_polarities(entity_attribute_pairs,polarity_labels,TRAIN_DATA)
gold_review, gold_ent_attrib, gold_polarity,gold_aux = \
read_xml_polarities(entity_attribute_pairs,polarity_labels,VAL_DATA)
print("extracting sentiment from texts")
sentiment_intensity_train = extract_setiment_scores(train_review)
sentiment_intensity_test = extract_setiment_scores(gold_review)
y_train = train_polarity
y_test = gold_polarity
words = gold_review
print("Tokenizing...")
X_train = [tokenize(x) for x in train_review]
X_test = [tokenize(x) for x in gold_review]
print("Vectorizing...")
X_train = numpy.array(
[vectorize(x, word2idx, MAX_LENGTH) for x in X_train])
X_test = numpy.array([vectorize(x, word2idx, MAX_LENGTH) for x in X_test])
print("Turning test and train data to numpy arrays")
# train and test sentence
X_train = numpy.array(X_train)
X_test = numpy.array(X_test)
# train and test E#A labels
y_train = numpy.array(y_train)
y_test = numpy.array(y_test)
# auxilary input - the aspect that is present in the sentence
train_aux = numpy.array(train_aux)
gold_aux = numpy.array(gold_aux)
# handcrafted feature - the sentiment intensity of each sentence in a 3 way scale
sentiment_intensity_train = numpy.array(sentiment_intensity_train)
sentiment_intensity_test = numpy.array(sentiment_intensity_test)
# entity and attribute pairs are given in this task and we can use them as extra features in each sentence
train_ent_attrib = numpy.array(train_ent_attrib)
gold_ent_attrib = numpy.array(gold_ent_attrib)
classes = len(polarity_labels)
return embeddings, classes, max_length, X_train, train_aux, sentiment_intensity_train, y_train, X_test, gold_aux, sentiment_intensity_test, y_test
from preprocess import SentimentDataset
from tools import set_logger, train_validation_split
from load_embeddings import load_word_vectors
from my_neural import CNNClassifier
import matplotlib.pyplot as plt
import copy
torch.manual_seed(1)
BATCH_SIZE = 15
EPOCHS = 20
vec_size = 300
datasets = "laptop"
default_embed_path = "word_embeds/amazon%s.txt" % vec_size
default_train_path = "train_data/reviews_Electronics_5.json.gz"
word2idx, idx2word, embeddings = load_word_vectors(default_embed_path,
vec_size)
logging = set_logger("slot3.csv")
logging.debug(
"Epoch,Train acuracy, train f1, train loss, test accuracy, test f1, test Loss"
)
###############################################################################################
train_sentences, emotion_for_sentence, all_categories_train = [], [], []
quota = [0.4, 0.2, 0.4]
max_sentences = 50000
sentences = MySentences(default_train_path, quota, max_sentences)
train_sentences, emotion_for_sentence, all_categories_train = sentences.get_sentiment(
)
import pickle
with open('dict.pickle', 'rb') as handle:
unserialized_data = pickle.load(handle)