예제 #1
0
def check_examples():
    dal = ReviewsDAL()
    r_stream = dal.load_words("train")
    for index in xrange(10):
        print "******* {} *******".format(index+1)
        r = next(r_stream)
        print r.sent.words
예제 #2
0
def test_load_review_words():
    dal = ReviewsDAL()
    r_stream = dal.load_reviews_words("unlabeled")

    for index in xrange(10):
        review = next(r_stream)
        print "*************** {} ***************".format(index + 1)
        print "sentiment: {}".format(review.sent.sentiment)
        print "words: {}".format(review.sent.words)

    dal.close()
예제 #3
0
    def words_stream(self):
        self._metas = []

        dal = ReviewsDAL()
        review_stream = dal.load_words(self._colname)
        for index, r in enumerate(review_stream):
            self._metas.append((r.id, r.sent.sentiment))
            yield r.sent.words

            if index % 300 == 0:
                print "{} examples loaded from mongodb[{}]".format(index + 1, self._colname)

        dal.close()
예제 #4
0
def read_save_mongodb(filename,labeled,colname,buffersize=300):
    r_stream = reviews_stream(filename,labeled)
    dal = ReviewsDAL()

    buffer = []
    for index,review in enumerate(r_stream):
        if index % buffersize == 0:
            dal.insert_many(colname,buffer)
            del buffer[:] # clear
            print "{} reviews saved into mongo[{}]".format(index,colname)

        buffer.append(review)

    dal.insert_many(colname,buffer)
    dal.close()

    print "----------- DONE -----------"
    print "totally {} reviews inserted into mongodb[{}]".format(index+1,colname)
예제 #5
0
def split_train_validation():
    """
    load samples from 'train' collection,
    draw some samples out, to use as validation set
    remove them from training set and insert those samples into 'validation' collection
    """
    random.seed(999)
    valid_ratio = 0.3

    dal = ReviewsDAL()
    train_ids = list(dal.load_ids("train"))
    total_train = len(train_ids)
    print "originally, there are {} reviews in train set".format(total_train)

    valid_ids = random.sample(train_ids,int(total_train * valid_ratio))
    print "randomly draw {} samples to use as validation".format(len(valid_ids))

    train_collect = dal._db['train']
    valid_collect = dal._db['validate']
    for index,valid_id in enumerate(valid_ids):
        # load from train collection
        cursor = train_collect.find({'_id':valid_id})
        review_dict = next(cursor)

        # insert into validation collection
        valid_collect.insert_one(review_dict)

        # remove from train collection
        result = train_collect.delete_one({'_id':valid_id})
        assert result.deleted_count == 1

        #
        if index % 100 == 0:
            print "{} reviews transferred from train to validation".format(index+1)
    print "*** totally {} reviews transferred from train to validation ***".format(index+1)

    print "now, train set has {} reviews".format(train_collect.count({}))
    print "now, validation set has {} reviews".format(valid_collect.count({}))
예제 #6
0
def test_load_ids():
    dal = ReviewsDAL()
    ids = dal.load_ids("train")