# -*- coding: utf-8 -*-
"""
Created on Fri Dec 02 15:25:22 2016
@author: changlue.she
"""
from __future__ import division
import numpy as np
from nltk.corpus import brown
from DeepLearning.BasicNeuron import baseNeuronLayer
from NLP.statisticLanguageModel import statisLM
#-----------------------------------------------------------------------------------------------------------------------------------------------
corps = brown.sents(categories=None)
corps = list(corps)
slm = statisLM(corps, 100)
#-----------------------------------------------------------------------------------------------------------------------------------------------
maxNgram = 5
sentWordhash = {}
for ngram in range(2, maxNgram):
for sent in slm.corps:
if len(sent) - ngram > 0:
sent = '#'.join(sent)
sent = '&' + sent + '*'
for idx in range(len(sent) - ngram + 1):
sentWordhash.setdefault(sent[idx:idx + ngram], 0)
sentWordhash[sent[idx:idx + ngram]] += 1
#-----------------------------------------------------------------------------------------------------------------------------------------------
nchar2code = {}
idx = 0
for nchar in sentWordhash:
if sentWordhash[nchar] > 10:
from __future__ import division from nltk.corpus import brown from DeepLearning.BasicNeuron import baseNeuronLayer from DeepLearning.CnnNeuron import word2vecCovLayer from NLP.statisticLanguageModel import statisLM import numpy as np #----------------------------------------------------------------------------------------------------------------------------------------------- corps = brown.sents(categories=None) corps = list(corps) slm = statisLM(corps, 50) #----------------------------------------------------------------------------------------------------------------------------------------------- window = 2 wordDim = 50 outDim = 50 outs = 1 hiddenFunc = 'tanh' outFunc = 'sigmoid' cnnlayer = word2vecCovLayer(window, wordDim, outDim, actfunc=hiddenFunc) outlayer = baseNeuronLayer(outDim, outs, actfunc=outFunc) #----------------------------------------------------------------------------------------------------------------------------------------------- '''if pickle from the save''' #import cPickle #dirs = "C:\\Users\\Administrator.NBJXUEJUN-LI\\Desktop\\project\\Python\\NLP\\savedObject\\CompCorpus\\" #slm = cPickle.load(open(dirs+"slm.pkl","rb")) #cnnlayerPara = cPickle.load(open(dirs+"cnnlayer.pkl","rb")) #outlayerPara = cPickle.load(open(dirs+"outlayer.pkl","rb")) #cnnlayer.W,cnnlayer.b = cnnlayerPara #outlayer.W,outlayer.b = outlayerPara #----------------------------------------------------------------------------------------------------------------------------------------------- ''' function