# -*- coding: utf-8 -*- """ Created on Fri Dec 02 15:25:22 2016 @author: changlue.she """ from __future__ import division import numpy as np from nltk.corpus import brown from DeepLearning.BasicNeuron import baseNeuronLayer from NLP.statisticLanguageModel import statisLM #----------------------------------------------------------------------------------------------------------------------------------------------- corps = brown.sents(categories=None) corps = list(corps) slm = statisLM(corps, 100) #----------------------------------------------------------------------------------------------------------------------------------------------- maxNgram = 5 sentWordhash = {} for ngram in range(2, maxNgram): for sent in slm.corps: if len(sent) - ngram > 0: sent = '#'.join(sent) sent = '&' + sent + '*' for idx in range(len(sent) - ngram + 1): sentWordhash.setdefault(sent[idx:idx + ngram], 0) sentWordhash[sent[idx:idx + ngram]] += 1 #----------------------------------------------------------------------------------------------------------------------------------------------- nchar2code = {} idx = 0 for nchar in sentWordhash: if sentWordhash[nchar] > 10:
from __future__ import division from nltk.corpus import brown from DeepLearning.BasicNeuron import baseNeuronLayer from DeepLearning.CnnNeuron import word2vecCovLayer from NLP.statisticLanguageModel import statisLM import numpy as np #----------------------------------------------------------------------------------------------------------------------------------------------- corps = brown.sents(categories=None) corps = list(corps) slm = statisLM(corps, 50) #----------------------------------------------------------------------------------------------------------------------------------------------- window = 2 wordDim = 50 outDim = 50 outs = 1 hiddenFunc = 'tanh' outFunc = 'sigmoid' cnnlayer = word2vecCovLayer(window, wordDim, outDim, actfunc=hiddenFunc) outlayer = baseNeuronLayer(outDim, outs, actfunc=outFunc) #----------------------------------------------------------------------------------------------------------------------------------------------- '''if pickle from the save''' #import cPickle #dirs = "C:\\Users\\Administrator.NBJXUEJUN-LI\\Desktop\\project\\Python\\NLP\\savedObject\\CompCorpus\\" #slm = cPickle.load(open(dirs+"slm.pkl","rb")) #cnnlayerPara = cPickle.load(open(dirs+"cnnlayer.pkl","rb")) #outlayerPara = cPickle.load(open(dirs+"outlayer.pkl","rb")) #cnnlayer.W,cnnlayer.b = cnnlayerPara #outlayer.W,outlayer.b = outlayerPara #----------------------------------------------------------------------------------------------------------------------------------------------- ''' function