def experiment2(experiment1, data,y,cnn_adr,reference_we,sent_pos,sent_neg,adr_lexicon): """ See Section 4.2 in ./final-report.pdf """ logger.info("\n\nLoading feature matrices for second experiment\n") tokenized_tweets = list(data['tok_text'].values) raw_text_tweet = list(data['raw_text'].values) base_feature = {"LSA" : experiment1.pop("LSA") , "CNN-ADR" : experiment1.pop("CNN-ADR")} X_neg = getNegFeatures(tokenized_tweets) X_sent = getSentFeatures(tokenized_tweets,sent_pos,sent_pos) X_adr = getADRlexFeatures(raw_text_tweet,adr_lexicon) X_sent_specific = concatenate([X_sent,X_neg]) X_tot_extra = concatenate([X_sent_specific,X_adr]) extra_feature = {"ADR-LEX" : X_adr, "SENT" : X_sent_specific,"TOT-EXTRA" : X_tot_extra } complete_feature = {} for name_base_X,base_X in base_feature.items(): for name_extra_X,extra_X in extra_feature.items(): complete_feature["{0}-{1}".format(name_base_X,name_extra_X)] = concatenate([base_X,extra_X]) return complete_feature
def apply(self, sentence, sentence_mask): state_below = self.table.apply(sentence) # make sure state_below: n_steps * batch_size * embedding if state_below.ndim == 2: n_steps = state_below.shape[0] embed = state_below.shape[1] state_below = state_below.reshape((n_steps, 1, embed)) hiddens_forward = self.forward.apply(state_below, sentence_mask) if sentence_mask is None: hiddens_backward = self.backward.apply(state_below[::-1]) else: hiddens_backward = self.backward.apply(state_below[::-1], sentence_mask[::-1]) training_c_components = [] training_c_components.append(hiddens_forward) training_c_components.append(hiddens_backward[::-1]) #annotaitons = T.concatenate(training_c_components, axis=2) annotaitons = concatenate(training_c_components, axis=training_c_components[0].ndim-1) return annotaitons
def match(self, atomsLists): """Function matches atoms from atomsLists to self.lattice.positions @return: match, referenceMatch, errors match is list of indices of places for atoms match[atomIndex] = placeIndex referenceMatch[atomIndex] = atomIndex on allAtomsList errors is a list of (max error, average error)""" errors = [] allAtoms = concatenate(atomsLists) forbiddenPosList = [] forbiddenAtomsList = [ ] #we'll need both of this to know which atom is banned posCount = len(atomsLists[0]) match = [0 for _ in range(posCount)] referenceMatch = [0 for _ in range(posCount)] self._initPosSearch(len(allAtoms)) for _ in range(posCount): atomIndex, placeIndex = self.matchPos(allAtoms, len(atomsLists), forbiddenPosList, forbiddenAtomsList) if not placeIndex is None: forbiddenPosList.append(placeIndex) forbiddenAtomsList += [ (atomIndex + n * len(atomsLists[0])) % len(allAtoms) for n in range(len(atomsLists)) ] if not placeIndex is None: pos = self.lattice.positions[placeIndex] errors.append(distance(pos.x0, allAtoms[atomIndex].x0)) if not placeIndex is None: match[atomIndex % len(atomsLists[0])] = placeIndex referenceMatch[atomIndex % len(atomsLists[0])] = atomIndex return match, referenceMatch, errors
def next(self): """ Returns lattice containing projection of the next frame of xyzFile. If file has finnished it returns None value """ frame = self.xyzFile.nextFrame() if frame is None: return None newFrame = XYZFrame() newFrame.boxVectors = self.lattice.boxVectors refFrame = XYZFrame() refFrame.boxVectors = self.lattice.boxVectors atomsLists = self.propagateAtomsThroughPbc(frame.atoms, frame.boxSize) allAtoms = concatenate(atomsLists) posCount = len(atomsLists[0]) match, referenceMatch, errors = self.match(atomsLists) for atomIndex in range(posCount): newFrame.atoms.append( XYZAtom(atomsLists[0][atomIndex].symbol, *self.lattice.positions[match[atomIndex]].x0)) for atomIndex in range(posCount): refFrame.atoms.append( XYZAtom(allAtoms[referenceMatch[atomIndex]].__repr__())) refFrame.atoms[-1].x += 15 for atomIndex in range(len(allAtoms)): refFrame.atoms.append(XYZAtom(allAtoms[atomIndex].__repr__())) refFrame.atoms[-1].x += 30 return ProjectedFrame(newFrame, refFrame, errors)
def next(self): """ Returns lattice containing projection of the next frame of xyzFile. If file has finnished it returns None value """ frame = self.xyzFile.nextFrame() if frame is None: return None newFrame = XYZFrame() newFrame.boxVectors = self.lattice.boxVectors refFrame = XYZFrame() refFrame.boxVectors = self.lattice.boxVectors atomsLists = self.propagateAtomsThroughPbc(frame.atoms, frame.boxSize) allAtoms = concatenate(atomsLists) posCount = len(atomsLists[0]) match, referenceMatch, errors = self.match(atomsLists) for atomIndex in range(posCount): newFrame.atoms.append(XYZAtom(atomsLists[0][atomIndex].symbol , *self.lattice.positions[match[atomIndex]].x0)) for atomIndex in range(posCount): refFrame.atoms.append(XYZAtom(allAtoms[referenceMatch[atomIndex]].__repr__())) refFrame.atoms[-1].x += 15 for atomIndex in range(len(allAtoms)): refFrame.atoms.append(XYZAtom(allAtoms[atomIndex].__repr__())) refFrame.atoms[-1].x += 30 return ProjectedFrame(newFrame, refFrame, errors)
def match(self, atomsLists): """Function matches atoms from atomsLists to self.lattice.positions @return: match, referenceMatch, errors match is list of indices of places for atoms match[atomIndex] = placeIndex referenceMatch[atomIndex] = atomIndex on allAtomsList errors is a list of (max error, average error)""" errors = [] allAtoms = concatenate(atomsLists) forbiddenPosList = [] forbiddenAtomsList = [] #we'll need both of this to know which atom is banned posCount = len(atomsLists[0]) match = [0 for _ in range(posCount)] referenceMatch = [0 for _ in range(posCount)] self._initPosSearch(len(allAtoms)) for _ in range(posCount): atomIndex, placeIndex = self.matchPos(allAtoms, len(atomsLists) , forbiddenPosList , forbiddenAtomsList) if not placeIndex is None: forbiddenPosList.append(placeIndex) forbiddenAtomsList += [(atomIndex + n * len(atomsLists[0])) % len(allAtoms) for n in range(len(atomsLists))] if not placeIndex is None: pos = self.lattice.positions[placeIndex] errors.append(distance(pos.x0, allAtoms[atomIndex].x0)) if not placeIndex is None: match[atomIndex % len(atomsLists[0])] = placeIndex referenceMatch[atomIndex % len(atomsLists[0])] = atomIndex return match, referenceMatch, errors
def __replace_href(self, tag): """Function for replace html ref to []""" if "<a href=" in str(tag.contents): if len(tag.contents) > 1: tag = concatenate(list(tag.contents)) else: tag = str(tag.contents)[1:len(str(tag.contents)) - 1] for k, v in self.rules.items(): tag = tag.replace(k, v) else: tag = tag.get_text() return tag
def getToken (self): """ Создать элементы из прикрепленных файлов. Отдельно картинки, отдельно все файлы """ attachesAll = [] attaches = Attachment (self.parser.page).attachmentFull attaches.sort (self.sortByLength, reverse=True) for attach in attaches: fname = os.path.basename (attach) if self.filterFile (fname): attach = Literal (fname) attachesAll.append (attach) finalToken = Literal (self.attachString) + concatenate (attachesAll) finalToken = finalToken.setParseAction (self.convertToLink)("attach") return finalToken
def build_sampler(self): # added by Longyue x_hist = T.ltensor3() x_hist_mask = T.tensor3() annotations_1 = self.encoder_hist_1.apply_1(x_hist, x_hist_mask) annotations_1 = annotations_1[-1] annotations_2 = self.encoder_hist_2.apply_2(annotations_1) annotations_3 = annotations_2[-1] x = T.lmatrix() # Build Networks # src_mask is None c = self.encoder.apply(x, None, annotations_3) #init_context = ctx[0, :, -self.n_hids_src:] # mean pooling init_context = c.mean(0) # added by Longyue init_context = concatenate([init_context, annotations_3], axis=annotations_3.ndim - 1) init_state = self.decoder.create_init_state(init_context) outs = [init_state, c, annotations_3] if not self.with_attention: outs.append(init_context) # compile function print 'Building compile_init_state_and_context function ...' self.compile_init_and_context = theano.function( [x, x_hist, x_hist_mask], outs, name='compile_init_and_context') print 'Done' y = T.lvector() cur_state = T.matrix() # if it is the first word, emb should be all zero, and it is indicated by -1 trg_emb = T.switch(y[:, None] < 0, T.alloc(0., 1, self.n_in_trg), self.table_trg.apply(y)) # added by Zhaopeng Tu, 2016-06-09 # for with_attention=False if self.with_attention and self.with_coverage: cov_before = T.tensor3() if self.coverage_type is 'linguistic': print 'Building compile_fertility ...' fertility = self.decoder._get_fertility(c) fertility = T.addbroadcast(fertility, 1) self.compile_fertility = theano.function( [c], [fertility], name='compile_fertility') print 'Done' else: fertility = None else: cov_before = None fertility = None # apply one step # modified by Zhaopeng Tu, 2016-04-29 # [next_state, ctxs] = self.decoder.apply(state_below=trg_emb, results = self.decoder.apply( state_below=trg_emb, init_state=cur_state, # added by Zhaopeng Tu, 2016-06-09 init_context=None if self.with_attention else init_context, c=c if self.with_attention else None, hist=annotations_3, # added by Longyue one_step=True, # added by Zhaopeng Tu, 2016-04-27 cov_before=cov_before, fertility=fertility) next_state = results[0] if self.with_attention: ctxs, alignment = results[1], results[2] if self.with_coverage: cov = results[3] else: # if with_attention=False, we always use init_context as the source representation ctxs = init_context readout = self.decoder.readout(next_state, ctxs, trg_emb) # maxout if self.maxout_part > 1: readout = self.decoder.one_step_maxout(readout) # apply dropout if self.dropout < 1.0: readout = Dropout(self.trng, readout, 0, self.dropout) # compute the softmax probability next_probs = self.logistic_layer.get_probs(readout) # sample from softmax distribution to get the sample next_sample = self.trng.multinomial(pvals=next_probs).argmax(1) # compile function print 'Building compile_next_state_and_probs function ...' inps = [y, cur_state] if self.with_attention: inps.append(c) else: inps.append(init_context) # added by Longyue inps.append(annotations_3) outs = [next_probs, next_state, next_sample] # added by Zhaopeng Tu, 2016-06-09 if self.with_attention: outs.append(alignment) # added by Zhaopeng Tu, 2016-04-29 if self.with_coverage: inps.append(cov_before) if self.coverage_type is 'linguistic': inps.append(fertility) outs.append(cov) self.compile_next_state_and_probs = theano.function( inps, outs, name='compile_next_state_and_probs') print 'Done' # added by Zhaopeng Tu, 2016-07-18 # for reconstruction if self.with_reconstruction: # Build Networks # trg_mask is None inverse_c = T.tensor3() # mean pooling inverse_init_context = inverse_c.mean(0) inverse_init_state = self.inverse_decoder.create_init_state( inverse_init_context) outs = [inverse_init_state] if not self.with_attention: outs.append(inverse_init_context) # compile function print 'Building compile_inverse_init_state_and_context function ...' self.compile_inverse_init_and_context = theano.function( [inverse_c], outs, name='compile_inverse_init_and_context') print 'Done' src = T.lvector() inverse_cur_state = T.matrix() trg_mask = T.matrix() # if it is the first word, emb should be all zero, and it is indicated by -1 src_emb = T.switch(src[:, None] < 0, T.alloc(0., 1, self.n_in_src), self.table_src.apply(src)) # apply one step # modified by Zhaopeng Tu, 2016-04-29 inverse_results = self.inverse_decoder.apply( state_below=src_emb, init_state=inverse_cur_state, # added by Zhaopeng Tu, 2016-06-09 init_context=None if self.with_attention else inverse_init_context, c=inverse_c if self.with_attention else None, c_mask=trg_mask, one_step=True) inverse_next_state = inverse_results[0] if self.with_attention: inverse_ctxs, inverse_alignment = inverse_results[ 1], inverse_results[2] else: # if with_attention=False, we always use init_context as the source representation inverse_ctxs = init_context inverse_readout = self.inverse_decoder.readout( inverse_next_state, inverse_ctxs, src_emb) # maxout if self.maxout_part > 1: inverse_readout = self.inverse_decoder.one_step_maxout( inverse_readout) # apply dropout if self.dropout < 1.0: inverse_readout = Dropout(self.srng, inverse_readout, 0, self.dropout) # compute the softmax probability inverse_next_probs = self.inverse_logistic_layer.get_probs( inverse_readout) # sample from softmax distribution to get the sample inverse_next_sample = self.srng.multinomial( pvals=inverse_next_probs).argmax(1) # compile function print 'Building compile_inverse_next_state_and_probs function ...' inps = [src, trg_mask, inverse_cur_state] if self.with_attention: inps.append(inverse_c) else: inps.append(inverse_init_context) outs = [ inverse_next_probs, inverse_next_state, inverse_next_sample ] # added by Zhaopeng Tu, 2016-06-09 if self.with_attention: outs.append(inverse_alignment) self.compile_inverse_next_state_and_probs = theano.function( inps, outs, name='compile_inverse_next_state_and_probs') print 'Done'
def build_trainer(self, src, src_mask, src_hist, src_hist_mask, trg, trg_mask, ite): # added by Longyue # checked by Zhaopeng: sentence dim = n_steps, hist_len, batch_size (4, 3, 25) # hist = (bath_size, sent_num, sent_len) --.T--> # hist = (sent_len, sent_num, bath_size) --lookup table--> # (sent_len, sent_num, bath_size, word_emb) --reshape--> # (sent_len, sent_num*bath_size, word_emb) --word-level rnn--> # (sent_len, sent_num*bath_size, hidden_size) --reshape--> # (sent_len, sent_num, bath_size, hidden_size) --[-1]--> # (sent_num, bath_size, hidden_size) --sent-level rnn--> # (sent_num, bath_size, hidden_size) --[-1]--> # (bath_size, hidden_size) = cross-sent context vector annotations_1 = self.encoder_hist_1.apply_1(src_hist, src_hist_mask) annotations_1 = annotations_1[-1] # get last hidden states annotations_2 = self.encoder_hist_2.apply_2(annotations_1) annotations_3 = annotations_2[-1] # get last hidden states #modified by Longyue annotations = self.encoder.apply(src, src_mask, annotations_3) # init_context = annotations[0, :, -self.n_hids_src:] # modification #1 # mean pooling init_context = (annotations * src_mask[:, :, None]).sum(0) / src_mask.sum(0)[:, None] #added by Longyue init_context = concatenate([init_context, annotations_3], axis=annotations_3.ndim - 1) trg_emb = self.table_trg.apply(trg) trg_emb_shifted = T.zeros_like(trg_emb) trg_emb_shifted = T.set_subtensor(trg_emb_shifted[1:], trg_emb[:-1]) # modified by Longyue hiddens, readout, alignment = self.decoder.run_pipeline( state_below=trg_emb_shifted, mask_below=trg_mask, init_context=init_context, c=annotations, c_mask=src_mask, hist=annotations_3) # apply dropout if self.dropout < 1.0: logger.info('Apply dropout with p = {}'.format(self.dropout)) readout = Dropout(self.trng, readout, 1, self.dropout) p_y_given_x = self.logistic_layer.get_probs(readout) self.cost = self.logistic_layer.cost(p_y_given_x, trg, trg_mask) / trg.shape[1] # self.cost = theano.printing.Print('likilihood cost:')(self.cost) # added by Zhaopeng Tu, 2016-07-12 # for reconstruction if self.with_reconstruction: # now hiddens is the annotations inverse_init_context = (hiddens * trg_mask[:, :, None] ).sum(0) / trg_mask.sum(0)[:, None] src_emb = self.table_src.apply(src) src_emb_shifted = T.zeros_like(src_emb) src_emb_shifted = T.set_subtensor(src_emb_shifted[1:], src_emb[:-1]) inverse_hiddens, inverse_readout, inverse_alignment = self.inverse_decoder.run_pipeline( state_below=src_emb_shifted, mask_below=src_mask, init_context=inverse_init_context, c=hiddens, c_mask=trg_mask) # apply dropout if self.dropout < 1.0: # logger.info('Apply dropout with p = {}'.format(self.dropout)) inverse_readout = Dropout(self.srng, inverse_readout, 1, self.dropout) p_x_given_y = self.inverse_logistic_layer.get_probs( inverse_readout) self.reconstruction_cost = self.inverse_logistic_layer.cost( p_x_given_y, src, src_mask) / src.shape[1] # self.reconstruction_cost = theano.printing.Print('reconstructed cost:')(self.reconstruction_cost) self.cost += self.reconstruction_cost * self.reconstruction_weight self.L1 = sum(T.sum(abs(param)) for param in self.params) self.L2 = sum(T.sum(param**2) for param in self.params) params_regular = self.L1 * 1e-6 + self.L2 * 1e-6 # params_regular = theano.printing.Print('params_regular:')(params_regular) # train cost train_cost = self.cost + params_regular # gradients grads = T.grad(train_cost, self.params) # apply gradient clipping here grads = grad_clip(grads, self.clip_c) # updates updates = adadelta(self.params, grads) # train function # modified by Longyue inps = [src, src_mask, src_hist, src_hist_mask, trg, trg_mask] self.train_fn = theano.function(inps, [train_cost], updates=updates, name='train_function')
def build_sampler(tparams, options, trng, use_noise): x = tensor.matrix('x', dtype='int64') xr = x[::-1] n_timesteps = x.shape[0] n_samples = x.shape[1] # word embedding (source), forward and backward emb = tparams['Wemb'][x.flatten()] emb = emb.reshape([n_timesteps, n_samples, options['dim_word']]) embr = tparams['Wemb'][xr.flatten()] embr = embr.reshape([n_timesteps, n_samples, options['dim_word']]) # encoder proj = get_layer(options['encoder'])[1](tparams, emb, options, prefix='encoder') projr = get_layer(options['encoder'])[1](tparams, embr, options, prefix='encoder_r') # concatenate forward and backward rnn hidden states ctx = concatenate([proj[0], projr[0][::-1]], axis=proj[0].ndim - 1) # get the input for decoder rnn initializer mlp ctx_mean = ctx.mean(0) # ctx_mean = concatenate([proj[0][-1],projr[0][-1]], axis=proj[0].ndim-2) init_state = get_layer('ff')[1](tparams, ctx_mean, options, prefix='ff_state', activ='tanh') print 'Building f_init...', outs = [init_state, ctx] f_init = theano.function([x], outs, name='f_init', profile=profile) print 'Done' # x: 1 x 1 y = tensor.vector('y_sampler', dtype='int64') init_state = tensor.matrix('init_state', dtype='float32') # if it's the first word, emb should be all zero and it is indicated by -1 emb = tensor.switch(y[:, None] < 0, tensor.alloc(0., 1, tparams['Wemb_dec'].shape[1]), tparams['Wemb_dec'][y]) # apply one step of conditional gru with attention proj = get_layer(options['decoder'])[1](tparams, emb, options, prefix='decoder', mask=None, context=ctx, one_step=True, init_state=init_state) # get the next hidden state next_state = proj[0] # get the weighted averages of context for this target word y ctxs = proj[1] logit_lstm = get_layer('ff')[1](tparams, next_state, options, prefix='ff_logit_lstm', activ='linear') logit_prev = get_layer('ff')[1](tparams, emb, options, prefix='ff_logit_prev', activ='linear') logit_ctx = get_layer('ff')[1](tparams, ctxs, options, prefix='ff_logit_ctx', activ='linear') logit = tensor.tanh(logit_lstm + logit_prev + logit_ctx) if options['use_dropout']: logit = dropout_layer(logit, use_noise, trng) logit = get_layer('ff')[1](tparams, logit, options, prefix='ff_logit', activ='linear') # compute the softmax probability next_probs = tensor.nnet.softmax(logit) # sample from softmax distribution to get the sample next_sample = trng.multinomial(pvals=next_probs).argmax(1) # compile a function to do the whole thing above, next word probability, # sampled word for the next target, next hidden state to be used print 'Building f_next..', inps = [y, ctx, init_state] outs = [next_probs, next_sample, next_state] f_next = theano.function(inps, outs, name='f_next', profile=profile) print 'Done' return f_init, f_next
def cross(a,b): """ Return list containing all combinations of elements of a and b.""" return concatenate([[(ai,bi) for bi in b] for ai in a])
def build_sampler(tparams, options, trng, use_noise): if options['input_token_level'] == 'character': x = tensor.tensor3('x', dtype='int64') x_mask = tensor.tensor3('x_mask', dtype='float32') n_characters = x.shape[0] n_words = x.shape[1] n_samples = x.shape[2] # extract character embeddings cemb = tparams['Cemb'][x.flatten()] cemb = cemb.reshape( [n_characters, n_words, n_samples, options['dim_char']]) # compute hidden states of character embeddings cproj = get_layer('conv')[1](tparams, cemb, options, options['char_emb_filters'], prefix='char_emb_conv', mask=x_mask) word_rep = tensor.max(cproj, axis=0) word_repr = word_rep[::-1] w_mask = x_mask.sum(0) w_mask = tensor.clip(w_mask, 0.0, 1.0) wr_mask = w_mask[::-1] else: x = tensor.matrix('x', dtype='int64') x_mask = tensor.matrix('x_mask', dtype='float32') n_words = x.shape[0] n_samples = x.shape[1] # for the backward rnn, we just need to invert x and x_mask xr = x[::-1] # reverse words xr_mask = x_mask[::-1] # extract word embeddings wemb = tparams['Wemb'][x.flatten()] wemb = wemb.reshape([n_words, n_samples, options['dim_word']]) wembr = tparams['Wemb'][xr.flatten()] wembr = wembr.reshape([n_words, n_samples, options['dim_word']]) w_mask, wr_mask = x_mask, xr_mask word_rep, word_repr = wemb, wembr encoder_vars = [x, x_mask] if options['enc_dir'] == 'forward' or \ options['enc_dir'] == 'bidir': # hidden states for new word embeddings for the forward rnn src_proj = get_layer(options['encoder'])[1](tparams, word_rep, options['encoder_dim'], options, prefix='word_encoder', mask=w_mask) if options['enc_dir'] == 'backward' or \ options['enc_dir'] == 'bidir': # hidden states for new word embeddings for the backward rnn src_projr = get_layer(options['encoder'])[1](tparams, word_repr, options['encoder_dim'], options, prefix='word_encoder_r', mask=wr_mask) if options['enc_dir'] == 'forward': ctx = src_proj[-1][0] ctx_mean = [state[0][-1] for state in src_proj] elif options['enc_dir'] == 'backward': ctx = src_projr[-1][0] ctx_mean = [state[0][-1] for state in src_projr] elif options['enc_dir'] == 'bidir': # concatenate forward and backward rnn hidden states ctx = concatenate([src_proj[-1][0], src_projr[-1][0][::-1]], axis=src_proj[-1][0].ndim - 1) ctx_mean = [ concatenate([fw_state[0][-1], bw_state[0][-1]], axis=fw_state[0].ndim - 2) for fw_state, bw_state in zip(src_proj, src_projr) ] elif options['enc_dir'] == 'none': ctx_mean = [(word_rep * x_mask[:, :, None]).sum(axis=0)] ctx_mean = [ctx_mean[0] / x_mask.sum(axis=0)[:, None]] # if avg ctx = word_rep assert ctx_mean[0].ndim == 2 assert len(ctx_mean) == len(options['encoder_dim']) init_state = get_layer('ff')[1](tparams, ctx_mean, options, prefix='ff_label_state') assert len(ctx_mean) == len(init_state) init_state = tensor.stack(init_state) # init_state: # layers x # samples x trg_hid_dim LOGGER.info('Building f_init') encoder_outs = [init_state, ctx] f_init = theano.function(encoder_vars, encoder_outs, name='f_init', profile=False) f_inits = [f_init] # y: 1 x 1 y = tensor.vector('y_sampler', dtype='int64') init_state = tensor.tensor3('init_state', dtype='float32') label_emb = tparams['Lemb'][y] # if it's the first word, emb should be all zero and it is indicated by -1 label_emb = label_emb * (y[:, None] >= 0) init_state_ = \ [ init_state[l] for l in xrange(len(options['decoder_dim'])) ] if options['enc_dir'] != 'none': # apply one step of conditional gru with attention label_proj = get_layer(options['decoder'])[1](tparams, label_emb, options['decoder_dim'], options, prefix='label_decoder', mask=None, context=ctx, context_mask=w_mask, one_step=True, init_state=init_state_) # next_label_state: # layers x # samples x rnn hid dim next_label_state, ctxs, dec_alphas = label_proj proj_h = next_label_state[-1] else: assert options['decoder'] == 'gru' assert len(init_state_) == 1 label_proj = get_layer(options['decoder'])[1]( tparams, label_emb, options['decoder_dim'], options, prefix='label_decoder', mask=None, one_step=True, init_state=init_state_[0]) assert type(label_proj) == list and len(label_proj) == 1 next_label_state = tensor.stack([label_proj[0][0]]) proj_h = label_proj[-1][0] assert proj_h.ndim == label_emb.ndim, \ 'expected {}: actual {}'.format(label_emb.ndim, proj_h.ndim) # XXX note that all of the context vectors are same in this case. ctxs = (ctx * x_mask[:, :, None]).sum(axis=0) / \ x_mask.sum(axis=0)[:, None] # XXX no alignment matrix dec_alphas = tensor.zeros((y.shape[0], x_mask.shape[0])) label_logit_lstm = get_layer('ff')[1](tparams, proj_h, options, prefix='ff_label_logit_lstm', activ=None) # characters in a label at t-1 to word at t label_logit_prev = get_layer('ff')[1](tparams, label_emb, options, prefix='ff_label_logit_prev', activ=None) label_logit_ctx = get_layer('ff')[1](tparams, ctxs, options, prefix='ff_label_logit_ctx', activ=None) label_logit = tensor.tanh(label_logit_lstm + label_logit_prev + label_logit_ctx) label_logit = get_layer('ff')[1](tparams, label_logit, options, prefix='ff_label_logit', activ=None) # compute the softmax probability next_label_probs = tensor.nnet.softmax(label_logit) # sample from softmax distribution to get the sample next_label_sample = trng.multinomial(pvals=next_label_probs).argmax(1) # compile a function to do the whole thing above, next label probability, # sampled label for the next target, next hidden state to be used LOGGER.info('Building f_label_next') f_lsamp_inps = [x_mask, y, ctx, init_state] f_lsamp_outs = [ next_label_probs, next_label_sample, next_label_state, dec_alphas ] f_next = theano.function(f_lsamp_inps, f_lsamp_outs, name='f_next', profile=False) f_nexts = [f_next] return f_inits, f_nexts
def build_model(tparams, options): """ Build a computational graph for model training Parameters: ------- tparams : dict Model parameters options : dict Model configurations Returns: ------- trng : Randomstream in Theano use_noise : TheanoSharedVariable encoder_vars : list This return value contains TheanoVariables used to construct part of the computational graph, especially used in the `encoder`. decoder_vars : list This return value contains TheanoVariables used to construct part of the computational graph, especially used in the `decoder`. opt_ret : dict costs : list A list of costs at word-level or both word-level and character-level costs """ opt_ret = dict() trng = MRG_RandomStreams(1234) use_noise = theano.shared(numpy.float32(0.)) word_dp = options['word_drop_prob'] word_dropout = {'use_noise': use_noise, 'dp': word_dp, 'trng': trng} if options['input_token_level'] == 'character': # description string: #characters x #words x #samples x = tensor.tensor3('x', dtype='int64') x_mask = tensor.tensor3('x_mask', dtype='float32') n_characters = x.shape[0] n_words = x.shape[1] n_samples = x.shape[2] # extract character embeddings cemb = tparams['Cemb'][x.flatten()] cemb = cemb.reshape( [n_characters, n_words, n_samples, options['dim_char']]) # compute hidden states of character embeddings in the source language cproj = get_layer('conv')[1](tparams, cemb, options, options['char_emb_filters'], prefix='char_emb_conv', mask=x_mask) # XXX max over time: # n_chars x # n_words x # n_samples x dim word_rep = tensor.max(cproj, axis=0) word_repr = word_rep[::-1] assert word_rep.ndim == 3 # NOTE w_mask should be a matrix of elements 1 or 0 w_mask = x_mask.sum(0) w_mask = tensor.clip(w_mask, 0.0, 1.0) wr_mask = w_mask[::-1] else: # description string: #words x #samples x = tensor.matrix('x', dtype='int64') x_mask = tensor.matrix('x_mask', dtype='float32') n_words = x.shape[0] n_samples = x.shape[1] # for the backward rnn, we just need to invert x and x_mask xr = x[::-1] # reverse words xr_mask = x_mask[::-1] # extract word embeddings wemb = tparams['Wemb'][x.flatten()] wemb = wemb.reshape([n_words, n_samples, options['dim_word']]) wembr = tparams['Wemb'][xr.flatten()] wembr = wembr.reshape([n_words, n_samples, options['dim_word']]) w_mask, wr_mask = x_mask, xr_mask word_rep, word_repr = wemb, wembr if options.get('fixed_embeddings', False): LOGGER.info('NOTE: word embeddings are NOT going to be updated!') word_rep = theano.gradient.zero_grad(word_rep) word_repr = theano.gradient.zero_grad(word_repr) y = tensor.matrix('y', dtype='int64') y_mask = tensor.matrix('y_mask', dtype='float32') encoder_vars = [x, x_mask] # collection of varaibles used in the encoder decoder_vars = [y, y_mask] # used in the decoder n_labels = y.shape[0] if options['enc_dir'] == 'forward' or \ options['enc_dir'] == 'bidir': # hidden states for new word embeddings for the forward rnn src_proj = get_layer(options['encoder'])[1](tparams, word_rep, options['encoder_dim'], options, prefix='word_encoder', mask=w_mask, **word_dropout) if options['enc_dir'] == 'backward' or \ options['enc_dir'] == 'bidir': # hidden states for new word embeddings for the backward rnn src_projr = get_layer(options['encoder'])[1](tparams, word_repr, options['encoder_dim'], options, prefix='word_encoder_r', mask=wr_mask, **word_dropout) if options['enc_dir'] == 'forward': ctx = src_proj[-1][0] ctx_mean = [state[0][-1] for state in src_proj] elif options['enc_dir'] == 'backward': ctx = src_projr[-1][0] ctx_mean = [state[0][-1] for state in src_projr] elif options['enc_dir'] == 'bidir': # context will be the concatenation of forward and backward rnns ctx = concatenate([src_proj[-1][0], src_projr[-1][0][::-1]], axis=src_proj[-1][0].ndim - 1) ctx_mean = [ concatenate([fw_state[0][-1], bw_state[0][-1]], axis=fw_state[0].ndim - 2) for fw_state, bw_state in zip(src_proj, src_projr) ] elif options['enc_dir'] == 'none': # since there is no source encoder, the following list contains only # a single element, which coressponds to a matrix of N x D where N is # the number of instances in a batch and D is the dimensionality of # instance vectors. ctx_mean = [(word_rep * x_mask[:, :, None]).sum(axis=0)] ctx_mean = [ctx_mean[0] / x_mask.sum(axis=0)[:, None]] # if avg assert ctx_mean[0].ndim == 2 assert type(ctx_mean) is list if options['enc_dir'] != 'none': assert len(ctx_mean) == len(options['encoder_dim']) # word embedding (target) label_emb = tparams['Lemb'][y.flatten()] label_emb = label_emb.reshape([n_labels, n_samples, options['dim_label']]) # We will shift the target sequence one time step # to the right. This is done because of the bi-gram connections in the # readout and decoder rnn. The first target will be all zeros and we will # not condition on the last output. label_emb_shifted = tensor.zeros_like(label_emb) label_emb_shifted = tensor.set_subtensor(label_emb_shifted[1:], label_emb[:-1]) label_emb = label_emb_shifted # initial decoder state init_state = get_layer('ff')[1](tparams, ctx_mean, options, prefix='ff_label_state') assert len(ctx_mean) == len(init_state) if options['enc_dir'] != 'none': # decoder - pass through the decoder conditional gru with attention label_proj = get_layer(options['decoder'])[1](tparams, label_emb, options['decoder_dim'], options, prefix='label_decoder', mask=y_mask, context=ctx, context_mask=w_mask, one_step=False, init_state=init_state, **word_dropout) # proj_h: hidden states of the decoder gru # ctxs: weighted averages of context, generated by attention module # dec_alphas: weights (alignment matrix) # num trg words x batch size x num src words proj_h_all, ctxs, opt_ret['dec_alphas'] = label_proj proj_h = proj_h_all[-1] # pick activations of the last hidden layer else: assert options['decoder'] == 'gru' label_proj = get_layer(options['decoder'])[1](tparams, label_emb, options['decoder_dim'], options, prefix='label_decoder', mask=y_mask, one_step=False, init_state=init_state, **word_dropout) proj_h = label_proj[-1][0] # XXX note that all of the context vectors are same in this case. ctxs = tensor.tile(ctx_mean[0], (proj_h.shape[0], 1, 1)) # XXX no alignment matrix opt_ret['dec_alphas'] = tensor.zeros( (y.shape[0], y.shape[1], x.shape[0])) # compute word probabilities # hidden at t to word at t label_logit_lstm = get_layer('ff')[1](tparams, proj_h, options, prefix='ff_label_logit_lstm', activ=None) # combined representation of label at t-1 to label at t label_logit_prev = get_layer('ff')[1](tparams, label_emb, options, prefix='ff_label_logit_prev', activ=None) # context at t to label at t label_logit_ctx = get_layer('ff')[1](tparams, ctxs, options, prefix='ff_label_logit_ctx', activ=None) label_logit = tensor.tanh(label_logit_lstm + label_logit_prev + label_logit_ctx) if options['use_dropout']: label_logit = dropout_layer(label_logit, use_noise, word_dp, trng) label_logit = get_layer('ff')[1](tparams, label_logit, options, prefix='ff_label_logit', activ=None) label_logit_shp = label_logit.shape label_logit = label_logit.reshape( [label_logit_shp[0] * label_logit_shp[1], label_logit_shp[2]]) label_probs = tensor.nnet.softmax(label_logit) # compute cost for labels y_flat = y.flatten() y_flat_idx = tensor.arange(y_flat.shape[0]) * \ options['n_labels'] + y_flat label_cost = -tensor.log(label_probs.flatten()[y_flat_idx]) label_cost = label_cost.reshape([y.shape[0], y.shape[1]]) label_cost = (label_cost * y_mask).sum(0) label_cost.name = 'label_cost' costs = [label_cost] return trng, use_noise, encoder_vars, decoder_vars, opt_ret, costs
def evaluate(self, sess, take): digest = Bunch(encoded=None, reconstructed=None, source=None, loss=.0, eval_loss=.0, dumb_loss=.0) blurred = inp.apply_gaussian(self.test_set, self._get_blur_sigma()) # Encode for i, batch in enumerate(self._batch_generator(blurred, shuffle=False)): encoding = self.encode.eval(feed_dict={self.input: batch[0]}) digest.encoded = ut.concatenate(digest.encoded, encoding) # Save encoding for visualization encoded_no_nan = np.nan_to_num(digest.encoded) self.embedding_assign.eval( feed_dict={self.embedding_test_ph: encoded_no_nan}) try: self.embedding_saver.save(sess, self.get_checkpoint_path() + EMB_SUFFIX) except: ut.print_info("Unexpected error: %s" % str(sys.exc_info()[0]), color=33) # Calculate expected evaluation expected = digest.encoded[1:-1] * 2 - digest.encoded[:-2] average = 0.5 * (digest.encoded[1:-1] + digest.encoded[:-2]) digest.size = len(expected) # evaluation summaries self.summary_writer.add_summary(self.eval_summs.eval( feed_dict={self.blur_ph: self._get_blur_sigma()}), global_step=self.get_past_epochs()) # evaluation losses for p in self._batch_permutation_generator(digest.size, shuffle=False): digest.loss += self.eval_loss.eval( feed_dict={ self.encoding: digest.encoded[p + 2], self.target: blurred[p + 2] }) digest.eval_loss += self.eval_loss.eval(feed_dict={ self.encoding: expected[p], self.target: blurred[p + 2] }) digest.dumb_loss += self.loss_ae.eval(feed_dict={ self.input: blurred[p], self.target: blurred[p + 2] }) # for batch in self._batch_generator(blurred, batches=1): # digest.source = batch[1][:take] # digest.reconstructed = self.decode.eval(feed_dict={self.input: batch[0]})[:take] # Reconstruction visualizations for p in self._batch_permutation_generator(digest.size, shuffle=True, batches=1): self.visualization_batch_perm = self.visualization_batch_perm if self.visualization_batch_perm is not None else p p = self.visualization_batch_perm digest.source = self.eval_decode.eval( feed_dict={self.encoding: expected[p]})[:take] digest.source = blurred[(p + 2)[:take]] digest.reconstructed = self.eval_decode.eval( feed_dict={self.encoding: average[p]})[:take] self._eval_image_summaries(blurred[p], digest.encoded[p], average[p], expected[p]) digest.dumb_loss = guard_nan(digest.dumb_loss) digest.eval_loss = guard_nan(digest.eval_loss) digest.loss = guard_nan(digest.loss) return digest
def download_video(cls, page_url, title, dest='video'): """从视频页面获取到视频链接进行下载""" res = get_page(page_url, **{'verify': False}) # 从网页源码获取视频链接 origin_txt = re.findall( r'<script>window.__playinfo__=(\{.*?\})</script>', res.text, re.S)[0] origin_json = json.loads(origin_txt, encoding='utf-8') urls = origin_json['durl'] # 下载视频 size = 0 chunk = 1024 content_size = sum([i['size'] for i in urls]) print('文件大小: %0.2fMB' % (content_size / chunk / 1024)) # 创建存放视频临时文件夹 create_folder(os.path.join(BASE_DIR, title)) start = time.time() # 循环下载视频 for i, data in enumerate(urls): url = data['url'] header = { 'Origin': 'https://www.bilibili.com', 'Referer': page_url, } try: # 请求视频链接 response = get_page(url, header=header, **{ 'verify': False, 'stream': True }) video_path = title + '/' + '{}.mp4'.format(i) # 下载视频 with open(video_path, 'wb') as file: for item in response.iter_content(chunk): file.write(item) file.flush() size += len(item) print('\r' + '[下载进度]:%s %0.2f%%' % ('>' * int(size * 50 / content_size), float(size / content_size) * 100), end='') except Exception as e: print(e) stop = time.time() print('\n' + '视频下载完成,耗时%.2f秒' % (stop - start)) # 合并视频 dest = os.path.join(BASE_DIR, dest) video_save_path = concatenate(title=title, dest=dest) video = {'title': title, 'url': page_url, 'path': video_save_path} return video
def next(self): """ Returns lattice containing projection of the next frame of xyzFile. If file has finnished it returns None value """ frame = self.xyzFile.nextFrame() if frame is None: return None newFrame = XYZFrame() newFrame.boxVectors = self.lattice.boxVectors refFrame = XYZFrame() refFrame.boxVectors = self.lattice.boxVectors atomsLists = self.propagateAtomsThroughPbc(frame.atoms, frame.boxSize) allAtoms = concatenate(atomsLists) #posCount = len(atomsLists[0]) errors = [] differences = [] #match, referenceMatch, errors = self.match(atomsLists) atoms = [] # print self._radius for pos in self.lattice.positions: ACounter = 0 BCounter = 0 for atom in allAtoms: dist = distance(atom.x0, pos.x0) atoms.append(atom) if atom.symbol == "DLPC": try: ACounter += 1. / (dist**2) except: ACounter = 1e308 else: if atom.symbol == "DOPC": try: BCounter += 1. / (dist**2) except: BCounter = 1e308 differences.append((ACounter - BCounter, pos.x0)) # if ACounter > BCounter: symbol = "DSPC" # else: symbol = "A" # newFrame.atoms.append(XYZAtom(symbol, *pos.x0)) errors.append(0.0) num = 0 for atom in atoms: refFrame.atoms.append(XYZAtom(atom.symbol, *atom.x0)) refFrame.atoms[-1].x += 17 for atom in allAtoms: refFrame.atoms.append(XYZAtom(atom.symbol, *atom.x0)) refFrame.atoms[-1].x += 48 for diff, pos in sorted(differences, key=lambda tup: tup[0]): num += 1 if num < 129: symbol = "DOPC" else: symbol = "DPPC" newFrame.atoms.append(XYZAtom(symbol, *pos)) #for atomIndex in range(posCount): # newFrame.atoms.append(XYZAtom(atomsLists[0][atomIndex].symbol # , *self.lattice.positions[match[atomIndex]].x0)) return ProjectedFrame(newFrame, refFrame, errors)
def sample(self, material: int, quantity: int = 1) -> Tuple[Tensor, Tuple[Tensor, SVBRDF]]: ''' Returns a Dataset tuple derived from the given texture with the specified batch size. Args: material: Index of the texture to sample from this Dataset. quantity: Number of samples to include in the returned batch. Returns: Tensor [B, 3, R, C] of cropped front-parallel "renderings" of the indicated texture which are suitable for consumption by an SVBRDF autoencoder, Tensor [B, R, C, 3] representing the ground-truth normal maps of the texture samples, and an SVBRDF embedded with the ground-truth parameter values for the texture samples. ''' assert 0 <= material < len( self.textures ), f'Material {material} falls outside the half-open range [0, {len(self.textures)}).' # Load the normals and SVBRDF parameters of the texture only once; hitting the disk is expensive! texture = self.textures[material] texture_normals = self._load_normals(texture) texture_parameters = self._load_parameters(texture) # The texture and crop dimensions are, of course, the same for each sample. num_texture_rows = self._dims['Texture'][0] num_texture_cols = self._dims['Texture'][1] num_crop_rows = self._dims['Crop'][0] num_crop_cols = self._dims['Crop'][1] # Initializing the batch Tensors with an empty Tensor() invokes special behaviour in utils.concatenate(). batch_inputs = Tensor() batch_normals = Tensor() batch_parameters = Tensor() for sample in range(quantity): lights, viewer = self._lights, self._viewer # Wrapping or reflection crops are not supported so an embedded rectangle will suffice. row_crop, col_crop = utils.sample_embedded_rectangle( num_outer_rows=num_texture_rows, num_inner_rows=num_crop_rows, num_outer_cols=num_texture_cols, num_inner_cols=num_crop_cols) # The SVBRDF is cloned here to avoid polluting the global SVBRDF with arbitrary parameter values. svbrdf = self._svbrdf.clone() svbrdf.parameters = texture_parameters[:, row_crop, col_crop] normals = texture_normals[:, row_crop, col_crop] # It should not matter if each picture in the batch was produced from a different rendering context. for transform in self._transforms: normals, svbrdf.parameters, lights, viewer = transform.apply( normals, svbrdf.parameters, lights, viewer) # The surface radiance can be interpreted as a perspective rendering from the location of the Viewer. radiance = shader.shade(surface=self._surface, normals=normals, lights=lights, viewer=viewer, svbrdf=svbrdf) inputs = torch.cat([radiance, self._radial_distance_field], dim=3).permute(0, 3, 1, 2) # Unlike torch.cat(), utils.concatenate() gracefully handles empty tensors in the first argument. batch_normals = utils.concatenate(batch_normals, normals) batch_parameters = utils.concatenate(batch_parameters, svbrdf.parameters) batch_inputs = utils.concatenate(batch_inputs, inputs) # Clone the SVBRDF (again) to avoid overwriting parameter values in a multiprocessing context. batch_svbrdf = self._svbrdf.clone() batch_svbrdf.parameters = batch_parameters # The structure of the Dataset tuple suggests an association between the normals and SVBRDF. return batch_inputs, (batch_normals, batch_svbrdf)
def build_sampler(tparams, options, trng): x = tensor.matrix('x', dtype='int64') xr = x[::-1] n_timesteps = x.shape[0] n_samples = x.shape[1] # word embedding (source), forward and backward emb = tparams['Wemb'][x.flatten()] emb = emb.reshape([n_timesteps, n_samples, options['dim_word_src']]) embr = tparams['Wemb'][xr.flatten()] embr = embr.reshape([n_timesteps, n_samples, options['dim_word_src']]) # encoder proj = get_layer(options['encoder'])[1](tparams, emb, options, prefix='encoder') projr = get_layer(options['encoder'])[1](tparams, embr, options, prefix='encoder_r') # concatenate forward and backward rnn hidden states ctx = concatenate([proj[0], projr[0][::-1]], axis=proj[0].ndim - 1) # get the input for decoder rnn initializer mlp # ctx_mean = ctx.mean(0) ctx_mean = concatenate([proj[0][-1], projr[0][-1]], axis=proj[0].ndim - 2) init_state = get_layer('ff')[1](tparams, ctx_mean, options, prefix='ff_state', activ=tensor.tanh) LOGGER.info('Building f_init') outs = [init_state, ctx] f_init = theano.function([x], outs, name='f_init', profile=False) # x: 1 x 1 y = tensor.vector('y_sampler', dtype='int64') init_state = tensor.matrix('init_state', dtype='float32') # if it's the first word, emb should be all zero and it is indicated by -1 emb = tensor.switch(y[:, None] < 0, tensor.alloc(0., 1, tparams['Wemb_dec'].shape[1]), tparams['Wemb_dec'][y]) # apply one step of conditional gru with attention proj = get_layer(options['decoder'])[1](tparams, emb, options, prefix='decoder', mask=None, context=ctx, one_step=True, init_state=init_state) # get the next hidden state next_state = proj[0] # get the weighted averages of context for this target word y ctxs = proj[1] dec_alphas = proj[2] logit_lstm = get_layer('ff')[1](tparams, next_state, options, prefix='ff_logit_lstm', activ=None) logit_prev = get_layer('ff')[1](tparams, emb, options, prefix='ff_logit_prev', activ=None) logit_ctx = get_layer('ff')[1](tparams, ctxs, options, prefix='ff_logit_ctx', activ=None) logit = tensor.tanh(logit_lstm + logit_prev + logit_ctx) logit = get_layer('ff')[1](tparams, logit, options, prefix='ff_logit', activ=None) # compute the softmax probability next_probs = tensor.nnet.softmax(logit) # sample from softmax distribution to get the sample next_sample = trng.multinomial(pvals=next_probs).argmax(1) # compile a function to do the whole thing above, next word probability, # sampled word for the next target, next hidden state to be used LOGGER.info('Building f_next') inps = [y, ctx, init_state] outs = [next_probs, next_sample, next_state, dec_alphas] f_next = theano.function(inps, outs, name='f_next', profile=False) return f_init, f_next
def build_model(tparams, options): opt_ret = dict() trng = MRG_RandomStreams(1234) use_noise = theano.shared(numpy.float32(0.)) # description string: #words x #samples x = tensor.matrix('x', dtype='int64') x_mask = tensor.matrix('x_mask', dtype='float32') y = tensor.matrix('y', dtype='int64') y_mask = tensor.matrix('y_mask', dtype='float32') # for the backward rnn, we just need to invert x and x_mask xr = x[::-1] xr_mask = x_mask[::-1] n_timesteps = x.shape[0] n_timesteps_trg = y.shape[0] n_samples = x.shape[1] # word embedding for forward rnn (source) emb = tparams['Wemb'][x.flatten()] emb = emb.reshape([n_timesteps, n_samples, options['dim_word_src']]) proj = get_layer(options['encoder'])[1](tparams, emb, options, prefix='encoder', mask=x_mask) # word embedding for backward rnn (source) embr = tparams['Wemb'][xr.flatten()] embr = embr.reshape([n_timesteps, n_samples, options['dim_word_src']]) projr = get_layer(options['encoder'])[1](tparams, embr, options, prefix='encoder_r', mask=xr_mask) # context will be the concatenation of forward and backward rnns ctx = concatenate([proj[0], projr[0][::-1]], axis=proj[0].ndim - 1) # mean of the context (across time) will be used to initialize decoder rnn # ctx_mean = (ctx * x_mask[:, :, None]).sum(0) / x_mask.sum(0)[:, None] # or you can use the last state of forward + backward encoder rnns ctx_mean = concatenate([proj[0][-1], projr[0][-1]], axis=proj[0].ndim - 2) # initial decoder state init_state = get_layer('ff')[1](tparams, ctx_mean, options, prefix='ff_state', activ=tensor.tanh) # word embedding (target), we will shift the target sequence one time step # to the right. This is done because of the bi-gram connections in the # readout and decoder rnn. The first target will be all zeros and we will # not condition on the last output. emb = tparams['Wemb_dec'][y.flatten()] emb = emb.reshape([n_timesteps_trg, n_samples, options['dim_word_trg']]) emb_shifted = tensor.zeros_like(emb) emb_shifted = tensor.set_subtensor(emb_shifted[1:], emb[:-1]) emb = emb_shifted # decoder - pass through the decoder conditional gru with attention proj = get_layer(options['decoder'])[1](tparams, emb, options, prefix='decoder', mask=y_mask, context=ctx, context_mask=x_mask, one_step=False, init_state=init_state) # hidden states of the decoder gru proj_h = proj[0] # weighted averages of context, generated by attention module ctxs = proj[1] # weights (alignment matrix) opt_ret['dec_alphas'] = proj[2] # compute word probabilities logit_lstm = get_layer('ff')[1](tparams, proj_h, options, prefix='ff_logit_lstm', activ=None) logit_prev = get_layer('ff')[1](tparams, emb, options, prefix='ff_logit_prev', activ=None) logit_ctx = get_layer('ff')[1](tparams, ctxs, options, prefix='ff_logit_ctx', activ=None) logit = tensor.tanh(logit_lstm + logit_prev + logit_ctx) if options['use_dropout']: logit = dropout_layer(logit, use_noise, trng) logit = get_layer('ff')[1](tparams, logit, options, prefix='ff_logit', activ=None) logit_shp = logit.shape probs = tensor.nnet.softmax( logit.reshape([logit_shp[0] * logit_shp[1], logit_shp[2]])) # cost y_flat = y.flatten() y_flat_idx = tensor.arange(y_flat.shape[0]) * options['n_words'] + y_flat cost = -tensor.log(probs.flatten()[y_flat_idx]) cost = cost.reshape([y.shape[0], y.shape[1]]) cost = (cost * y_mask).sum(0) return trng, use_noise, x, x_mask, y, y_mask, opt_ret, cost
def next(self): """ Returns lattice containing projection of the next frame of xyzFile. If file has finnished it returns None value """ frame = self.xyzFile.nextFrame() if frame is None: return None newFrame = XYZFrame() newFrame.boxVectors = self.lattice.boxVectors refFrame = XYZFrame() refFrame.boxVectors = self.lattice.boxVectors atomsLists = self.propagateAtomsThroughPbc(frame.atoms, frame.boxSize) allAtoms = concatenate(atomsLists) #posCount = len(atomsLists[0]) errors = [] differences = [] #match, referenceMatch, errors = self.match(atomsLists) atoms = [] # print self._radius for pos in self.lattice.positions: ACounter = 0 BCounter = 0 for atom in allAtoms: dist = distance( atom.x0, pos.x0 ) atoms.append(atom) if atom.symbol == "DLPC": try: ACounter += 1. / ( dist**2 ) except: ACounter = 1e308 else: if atom.symbol == "DOPC": try: BCounter += 1. / ( dist**2 ) except: BCounter = 1e308 differences.append( (ACounter - BCounter, pos.x0) ) # if ACounter > BCounter: symbol = "DSPC" # else: symbol = "A" # newFrame.atoms.append(XYZAtom(symbol, *pos.x0)) errors.append(0.0) num = 0 for atom in atoms: refFrame.atoms.append(XYZAtom(atom.symbol, *atom.x0)) refFrame.atoms[-1].x += 17 for atom in allAtoms: refFrame.atoms.append(XYZAtom(atom.symbol, *atom.x0)) refFrame.atoms[-1].x += 48 for diff, pos in sorted(differences, key = lambda tup: tup[0]): num += 1 if num < 129: symbol = "DOPC" else: symbol = "DPPC" newFrame.atoms.append(XYZAtom(symbol, *pos)) #for atomIndex in range(posCount): # newFrame.atoms.append(XYZAtom(atomsLists[0][atomIndex].symbol # , *self.lattice.positions[match[atomIndex]].x0)) return ProjectedFrame(newFrame, refFrame, errors)
def build_model(tparams, options): """ Computation graph for the model """ opt_ret = dict() use_noise = theano.shared(numpy.asarray(1., dtype=theano.config.floatX)) try: trng = RandomStreams(1234, use_cuda=True) except: print "Could not apply use_cuda==True in RandonStreams ..." trng = RandomStreams(1234) xs = [] xmasks = [] langs = options['langs'] for lang in langs: # description string: #words x #samples x_lang = tensor.matrix('x_%s' % lang, dtype='int64') mask_lang = tensor.matrix('mask_%s' % lang, dtype='float32') xs.append(x_lang) xmasks.append(mask_lang) xs_r = [] xmasks_r = [] if options['bidirectional_enc']: for i, lang in enumerate(langs): x_lang = xs[i] mask_lang = xmasks[i] # reverse x_lang_r = x_lang[::-1] mask_lang_r = mask_lang[::-1] xs_r.append(x_lang_r) xmasks_r.append(mask_lang_r) sents_all = [] im = tensor.matrix('im', dtype='float32') n_samples = im.shape[0] for i, lang in enumerate(langs): x_lang = xs[i] mask_lang = xmasks[i] n_timesteps_lang = x_lang.shape[0] n_samples_lang = x_lang.shape[1] if options['use_dropout']: # dropout probs for the word embeddings retain_probability_emb = 1 - options['dropout_embedding'] # dropout probs for the RNN hidden states retain_probability_hidden = 1 - options['dropout_hidden'] # dropout probs for the source words retain_probability_source = 1 - options['dropout_source'] # hidden states rec_dropout = shared_dropout_layer( (2, n_samples_lang, options['dim']), use_noise, trng, retain_probability_hidden) rec_dropout_r = shared_dropout_layer( (2, n_samples_lang, options['dim']), use_noise, trng, retain_probability_hidden) # word embeddings emb_dropout = shared_dropout_layer( (2, n_samples_lang, options['dim_word']), use_noise, trng, retain_probability_emb) emb_dropout_r = shared_dropout_layer( (2, n_samples_lang, options['dim_word']), use_noise, trng, retain_probability_emb) # source words source_dropout = shared_dropout_layer( (n_timesteps_lang, n_samples_lang, 1), use_noise, trng, retain_probability_source) source_dropout = tensor.tile(source_dropout, (1, 1, options['dim_word'])) else: # hidden states rec_dropout = theano.shared(numpy.array([1.] * 2, dtype='float32')) rec_dropout_r = theano.shared( numpy.array([1.] * 2, dtype='float32')) # word embeddings emb_dropout = theano.shared(numpy.array([1.] * 2, dtype='float32')) emb_dropout_r = theano.shared( numpy.array([1.] * 2, dtype='float32')) # Word embedding (for a particular language `lang`) # forward emb_lang = tparams['Wemb_%s' % lang][x_lang.flatten()] emb_lang = emb_lang.reshape( [n_timesteps_lang, n_samples_lang, options['dim_word']]) if options['use_dropout']: emb_lang *= source_dropout if options['bidirectional_enc']: x_lang_r = xs_r[i] mask_lang_r = xmasks_r[i] # backward lang encoder emb_lang_r = tparams['Wemb_%s' % lang][x_lang_r.flatten()] emb_lang_r = emb_lang_r.reshape( [n_timesteps_lang, n_samples_lang, options['dim_word']]) if options['use_dropout']: emb_lang_r *= source_dropout[::-1] # Encode sentence in language `lang` if options['encoder_%s' % lang] == 'bow': sents_lang = (emb_lang * mask_lang[:, :, None]).sum(0) else: # iteratively push input from first hidden layer until the last for i in range(int(options['n_enc_hidden_layers'])): layer_name_prefix = 'encoder_%s_%i' % (lang, i) # if first hidden layer use wembs, otherwise output of previous hidden layer layer_below = emb_lang if i == 0 else layer_below[0] # do not apply dropout on word embeddings layer #if options['use_dropout'] and i>0: # layer_below = dropout_layer(layer_below, use_noise, trng, prob=options['dropout_prob']) layer_below = get_layer(options['encoder_%s' % lang])[1]( tparams, layer_below, options, None, prefix=layer_name_prefix, mask=mask_lang, emb_dropout=emb_dropout, rec_dropout=rec_dropout) if i == int(options['n_enc_hidden_layers']) - 1: # sentence embeddings (projections) are the output of the last hidden layer proj_lang = layer_below # apply forward and backward steps and concatenate both if options['bidirectional_enc']: # concatenate forward and backward pass RNNs # iteratively push input from first hidden layer until the last for i in range(int(options['n_enc_hidden_layers'])): layer_name_prefix = 'encoder_%s_r_%i' % (lang, i) # if first hidden layer use wembs, else output of prev hidden layer layer_below = emb_lang_r if i == 0 else layer_below[0] # do not apply dropout on word embeddings layer #if options['use_dropout'] and i>0: # layer_below = dropout_layer(layer_below, use_noise, trng, prob=options['dropout_prob']) layer_below = get_layer(options['encoder_%s' % lang])[1]( tparams, layer_below, options, None, prefix=layer_name_prefix, mask=mask_lang_r, emb_dropout=emb_dropout_r, rec_dropout=rec_dropout_r) if i == int(options['n_enc_hidden_layers']) - 1: # sentence embeddings (projections) are the output of the last hidden layer proj_lang_r = layer_below # use the last state of forward + backward encoder rnns sents_lang = concatenate( [proj_lang[0][-1], proj_lang_r[0][-1]], axis=proj_lang[0].ndim - 2) else: sents_lang = proj_lang[0][-1] if options['use_dropout']: sents_lang *= shared_dropout_layer( (n_samples_lang, options['dim']), use_noise, trng, retain_probability_hidden) # project sentences into multimodal space sents_mm = get_layer('ff')[1](tparams, sents_lang, options, prefix='ff_sentence_mm', activ='linear') if options['attention_type'] == 'dot': sents_mm = l2norm(sents_mm) if options['use_dropout']: sents_mm *= shared_dropout_layer( (n_samples_lang, options['dim_multimodal']), use_noise, trng, retain_probability_hidden) sents_all.append(sents_mm) # Encode images images = get_layer('ff')[1](tparams, im, options, prefix='ff_image_mm', activ='linear') if options['attention_type'] == 'dot': images = l2norm(images) if options['use_dropout']: images *= shared_dropout_layer((n_samples, options['dim_multimodal']), use_noise, trng, retain_probability_hidden) # Compute loss lambda_img_sent = options['lambda_img_sent'] lambda_sent_sent = options['lambda_sent_sent'] if options['use_all_costs']: cost = contrastive_loss_all(tparams, options, images, sents_all, lambda_img_sent, lambda_sent_sent) else: cost = contrastive_loss(tparams, options, images, sents_all) # return flattened inputs inps = [] inps.extend(xs) inps.extend(xmasks) inps.append(im) return trng, inps, cost
def build_model(tparams, options): opt_ret = dict() trng = RandomStreams(1234) use_noise = theano.shared(np.float32(0.)) # description string: #words x #samples x = tensor.matrix('x', dtype='int64') x_mask = tensor.matrix('x_mask', dtype='float32') y = tensor.matrix('y', dtype='int64') y_mask = tensor.matrix('y_mask', dtype='float32') # for the backward rnn, we just need to invert x and x_mask xr = x[::-1] xr_mask = x_mask[::-1] n_timesteps = x.shape[0] n_timesteps_trg = y.shape[0] n_samples = x.shape[1] # word embedding for forward rnn (source) emb = tparams['Wemb'][x.flatten()] emb = emb.reshape([n_timesteps, n_samples, options['dim_word']]) proj = get_layer(options['encoder'])[1](tparams, emb, options, prefix='encoder', mask=x_mask) # word embedding for backward rnn (source) embr = tparams['Wemb'][xr.flatten()] embr = embr.reshape([n_timesteps, n_samples, options['dim_word']]) projr = get_layer(options['encoder'])[1](tparams, embr, options, prefix='encoder_r', mask=xr_mask) # context will be the concatenation of forward and backward rnns ctx = concatenate([proj[0], projr[0][::-1]], axis=proj[0].ndim - 1) # mean of the context (across time) will be used to initialize decoder rnn ctx_mean = (ctx * x_mask[:, :, None]).sum(0) / x_mask.sum(0)[:, None] # or you can use the last state of forward + backward encoder rnns # ctx_mean = concatenate([proj[0][-1], projr[0][-1]], axis=proj[0].ndim-2) # initial decoder state init_state = get_layer('ff')[1](tparams, ctx_mean, options, prefix='ff_state', activ='tanh') # word embedding (target), we will shift the target sequence one time step # to the right. This is done because of the bi-gram connections in the # readout and decoder rnn. The first target will be all zeros and we will # not condition on the last output. emb = tparams['Wemb_dec'][y.flatten()] emb = emb.reshape([n_timesteps_trg, n_samples, options['dim_word']]) emb_shifted = tensor.zeros_like(emb) emb_shifted = tensor.set_subtensor(emb_shifted[1:], emb[:-1]) emb = emb_shifted # decoder - pass through the decoder conditional gru with attention proj = get_layer(options['decoder'])[1](tparams, emb, options, prefix='decoder', mask=y_mask, context=ctx, context_mask=x_mask, one_step=False, init_state=init_state) # hidden states of the decoder gru proj_h = proj[0] # n_timestep * n_sample * dim # weighted averages of context, generated by attention module ctxs = proj[1] # weights (alignment matrix) opt_ret['dec_alphas'] = proj[2] # compute word probabilities logit_lstm = get_layer('ff')[1](tparams, proj_h, options, prefix='ff_logit_lstm', activ='linear') logit_prev = get_layer('ff')[1](tparams, emb, options, prefix='ff_logit_prev', activ='linear') logit_ctx = get_layer('ff')[1](tparams, ctxs, options, prefix='ff_logit_ctx', activ='linear') logit = tensor.tanh(logit_lstm + logit_prev + logit_ctx) # n_timestep * n_sample * dim_word if options['use_dropout']: logit = dropout_layer(logit, use_noise, trng) logit = get_layer('ff')[1](tparams, logit, options, prefix='ff_logit', activ='linear') # n_timestep * n_sample * n_words logit_shp = logit.shape probs = tensor.nnet.softmax(logit.reshape([logit_shp[0] * logit_shp[1], logit_shp[2]])) # cost y_flat = y.flatten() y_flat_idx = tensor.arange(y_flat.shape[0]) * options['n_words'] + y_flat cost = -tensor.log(probs.flatten()[y_flat_idx]) cost = cost.reshape([y.shape[0], y.shape[1]]) cost = (cost * y_mask).sum(0) return trng, use_noise, x, x_mask, y, y_mask, opt_ret, cost, ctx_mean
def build_sentence_encoders(tparams, options): """ Sentence encoder only to be used at test time """ opt_ret = dict() trng = RandomStreams(1234) #xs, masks, sents_all = [], [], [] in_outs = [] langs = options['langs'] for lang in langs: # description string: #words x #samples # forward x = tensor.matrix('x_%s' % lang, dtype='int64') mask = tensor.matrix('x_mask_%s' % lang, dtype='float32') n_timesteps = x.shape[0] n_samples = x.shape[1] # Word embedding (forward) emb = tparams['Wemb_%s' % lang][x.flatten()].reshape( [n_timesteps, n_samples, options['dim_word']]) if options['bidirectional_enc']: # backward RNN x_r = x[::-1] mask_r = mask[::-1] emb_r = tparams['Wemb_%s' % lang][x_r.flatten()].reshape( [n_timesteps, n_samples, options['dim_word']]) if options['use_dropout']: retain_probability_emb = 1 - options['dropout_embedding'] retain_probability_hidden = 1 - options['dropout_hidden'] retain_probability_source = 1 - options['dropout_source'] rec_dropout = theano.shared( numpy.array([retain_probability_hidden] * 2, dtype='float32')) rec_dropout_r = theano.shared( numpy.array([retain_probability_hidden] * 2, dtype='float32')) emb_dropout = theano.shared( numpy.array([retain_probability_emb] * 2, dtype='float32')) emb_dropout_r = theano.shared( numpy.array([retain_probability_emb] * 2, dtype='float32')) source_dropout = theano.shared( numpy.float32(retain_probability_source)) emb *= source_dropout if options['bidirectional_enc']: embr *= source_dropout else: rec_dropout = theano.shared(numpy.array([1.] * 2, dtype='float32')) rec_dropout_r = theano.shared( numpy.array([1.] * 2, dtype='float32')) emb_dropout = theano.shared(numpy.array([1.] * 2, dtype='float32')) emb_dropout_r = theano.shared( numpy.array([1.] * 2, dtype='float32')) # Encode sentences if options['encoder_%s' % lang] == 'bow': sents = (emb * mask[:, :, None]).sum(0) else: # iteratively push input from first hidden layer until the last for i in range(int(options['n_enc_hidden_layers'])): layer_name_prefix = 'encoder_%s_%i' % (lang, i) # if first layer input are wembs, otherwise input will be output of last hidden layer layer_below = emb if i == 0 else layer_below[0] layer_below = get_layer(options['encoder_%s' % lang])[1]( tparams, layer_below, options, None, prefix=layer_name_prefix, mask=mask, emb_dropout=emb_dropout, rec_dropout=rec_dropout) if i == int(options['n_enc_hidden_layers']) - 1: # sentence embeddings (projections) are the output of the last hidden layer proj = layer_below if options['bidirectional_enc']: for i in range(int(options['n_enc_hidden_layers'])): layer_name_prefix = 'encoder_%s_r_%i' % (lang, i) # if first layer input are wembs, otherwise input will be output of last hidden layer layer_below = emb_r if i == 0 else layer_below[0] layer_below = get_layer(options['encoder_%s' % lang])[1]( tparams, layer_below, options, None, prefix=layer_name_prefix, mask=mask_r, emb_dropout=emb_dropout_r, rec_dropout=rec_dropout_r) if i == int(options['n_enc_hidden_layers']) - 1: # sentence embeddings (projections) are the output of the last hidden layer proj_r = layer_below # use last hidden state of forward and backward RNNs sents = concatenate([proj[0][-1], proj_r[0][-1]], axis=proj[0].ndim - 2) else: sents = proj[0][-1] if options['use_dropout']: sents *= shared_dropout_layer((n_samples, options['dim']), use_noise, trng, retain_probability_hidden) # project sentences into multimodal space sents_mm = get_layer('ff')[1](tparams, sents, options, prefix='ff_sentence_mm', activ='linear') if not 'attention_type' in options or options[ 'attention_type'] == 'dot': sents_mm = l2norm(sents_mm) if options['use_dropout']: sents_mm *= shared_dropout_layer( (n_samples, options['dim_multimodal']), use_noise, trng, retain_probability_hidden) # outputs per language in_outs.append(([x, mask], sents_mm)) return trng, in_outs
def build_encoder(self, dim_emb=0, factors=1, dim_per_factor=(), encoder_layers=1, encoder='gru', dropout=False, dropout_word=0., dropout_emb=0., dropout_rec=0., trng=None, use_noise=True, **kwargs): """ Build the bi-directional encoder (bi-gru / bi-lstm) MUST match nmt.model.build_encoder for multitasking :param dim_emb: :param factors: :param dim_per_factor: :param encoder_layers: :param encoder: gru or lstm :param dropout: :param dropout_word: :param dropout_emb: :param dropout_rec: :param trng: :param use_noise: apply dropout (use_noise) or not (test) :param kwargs: :return: """ logger.warn('Building encoder - use_noise: {}'.format(use_noise)) assert sum(dim_per_factor) == dim_emb, 'sum dim_per_factor != dim_emb' dropout = dropout and use_noise # no dropout during test time # input to forward rnn (#factors x #words x #batch_size) x = tensor.tensor3('x_word', dtype='int64') x.tag.test_value = np.random.randint(1000, size=(1, 3, 2)) # input to the masking function (#words x #batch_size) # mask is set to 0 when we are padding the input # see prepare_batch() for more details x_mask = tensor.matrix('x_mask', dtype=floatX) x_mask.tag.test_value = np.ones([3, 2]).astype('float32') x_mask.tag.test_value[2, 0] = 0 x_mask.tag.test_value[1, 1] = 0 x_mask.tag.test_value[2, 1] = 0 if kwargs['verbose'] and kwargs['compute_test_values'] == 'warn': logger.warn(x.tag.test_value) logger.warn(x_mask.tag.test_value) # input to backward rnn (x and x_mask reversed) x_bw = x[:, ::-1] x_mask_bw = x_mask[::-1] if kwargs['verbose']: logger.warn(x_bw.tag.test_value) logger.warn(x_mask_bw.tag.test_value) n_timesteps = x.shape[1] # length of longest sentence in batch batch_size = x.shape[2] # size of this batch (can vary!) # forward RNN # first build the forward embeddings # we do the concatenate() in the case that we have multiple factors # for the input data. Not currently used in this code but inherited # from nematus. emb_fw = [] for factor in range(factors): emb_fw.append(self.theano_params[embedding_name(factor)][ x[factor].flatten()]) emb_fw = concatenate(emb_fw, axis=1) emb_fw = emb_fw.reshape([n_timesteps, batch_size, dim_emb]) # drop out whole words by zero-ing their embeddings if dropout and dropout_word > 0.: logger.warn('Using word dropout (p={})'.format(dropout_word)) p = 1 - dropout_word word_drop = inv_dropout_mask((n_timesteps, batch_size, 1), trng, p) word_drop = tensor.tile(word_drop, (1, 1, dim_emb)) emb_fw *= word_drop # now build the forward rnn layer fw_layers = [ get_layer(encoder)[1](self.theano_params, emb_fw, trng=trng, prefix='enc_fw_0', mask=x_mask, dropout=dropout, dropout_W=dropout_emb, dropout_U=dropout_rec) ] # backward rnn # first build the backward embeddings # Same deal with the concatenate() of the factors as emb_fw. emb_bw = [] for factor in range(factors): emb_bw.append(self.theano_params[embedding_name(factor)][ x_bw[factor].flatten()]) emb_bw = concatenate(emb_bw, axis=1) emb_bw = emb_bw.reshape([n_timesteps, batch_size, dim_emb]) # drop out the same words as above in forward rnn if dropout and dropout_word > 0.: logger.warn('Also dropping bw words (p={})'.format(dropout_word)) emb_bw *= word_drop[::-1] # now build the backward rnn layer bw_layers = [ get_layer(encoder)[1](self.theano_params, emb_bw, trng=trng, prefix='enc_bw_0', mask=x_mask_bw, dropout=dropout, dropout_W=dropout_emb, dropout_U=dropout_rec) ] # add additional layers if Deep Encoder is specified for i in range(1, encoder_layers): # add additional layers if specified input_states = concatenate( (fw_layers[i - 1][0], bw_layers[i - 1][0][::-1]), axis=2) fw_layers.append( get_layer(encoder)[1](self.theano_params, input_states, trng=trng, prefix='enc_fw_{}'.format(i), mask=x_mask, dropout=dropout, dropout_W=dropout_emb, dropout_U=dropout_rec)) bw_layers.append( get_layer(encoder)[1](self.theano_params, input_states[::-1], trng=trng, prefix='enc_bw_{}'.format(i), mask=x_mask_bw, dropout=dropout, dropout_W=dropout_emb, dropout_U=dropout_rec)) # combine final layers of forward and backward RNNs # this is a concatenated vector, which means it has dim=dim*2 # [::-1] means reverse the bw_layers states = concatenate((fw_layers[-1][0], bw_layers[-1][0][::-1]), axis=2) if kwargs['verbose']: logger.info("encoder_states shape {}".format( states.tag.test_value.shape)) if kwargs['mean_birnn']: # calculate a 2D vector over time but summing over axis 0 mlp_input = (states * x_mask[:, :, None]).sum(0) / x_mask.sum(0)[:, None] elif kwargs['final_birnn']: mlp_input = concatenate( (fw_layers[-1][-1][-1, :, :], bw_layers[-1][-1][-1, :, :]), axis=fw_layers[-1][-1].ndim - 2) if kwargs['verbose']: logger.info("FW LAYERS shape {}".format( fw_layers[-1][0][-1, :, :].tag.test_value.shape)) logger.info(fw_layers[-1][0][-1, :, :].tag.test_value) logger.info("BW LAYERS shape {}".format( bw_layers[-1][0][-1, :, :].tag.test_value.shape)) logger.info(bw_layers[-1][0][-1, :, :].tag.test_value) logger.info("CONCAT shape {}".format( mlp_input.tag.test_value.shape)) logger.info(mlp_input.tag.test_value) # Why don't we return the reverse mask? return x, x_mask, states, mlp_input