def __init__(self, **kwargs): super().__init__(**kwargs) # Define decoder with tf.variable_scope('decoder'): cell_decoder = rnn_cell(num_units=self.config.num_units, num_cells=self.config.num_cells, keep_prob=self.keep_prob_ph, add_dropout=self.add_dropout) self.decoder = Decoder( cell=cell_decoder, max_iterations=self.config.max_dec_seq_length + 1, infer_token_prob=self.infer_token_prob, alphabet=self.alphabet, use_scheduled_sampling=False, check_seq_lengths=True, parallel_iterations=self.parallel_iterations, swap_memory=self.swap_memory) # Inputs for the one-step-at-a-time decoding self.decoder_inputs = tf.placeholder(tf.int32, shape=[None], name='decoder_inputs') self.decoder_state = tf.placeholder( tf.float32, shape=[None, 2, self.config.num_cells, self.config.num_units], name='decoder_state')
def decodefunc(args): message = args.message key = args.key levels = args.levels pattern = args.pattern shift = args.caesar show_all = args.show_all decoder = Decoder() decoders = [ "base16", "base32", "base64", "base85", "autokey", "atbash", "baconian", "beaufort", "caesar", "morse", "rot13", "rot47", "playfair", "vigenere" ] combdecs = list(product(decoders, repeat=levels)) for arraydecs in combdecs: translated = message flow = "" for dec in arraydecs: translated = decoder.decode(dec, translated, shift, key) if translated is None: if show_all is not None and show_all: print(Fore.RED + " > ".join(arraydecs)) break if translated is not None: if pattern is None or (pattern and pattern in translated): print(Fore.WHITE + " > ".join(arraydecs) + ": " + Fore.BLUE + translated)
def bit_error_probability(p, matrix): decoder = Decoder(p, matrix) diff = 0 k = len(matrix) for i in range(N): sequences = generate_sequences(k) for sequence in sequences: codeword = np.mod(np.sum(np.multiply(matrix, sequence), axis=0), 2) received = np_to_str(spoil_codeword(codeword, p)) diff += difference(sequence.tolist(), [[int(j)] for j in decoder.decode(received)]) return diff / float((N * k * (2**k)))
def __init__(self): self.config = Config() self.cookies = login() logger.info(self.cookies) self.base_path = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), self.config.config['output']['path']) self.live_infos = Infos() self.display = Display() self.decoder = Decoder() self.uploader = Upload() self.threadRecorder = threadRecorder() logger.info('基路径:%s' % (self.base_path)) self.load_room_info() self.get_live_url() logger.info('初始化完成')
def decodefunc(message, key, levels, pattern): decoder = Decoder() decoders = [ "base16", "base32", "base64", "base85", "morse", "rot13", "rot47", "vigenere" ] combdecs = list(product(decoders, repeat=levels)) for arraydecs in combdecs: translated = message flow = "" for dec in arraydecs: flow = flow + " " + dec + " >" translated = decoder.decode(dec, translated, key) if translated is None: break if translated is not None: if pattern is None or (pattern and pattern in translated): print(Fore.WHITE + flow + " " + Fore.BLUE + translated)
def __init__(self, input_shape, latent_dim, filter_factor, activation='relu', hierarchical=False, vampprior=False, expected_value=False, hidden_dim=300, data_set='mnist'): super(Vae, self).__init__() self.latent_dim = latent_dim self.vampprior = vampprior self.expected_value = expected_value self.hierarchical = hierarchical print(f'vampprior: {vampprior}\n' f'expected_value: {expected_value}\n' f'hierarchical: {hierarchical}') self.encoder = Encoder( input_shape=input_shape, latent_dim=latent_dim if not hierarchical else int(latent_dim / 2), activation=activation, factor=filter_factor, name='encoder_z') if hierarchical: self.encoder_z0 = Encoder(input_shape=input_shape, latent_dim=int(latent_dim / 2), activation=activation, factor=filter_factor, conditional_on_other_z=True, name='encoder_z0') self.connect_z0 = tf.keras.Sequential([ tf.keras.layers.InputLayer(input_shape=int(latent_dim / 2)), tf.keras.layers.Dense(hidden_dim, activation=activation), tf.keras.layers.Dense(hidden_dim, activation=activation) ]) self.p_z0 = DensityLayer(int(latent_dim / 2)) self.decoder = Decoder(input_shape=input_shape, latent_dim=latent_dim, activation=activation, factor=filter_factor) if self.vampprior: self.batch_size_u = 500 self.pseudo_inputs_layer = TrainablePseudoInputs( self.batch_size_u, data_set)
class Live(): def __init__(self): self.config = Config() self.cookies = login() logger.info(self.cookies) self.base_path = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), self.config.config['output']['path']) self.live_infos = Infos() self.display = Display() self.decoder = Decoder() self.uploader = Upload() self.threadRecorder = threadRecorder() logger.info('基路径:%s' % (self.base_path)) self.load_room_info() self.get_live_url() logger.info('初始化完成') def create_duration(self, start_time, end_time): t = datetime.datetime.now() tt = t.strftime('%Y%m%d %H%M%S') if start_time == end_time == '0': return '0' tmp = datetime.datetime.strptime(tt.split(' ')[0] + ' %s' % start_time, '%Y%m%d %H%M%S') if t > tmp: base_time1 = tt.split(' ')[0] base_time2 = (t + datetime.timedelta(days=1)).strftime('%Y%m%d %H%M%S').split(' ')[0] else: base_time1 = (t - datetime.timedelta(days=1)).strftime('%Y%m%d %H%M%S').split(' ')[0] base_time2 = tt.split(' ')[0] if start_time > end_time: start_time = '%s %s' % (base_time1, start_time) end_time = '%s %s' % (base_time2, end_time) else: start_time = '%s %s' % (tt.split(' ')[0], start_time) end_time = '%s %s' % (tt.split(' ')[0], end_time) return '%s-%s' % (start_time, end_time) def check_live(self, key): duration = self.live_infos.get(key)['duration'] if duration == '0': return True lst = duration.split('-') now_time = datetime.datetime.now() if len(lst) == 2: start_time = datetime.datetime.strptime(lst[0], '%Y%m%d %H%M%S') end_time = datetime.datetime.strptime(lst[1], '%Y%m%d %H%M%S') if now_time > start_time and now_time < end_time: return True else: logger.debug('%s[RoomID:%s]不在直播时间段' % (self.live_infos.get(key)['uname'], key)) return False else: return False def load_room_info(self): live_infos = self.live_infos.copy() for lst in self.config.config['live']['room_info']: if lst[0] not in live_infos: live_info = {} live_info['record_start_time'] = '' live_info['queue_status'] = 0 live_info['recording'] = 0 live_info['finish_time'] = '' else: live_info = live_infos[lst[0]] live_info['need_rec'] = lst[1] live_info['need_mask'] = lst[2] live_info['maxsecond'] = lst[3] live_info['need_upload'] = lst[4] live_info['duration'] = self.create_duration(lst[5], lst[6]) live_info['cookies'] = self.cookies live_info['base_path'] = self.base_path self.live_infos.update(lst[0],live_info) def load_realtime(self): ''' 实时加载配置,更新房间信息 ''' self.config.load_cfg() # logger.info(self.config.config) room_lst = [i[0] for i in self.config.config['live']['room_info']] del_lst = [] for key in self.live_infos.copy(): if key not in room_lst: del_lst.append(key) for key in del_lst: self.live_infos.delete(key) self.load_room_info() def judge_in(self, key): room_lst = [i[0] for i in self.config.config['live']['room_info']] if key not in room_lst: return False return True def judge_download(self,key): if not self.judge_in(key): return False live_info = self.live_infos.copy()[key] if live_info['live_status'] != 1: live_info['recording'] =0 self.live_infos.update(key,live_info) return False elif not self.check_live(key) and live_info['live_status'] == 1: live_info['recording'] =2 self.live_infos.update(key,live_info) return False elif self.check_live(key) and live_info['live_status'] == 1 and live_info['need_rec'] == '0': live_info['recording']=3 self.live_infos.update(key,live_info) return False elif live_info['live_status'] == 1 and live_info['need_rec'] == '1': live_info['recording']=1 self.live_infos.update(key,live_info) return True else: logger.warning('%s[RoomID:%s]进入了未知的分支呢' % (live_info['uname'],key)) live_info['recording'] =0 self.live_infos.update(key,live_info) return False def get_live_url(self): ''' 获取所有监听直播间的信息 ''' room_lst = [i[0] for i in self.config.config['live']['room_info']] for id in room_lst: info = None while info is None: try: info = live.get_room_info(id, cookies=self.cookies) time.sleep(0.3) except: logger.error('[RoomID:%s]获取信息失败,重新尝试' % (id)) continue live_info = self.live_infos.copy()[id] live_info['room_id'] = id live_info['real_id'] = info['room_info']['room_id'] try: if live_info['live_status'] != 1 and info['room_info']['live_status'] == 1: logger.info('%s[RoomID:%s]开播了' % (live_info['uname'], id)) toaster = ToastNotifier() toaster.show_toast("开播通知", '%s[RoomID:%s]开播了' % (live_info['uname'], id), icon_path=None, duration=3) except: pass try: live_info['live_status'] = info['room_info']['live_status'] live_info['uid'] = info['room_info']['uid'] live_info['uname'] = info['anchor_info']['base_info']['uname'] live_info['save_name'] = '%s_%s.flv' % ( live_info['uname'], time.strftime("%Y%m%d%H%M%S", time.localtime())) live_info['title'] = info['room_info']['title'] live_info['live_start_time'] = info['room_info']['live_start_time'] self.live_infos.update(id,live_info) logger.debug( '%s[RoomID:%s]直播状态\t%s' % (live_info['uname'], id, live_info['live_status'])) except Exception as e: logger.critical(e) logger.error('[RoomID:%s]房间信息更新失败' % (id)) logger.error(info) # logger.info(self.live_infos.copy()) def get_stream(self, key): ''' 获取直播流 :param key: 房间显示id :return: stream ''' if not self.judge_in(key): return None live_info = self.live_infos.copy()[key] logger.info('%s[RoomID:%s]获取直播流' % (live_info['uname'], key)) session = streamlink.Streamlink() session.set_option("http-cookies", self.cookies) session.set_option("http-headers", headers) log_path = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), 'log', 'stream.log') session.set_loglevel("debug") session.set_logoutput(open(log_path, 'a',encoding='utf-8')) streams = None while streams is None: try: streams = session.streams('https://live.bilibili.com/%s' % key) except: logger.warning('%s[RoomID:%s]获取直播流失败,正在重试' % (live_info['uname'], key)) time.sleep(1) if streams == {}: logger.error('%s[RoomID:%s]未获取到直播流,可能是下播或者网络问题' % (live_info['uname'], key)) return None if 'best' in streams: logger.info('%s[RoomID:%s]获取到best直播流' % (live_info['uname'], key)) return streams['best'] elif 'source' in streams: logger.info('%s[RoomID:%s]获取到source直播流' % (live_info['uname'], key)) return streams['source'] elif 'worst' in streams: logger.info('%s[RoomID:%s]获取到worst直播流' % (live_info['uname'], key)) return streams['worst'] else: logger.info('%s[RoomID:%s]未获取到直播流' % (live_info['uname'], key)) return None def unlive(self, key, unlived): if not self.judge_in(key): return None live_info = self.live_infos.copy()[key] logger.info('%s[RoomID:%s]似乎下播了' % (live_info['uname'], key)) live_info['recording'] = 0 logger.info('%s[RoomID:%s]录制结束,录制了%.2f分钟' % (live_info['uname'], key, ( datetime.datetime.now() - datetime.datetime.strptime( live_info['record_start_time'], '%Y-%m-%d %H:%M:%S')).total_seconds() / 60.0)) live_info['record_start_time'] = '' if unlived: logger.info('%s[RoomID:%s]确认下播,加入转码上传队列' % (live_info['uname'], key)) # if live_info['need_upload'] == '1': # live_info['filename'] = live_info['uname'] + live_info['duration'].split('-')[0].split(' ')[0] # live_info['filepath'] = os.path.join(self.base_path, live_info['uname'], '%s_%s' % (live_info['uname'], live_info['duration'].split('-')[0].split(' ')[0])) # if live_info['need_mask'] == '1': # live_info['filepath'] += '_mask.mp4' # else: # live_info['filepath'] += '.mp4' self.decoder.enqueue(key) self.live_infos.update(key,live_info) def download_live(self, key): if not self.judge_in(key): return None save_path = os.path.join(self.base_path, self.live_infos.get(key)['uname'], 'recording') logger.info('%s[RoomID:%s]准备下载直播流,保存在%s' % (self.live_infos.get(key)['uname'], key, save_path)) self.live_infos.get(key)['recording'] = 1 if not os.path.exists(save_path): os.makedirs(save_path) stream = self.get_stream(key) if stream is None: logger.error('%s[RoomID:%s]获取直播流失败' % (self.live_infos.get(key)['uname'], key)) self.live_infos.get(key)['record_start_time'] = '' self.live_infos.get(key)['recording'] = 0 return filename = os.path.join(save_path, self.live_infos.get(key)['save_name']) self.live_infos.get(key)['record_start_time'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') try: fd = stream.open() except Exception as e: self.unlive(key,unlived=False) logger.critical('%s[RoomID:%s]fd open error' % (self.live_infos.get(key)['uname'], key)) logger.error(e) return with open(filename, 'wb') as f: while self.judge_in(key) and self.live_infos.get(key)['live_status'] == 1 and self.live_infos.get(key)[ 'need_rec'] == '1' and self.check_live(key): try: data = fd.read(1024 * 8) if len(data) > 0: f.write(data) else: fd.close() logger.warning('%s[RoomID:%s]直播流断开,尝试重连' % (self.live_infos.get(key)['uname'], key)) stream = self.get_stream(key) if stream is None: logger.warning('%s[RoomID:%s]重连失败' % (self.live_infos.get(key)['uname'], key)) self.unlive(key, True) return else: logger.info('%s[RoomID:%s]重连成功' % (self.live_infos.get(key)['uname'], key)) fd = stream.open() except Exception as e: fd.close() self.unlive(key,unlived=False) logger.critical('%s[RoomID:%s]遇到了什么问题' % (self.live_infos.get(key)['uname'], key)) logger.error(e) return fd.close() self.unlive(key, True) def run(self): while True: time.sleep(1) self.load_realtime() self.get_live_url() live_infos = self.live_infos.copy() for key in live_infos: if live_infos[key]['recording'] != 1 and self.judge_download(key): self.threadRecorder.add('download_live_%s' % (key),self.download_live,[key,],False) time.sleep(0.2) def start(self): self.threadRecorder.add('display_run',self.display.run,None,False) self.threadRecorder.add('decoder_run',self.decoder.run,None,False) self.threadRecorder.add('uploader_run',self.uploader.run,None,False) self.threadRecorder.add('live_run',self.run,None,False)
# -*- coding: utf-8 -*- """NewsCartSendgridAPI SendgridService This module contains helper classes and methods for interacting with the Sendgrid API Documentation can be found here: https://sendgrid.com/docs/Integrate/Code_Examples/v2_Mail/python.html Code Samples found here: https://github.com/sendgrid/sendgrid-python/blob/master/examples/helpers/mail/mail_example.py """ from utils.decoder import Decoder from sendgrid.helpers.mail import Mail, Personalization, Email, Content DECODER = Decoder() class TemplateBuilder(object): """ Helper class to build Sendgrid compatible Mailer Template """ def __init__(self, config): if config is None: raise ValueError('Missing config for SendgridService Constructor') self.template_config = config self.template = None def build_template(self): """ Builds mailer template based on given config """
class RNN_BLSTM(RNN): def __init__(self, **kwargs): super().__init__(**kwargs) # Define encoder with tf.variable_scope('encoder'): cell_encoder = rnn_cell(num_units=self.config.num_units, num_cells=self.config.num_cells, keep_prob=self.keep_prob_ph, add_dropout=self.add_dropout) self.encoder = BidirectionalEncoder( cell=cell_encoder, parallel_iterations=self.parallel_iterations, swap_memory=self.swap_memory) # Define decoder with tf.variable_scope('decoder'): cell_decoder = rnn_cell( num_units=2 * self.config.num_units, # Because we use bidirectional encoder num_cells=self.config.num_cells, keep_prob=self.keep_prob_ph, add_dropout=self.add_dropout) self.decoder = Decoder( cell=cell_decoder, max_iterations=self.config.max_dec_seq_length + 1, infer_token_prob=self.infer_token_prob, use_scheduled_sampling=self.config.scheduled_sampling, alphabet=self.alphabet, parallel_iterations=self.parallel_iterations, swap_memory=self.swap_memory) # Inputs for the one-step-at-a-time decoding self.decoder_inputs = tf.placeholder(tf.int32, shape=[None], name='decoder_inputs') self.decoder_state = tf.placeholder( tf.float32, shape=[None, 2, self.config.num_cells, 2 * self.config.num_units], name='decoder_state') def setup_network(self): # Setup character embedding embedded_encoder_input, embedded_decoder_input, embed_func = self.setup_character_embedding( ) # Output projection with tf.variable_scope('alphabet_projection') as scope: self.projection_W, self.projection_b = intialize_projections( input_size=2 * self.config.num_units, # Because we use bidirectional encoder output_size=self.config.alphabet_size, scope=scope) # Define alphabet projection function def project_func(output): return projection(output, W=self.projection_W, b=self.projection_b) # Encoder with tf.variable_scope('encoder') as scope: enc_outputs, enc_final_state = self.encoder.encode( inputs=embedded_encoder_input, seq_lengths=self.encoder_sequence_length, scope=scope) # Set decoder initial state and encoder outputs based on the binary # mode input value # - If `self.is_lm_mode=0` Use the passed initial state from encoder # - If `self.is_lm_mode=1` Use the zero vector enc_outputs, self.enc_final_state = select_decoder_inputs( is_lm_mode=self.is_lm_mode, enc_outputs=enc_outputs, initial_state=enc_final_state) # Pack state to tensor self.enc_final_state_tensor = pack_state_tuple(self.enc_final_state) # Define decoder with tf.variable_scope('decoder'): dec_outputs, dec_final_state = self.decoder.decode( inputs=embedded_decoder_input, initial_state=self.enc_final_state, seq_length=self.decoder_sequence_length, embed_func=embed_func, project_func=project_func) # Project output to alphabet size and reshape dec_outputs = tf.reshape(dec_outputs, [-1, 2 * self.config.num_units]) dec_outputs = projection(dec_outputs, W=self.projection_W, b=self.projection_b) dec_outputs = tf.reshape(dec_outputs, [ -1, self.config.max_dec_seq_length + 1, self.config.alphabet_size ]) if self.prediction_mode: dec_outputs = self.decoder_logits # Define loss self.setup_losses(dec_outputs=dec_outputs, target_chars=self.target_chars, decoder_sequence_length=self.decoder_sequence_length) if self.prediction_mode: # Look up inputs decoder_inputs_embedded = tf.nn.embedding_lookup( self.embedding_matrix, self.decoder_inputs, name='decoder_input') is_lm_mode_tensor = tf.to_float( tf.expand_dims(self.is_lm_mode, axis=1)) decoder_inputs = tf.concat( [decoder_inputs_embedded, is_lm_mode_tensor], axis=1) # Unpack state initial_state = unpack_state_tensor(self.decoder_state) with tf.variable_scope('decoder', reuse=True): decoder_output, decoder_final_state = self.decoder.predict( inputs=decoder_inputs, initial_state=initial_state) # Project output to alphabet size self.decoder_output = projection(decoder_output, W=self.projection_W, b=self.projection_b, name='decoder_output') # Compute decayed logits self.decoder_probs_decayed = compute_decayed_probs( logits=self.decoder_output, decay_parameter_ph=self.probs_decay_parameter) # Pack state to tensor self.decoder_final_state = pack_state_tuple( decoder_final_state, name='decoder_final_state')
def main(): # Setup input ports colorSensorStop = ev3.LightSensor('in1') colorSensorLeft = ev3.ColorSensor('in2') colorSensorRight = ev3.ColorSensor('in3') assert colorSensorStop.connected, "LightSensorStop(ColorSensor) is not connected" assert colorSensorLeft.connected, "LightSensorLeft(ColorSensor) is not conected" assert colorSensorRight.connected, "LightSensorRight(ColorSensor) is not conected" #colorSensorStop.MODE_REFLECT #colorSensorLeft.raw #colorSensorRight.raw # Setup output ports mDiff = MoveDifferential(OUTPUT_A, OUTPUT_D, EV3Tire, 15 * STUD_MM) mDiff.left_motor.polarity = "normal" mDiff.right_motor.polarity = "normal" # setup gracefully shutdown def signal_handler(sig, frame): print('Shutting down gracefully') mDiff.stop() exit(0) signal.signal(signal.SIGINT, signal_handler) print('Press Ctrl+C to exit') # setup decoder and define chain of actions #string_of_actions = "urrruulldddl" #string_of_actions = "fffblfrffrfrffb" #"ffffrfrfrfblfflffrflfb" string_of_actions = "llll.uddllu.r.r.r.r.rdr.u.uruulld.r.rlddllu.luulld.rur.d.dull.d.rd.r.r.rdr.u.uruurrd.lul.dulld.rddlllluur.dld.r.r.rdr.u.udlllldllu.r.r.r.r.rdr.u" #string_of_actions = "lffffrffbtffrffrfrffffffbrflflfffbrflfflfflflfffbtflffrffrflffbrfflfflflffblfrffffbtflfflffblffbrflffffbrflflfffbrflfflfflflffblfrfrffbtfrffrfrffbrflfflfffrffffrffrfrffbrflflffffbrflflfffbtfrfffflfrffrfrffffffbrflflff" #string_of_actions = "lfffrfflf" #string_of_actions = "l" decoder = Decoder(string_of_actions, DEFINED_ACTIONS) current_action = decoder.get_next_action() while current_action != NOT_DEFINED: # current action == -1 -> no more actions to execute if (current_action == FORWARD): move_forward(mDiff, colorSensorStop, colorSensorLeft, colorSensorRight) current_action = decoder.get_next_action() #if(current_action == FORWARD_GYRO): # move_forward_gyro(mDiff, colorSensorStop, colorSensorFollow, gyro) # current_action = decoder.get_next_action() # if(current_action == FORWARD): # move_forward_dual(mDiff, colorSensorStop, colorSensorLeft, colorSensorRight) # current_action = decoder.get_next_action() elif (current_action == CONTINUE): cont_forward(mDiff, colorSensorStop, colorSensorLeft, colorSensorRight) current_action = decoder.get_next_action() elif (current_action == BACKWARD): move_backward(mDiff, colorSensorStop) current_action = decoder.get_next_action() elif (current_action == ONE_EIGHTY): turn_one_eighty(mDiff) current_action = decoder.get_next_action() elif (current_action == LEFT): turn_left(mDiff) current_action = decoder.get_next_action() elif (current_action == RIGHT): turn_right(mDiff) current_action = decoder.get_next_action() else: raise TypeError( "No switch case implemented for action/behaviour: {}".format( current_action))
BATCH_SIZE = 64 dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train)).shuffle(BATCH_SIZE) dataset = dataset.batch(BATCH_SIZE, drop_remainder=True) source_batch, target_batch = next(iter(dataset)) BUFFER_SIZE = len(x_train) steps_per_epoch = BUFFER_SIZE//BATCH_SIZE embedding_dim = 256 units = 300 #creating encoder, attention, decoder layers objects encoder = Encoder(english_vocab, embedding_dim, units, BATCH_SIZE) attention_layer = Attention(10) decoder = Decoder(hindi_vocab, embedding_dim, units, BATCH_SIZE) optimizer = Adam() loss = SparseCategoricalCrossentropy(from_logits=True, reduction='nonTrue') def loss_function(y_true, y_pred): mask = tf.math.logical_not(tf.math.equal(y_true, 0)) loss_ = loss(y_true, y_pred) mask = tf.cast(mask, dtype=loss_.dtype) loss_ = loss_*mask return tf.reduce_mean(loss_) #TRAIN def train(source, target, encoder_hidden): loss = 0 with tf.GradientTape() as tape:
class RNN_LM(RNN): def __init__(self, **kwargs): super().__init__(**kwargs) # Define decoder with tf.variable_scope('decoder'): cell_decoder = rnn_cell(num_units=self.config.num_units, num_cells=self.config.num_cells, keep_prob=self.keep_prob_ph, add_dropout=self.add_dropout) self.decoder = Decoder( cell=cell_decoder, max_iterations=self.config.max_dec_seq_length + 1, infer_token_prob=self.infer_token_prob, alphabet=self.alphabet, use_scheduled_sampling=False, check_seq_lengths=True, parallel_iterations=self.parallel_iterations, swap_memory=self.swap_memory) # Inputs for the one-step-at-a-time decoding self.decoder_inputs = tf.placeholder(tf.int32, shape=[None], name='decoder_inputs') self.decoder_state = tf.placeholder( tf.float32, shape=[None, 2, self.config.num_cells, self.config.num_units], name='decoder_state') def setup_network(self): # Setup character embedding (defines `self.embedding_matrix`) with tf.device('/cpu:0'), tf.variable_scope(name_or_scope='embedding'): self.embedding_matrix = tf.get_variable( shape=[self.config.alphabet_size, self.config.embedding_size], initializer=tf.contrib.layers.xavier_initializer(), name='W') # Gather slices from `params` according to `indices` embedded_decoder_input = tf.nn.embedding_lookup( self.embedding_matrix, self.decoder_input_chars, name='dec_input') def embed_func(input_chars): return tf.gather(self.embedding_matrix, input_chars) # Output projection with tf.variable_scope('alphabet_projection') as scope: self.projection_W, self.projection_b = intialize_projections( input_size=self.config.num_units, output_size=self.config.alphabet_size, scope=scope) # Define alphabet projection function def project_func(output): return projection(output, W=self.projection_W, b=self.projection_b) # Define initial state as zero states self.enc_final_state = self.decoder.cell.zero_state( batch_size=tf.shape(embedded_decoder_input)[0], dtype=tf.float32) # Define decoder with tf.variable_scope('decoder'): dec_outputs, dec_final_state = self.decoder.decode( inputs=embedded_decoder_input, initial_state=self.enc_final_state, seq_length=self.decoder_sequence_length, embed_func=embed_func, project_func=project_func) # Project output to alphabet size and reshape dec_outputs = tf.reshape(dec_outputs, [-1, self.config.num_units]) dec_outputs = projection(dec_outputs, W=self.projection_W, b=self.projection_b) dec_outputs = tf.reshape(dec_outputs, [ -1, self.config.max_dec_seq_length + 1, self.config.alphabet_size ]) # self.packed_dec_final_state = pack_state_tuple(dec_final_state) if self.prediction_mode: dec_outputs = self.decoder_logits # Define loss self.setup_losses(dec_outputs=dec_outputs, target_chars=self.target_chars, decoder_sequence_length=self.decoder_sequence_length) if self.prediction_mode: # Pack state to tensor self.enc_final_state_tensor = pack_state_tuple( self.enc_final_state) # Look up inputs decoder_inputs_embedded = tf.nn.embedding_lookup( self.embedding_matrix, self.decoder_inputs, name='decoder_input') # Unpack state initial_state = unpack_state_tensor(self.decoder_state) with tf.variable_scope('decoder', reuse=True): decoder_output, decoder_final_state = self.decoder.predict( inputs=decoder_inputs_embedded, initial_state=initial_state) # Project output to alphabet size self.decoder_output = projection(decoder_output, W=self.projection_W, b=self.projection_b, name='decoder_output') self.decoder_probs = tf.nn.softmax(self.decoder_output, name='decoder_probs') self.probs_decay_parameter = tf.placeholder( tf.float64, shape=(), name='probs_decay_parameter') self.decoder_probs_decayed = tf.pow( tf.cast(self.decoder_probs, tf.float64), self.probs_decay_parameter) decoder_probs_sum = tf.expand_dims(tf.reduce_sum( self.decoder_probs_decayed, axis=1), axis=1) decoder_probs_sum = tf.tile(decoder_probs_sum, [1, self.config.alphabet_size]) self.decoder_probs_decayed = self.decoder_probs_decayed / decoder_probs_sum # Pack state to tensor self.decoder_final_state = pack_state_tuple( decoder_final_state, name='decoder_final_state') def setup_optimizer(self): # Get trainable variables params = tf.trainable_variables() # Define optimizer algorithm # optimizer = tf.train.AdamOptimizer(self.learning_rate) optimizer = tf.train.MomentumOptimizer( learning_rate=self.learning_rate, momentum=0.99, use_locking=False, name='Momentum', use_nesterov=False) # Get and clip gradients self.gradients = tf.gradients(self.losses, params) clipped_gradients, norm = tf.clip_by_global_norm( t_list=self.gradients, clip_norm=self.config.max_gradient_norm, name='gradient-clipping') if self.create_summary: for variable in params: self.train_summaries.append( tf.summary.histogram(variable.name, variable)) for gradient in self.gradients: if gradient is not None: self.train_summaries.append( tf.summary.histogram(gradient.name, gradient)) # Define update operation self.update_op = optimizer.apply_gradients( grads_and_vars=zip(clipped_gradients, params), global_step=self.global_step) def train_op(self, session, enc_input, dec_input, dec_target, enc_input_length, dec_input_length, is_lm_mode, **kwargs): assert not self.prediction_mode _, mean_loss, mean_prob_x, summary, global_step = session.run( [ self.update_op, self.mean_loss, self.mean_prob_x, self.train_summary, self.global_step ], { self.decoder_input_chars: dec_input, self.target_chars: dec_target, self.decoder_sequence_length: dec_input_length, self.keep_prob_ph: self.config.keep_prob, }) return { 'mean_loss': mean_loss, 'mean_prob_x': mean_prob_x, 'summary': summary, 'global_step': global_step } def val_op(self, session, enc_input, dec_input, dec_target, enc_input_length, dec_input_length, is_lm_mode, **kwargs): assert not self.prediction_mode mean_loss, mean_prob_x, summary, global_step = session.run( [ self.mean_loss, self.mean_prob_x, self.val_summary, self.global_step ], { self.decoder_input_chars: dec_input, self.target_chars: dec_target, self.decoder_sequence_length: dec_input_length, self.keep_prob_ph: 1.0, }) return { 'mean_loss': mean_loss, 'mean_prob_x': mean_prob_x, 'summary': summary, 'global_step': global_step } def predict(self, session, lm_predict_func=None, **kwargs): assert self.prediction_mode def decode_func(inputs, state): output, probs, state = session.run(fetches=[ self.decoder_output, self.decoder_probs, self.decoder_final_state ], feed_dict={ self.decoder_inputs: inputs, self.decoder_state: state }) if lm_predict_func is not None: lm_output, lm_probs, lm_state = lm_predict_func(inputs, state) return {'output': output, 'probs': probs, 'state': state} def loss_func(logits, targets, input_length): return session.run( fetches=[self.mean_loss_batch, self.mean_prob_x_batch], feed_dict={ self.decoder_logits: logits, self.target_chars: targets, self.decoder_sequence_length: input_length }) # Construct vector of <GO_ID> tokens as initial input batch_size = kwargs['enc_input'].shape[0] dec_target = kwargs['dec_target'] dec_input_length = kwargs['dec_input_length'] initial_inputs = np.full(shape=(batch_size, ), fill_value=self.alphabet.GO_ID, dtype=np.float32) max_iterations = self.config.max_dec_seq_length + 1 # Define initial state initial_state = self.decoder.cell.zero_state(batch_size, dtype=tf.float32) initial_state = pack_state_tuple(initial_state) initial_state = session.run(initial_state) extra_features = {} # Initialize predictor if self.sample_type == 'beam': predictor = BeamSearchPredictor(batch_size=batch_size, max_length=max_iterations, alphabet=self.alphabet, decode_func=decode_func, loss_func=loss_func, beam_size=self.beam_size) elif self.sample_type == 'sample': predictor = SamplingPredictor( batch_size=batch_size, max_length=max_iterations, alphabet=self.alphabet, decode_func=decode_func, loss_func=loss_func, num_samples=self.beam_size, ) else: raise KeyError('Invalid sample_type provided!') # Predict sequence candidates final_candidates, final_logits, loss_candidates, prob_x_candidates = predictor.predict_sequences( initial_state=initial_state, target=dec_target, input_length=dec_input_length, features=extra_features) # Remove predictions after the `<EOS>` id for i, j, k in zip(*np.where( final_candidates == self.alphabet.EOS_ID)): final_candidates[i, j, k + 1:] = 0 return { 'candidates': final_candidates, 'loss_candidates': loss_candidates, 'prob_x_candidates': prob_x_candidates } def compute_probabilities(self, session, inputs, state): assert self.prediction_mode output, probs, state = session.run(fetches=[ self.decoder_output, self.decoder_probs, self.decoder_final_state ], feed_dict={ self.decoder_inputs: inputs, self.decoder_state: state }) return {'output': output, 'probs': probs, 'state': state}
default=-1, help='gpu id, set to -1 if use cpu mode') parser.add_argument('--inputpath') parser.add_argument('--outputpath') parser.add_argument('--modeltype', choices=[ 'bilstm-crf', 'bilstm-partial-crf', 'bert-finetune', 'bert-featurebased' ]) args = parser.parse_args() if args.gpu >= 0: use_cuda = True torch.cuda.set_device(args.gpu) else: use_cuda = False decoder = Decoder(save_model_path=args.save_model_path, batch_size=args.batch_size, if_decode_on_gpu=use_cuda, data_path=args.save_modelinfo_path, model_type=args.modeltype) start_time = time.time() print("Decode Input:", args.inputpath) print("Decode Output:", args.outputpath) print("Using GPU:", args.gpu) decoder.decode_file_writeoutput(args.inputpath, args.outputpath) end_time = time.time() print("Decode complete, time: %.2fs" % (end_time - start_time))