def __init__(self, **kwargs):
super().__init__(**kwargs)
# Define decoder
with tf.variable_scope('decoder'):
cell_decoder = rnn_cell(num_units=self.config.num_units,
num_cells=self.config.num_cells,
keep_prob=self.keep_prob_ph,
add_dropout=self.add_dropout)
self.decoder = Decoder(
cell=cell_decoder,
max_iterations=self.config.max_dec_seq_length + 1,
infer_token_prob=self.infer_token_prob,
alphabet=self.alphabet,
use_scheduled_sampling=False,
check_seq_lengths=True,
parallel_iterations=self.parallel_iterations,
swap_memory=self.swap_memory)
# Inputs for the one-step-at-a-time decoding
self.decoder_inputs = tf.placeholder(tf.int32,
shape=[None],
name='decoder_inputs')
self.decoder_state = tf.placeholder(
tf.float32,
shape=[None, 2, self.config.num_cells, self.config.num_units],
name='decoder_state')
def decodefunc(args):
message = args.message
key = args.key
levels = args.levels
pattern = args.pattern
shift = args.caesar
show_all = args.show_all
decoder = Decoder()
decoders = [
"base16", "base32", "base64", "base85", "autokey", "atbash",
"baconian", "beaufort", "caesar", "morse", "rot13", "rot47",
"playfair", "vigenere"
]
combdecs = list(product(decoders, repeat=levels))
for arraydecs in combdecs:
translated = message
flow = ""
for dec in arraydecs:
translated = decoder.decode(dec, translated, shift, key)
if translated is None:
if show_all is not None and show_all:
print(Fore.RED + " > ".join(arraydecs))
break
if translated is not None:
if pattern is None or (pattern and pattern in translated):
print(Fore.WHITE + " > ".join(arraydecs) + ": " + Fore.BLUE +
translated)
def bit_error_probability(p, matrix):
decoder = Decoder(p, matrix)
diff = 0
k = len(matrix)
for i in range(N):
sequences = generate_sequences(k)
for sequence in sequences:
codeword = np.mod(np.sum(np.multiply(matrix, sequence), axis=0), 2)
received = np_to_str(spoil_codeword(codeword, p))
diff += difference(sequence.tolist(),
[[int(j)] for j in decoder.decode(received)])
return diff / float((N * k * (2**k)))
def __init__(self):
self.config = Config()
self.cookies = login()
logger.info(self.cookies)
self.base_path = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))),
self.config.config['output']['path'])
self.live_infos = Infos()
self.display = Display()
self.decoder = Decoder()
self.uploader = Upload()
self.threadRecorder = threadRecorder()
logger.info('基路径:%s' % (self.base_path))
self.load_room_info()
self.get_live_url()
logger.info('初始化完成')
def decodefunc(message, key, levels, pattern):
decoder = Decoder()
decoders = [
"base16", "base32", "base64", "base85", "morse", "rot13", "rot47",
"vigenere"
]
combdecs = list(product(decoders, repeat=levels))
for arraydecs in combdecs:
translated = message
flow = ""
for dec in arraydecs:
flow = flow + " " + dec + " >"
translated = decoder.decode(dec, translated, key)
if translated is None:
break
if translated is not None:
if pattern is None or (pattern and pattern in translated):
print(Fore.WHITE + flow + " " + Fore.BLUE + translated)
def __init__(self,
input_shape,
latent_dim,
filter_factor,
activation='relu',
hierarchical=False,
vampprior=False,
expected_value=False,
hidden_dim=300,
data_set='mnist'):
super(Vae, self).__init__()
self.latent_dim = latent_dim
self.vampprior = vampprior
self.expected_value = expected_value
self.hierarchical = hierarchical
print(f'vampprior: {vampprior}\n'
f'expected_value: {expected_value}\n'
f'hierarchical: {hierarchical}')
self.encoder = Encoder(
input_shape=input_shape,
latent_dim=latent_dim if not hierarchical else int(latent_dim / 2),
activation=activation,
factor=filter_factor,
name='encoder_z')
if hierarchical:
self.encoder_z0 = Encoder(input_shape=input_shape,
latent_dim=int(latent_dim / 2),
activation=activation,
factor=filter_factor,
conditional_on_other_z=True,
name='encoder_z0')
self.connect_z0 = tf.keras.Sequential([
tf.keras.layers.InputLayer(input_shape=int(latent_dim / 2)),
tf.keras.layers.Dense(hidden_dim, activation=activation),
tf.keras.layers.Dense(hidden_dim, activation=activation)
])
self.p_z0 = DensityLayer(int(latent_dim / 2))
self.decoder = Decoder(input_shape=input_shape,
latent_dim=latent_dim,
activation=activation,
factor=filter_factor)
if self.vampprior:
self.batch_size_u = 500
self.pseudo_inputs_layer = TrainablePseudoInputs(
self.batch_size_u, data_set)
class Live():
def __init__(self):
self.config = Config()
self.cookies = login()
logger.info(self.cookies)
self.base_path = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))),
self.config.config['output']['path'])
self.live_infos = Infos()
self.display = Display()
self.decoder = Decoder()
self.uploader = Upload()
self.threadRecorder = threadRecorder()
logger.info('基路径:%s' % (self.base_path))
self.load_room_info()
self.get_live_url()
logger.info('初始化完成')
def create_duration(self, start_time, end_time):
t = datetime.datetime.now()
tt = t.strftime('%Y%m%d %H%M%S')
if start_time == end_time == '0':
return '0'
tmp = datetime.datetime.strptime(tt.split(' ')[0] + ' %s' % start_time, '%Y%m%d %H%M%S')
if t > tmp:
base_time1 = tt.split(' ')[0]
base_time2 = (t + datetime.timedelta(days=1)).strftime('%Y%m%d %H%M%S').split(' ')[0]
else:
base_time1 = (t - datetime.timedelta(days=1)).strftime('%Y%m%d %H%M%S').split(' ')[0]
base_time2 = tt.split(' ')[0]
if start_time > end_time:
start_time = '%s %s' % (base_time1, start_time)
end_time = '%s %s' % (base_time2, end_time)
else:
start_time = '%s %s' % (tt.split(' ')[0], start_time)
end_time = '%s %s' % (tt.split(' ')[0], end_time)
return '%s-%s' % (start_time, end_time)
def check_live(self, key):
duration = self.live_infos.get(key)['duration']
if duration == '0':
return True
lst = duration.split('-')
now_time = datetime.datetime.now()
if len(lst) == 2:
start_time = datetime.datetime.strptime(lst[0], '%Y%m%d %H%M%S')
end_time = datetime.datetime.strptime(lst[1], '%Y%m%d %H%M%S')
if now_time > start_time and now_time < end_time:
return True
else:
logger.debug('%s[RoomID:%s]不在直播时间段' % (self.live_infos.get(key)['uname'], key))
return False
else:
return False
def load_room_info(self):
live_infos = self.live_infos.copy()
for lst in self.config.config['live']['room_info']:
if lst[0] not in live_infos:
live_info = {}
live_info['record_start_time'] = ''
live_info['queue_status'] = 0
live_info['recording'] = 0
live_info['finish_time'] = ''
else:
live_info = live_infos[lst[0]]
live_info['need_rec'] = lst[1]
live_info['need_mask'] = lst[2]
live_info['maxsecond'] = lst[3]
live_info['need_upload'] = lst[4]
live_info['duration'] = self.create_duration(lst[5], lst[6])
live_info['cookies'] = self.cookies
live_info['base_path'] = self.base_path
self.live_infos.update(lst[0],live_info)
def load_realtime(self):
'''
实时加载配置，更新房间信息
'''
self.config.load_cfg()
# logger.info(self.config.config)
room_lst = [i[0] for i in self.config.config['live']['room_info']]
del_lst = []
for key in self.live_infos.copy():
if key not in room_lst:
del_lst.append(key)
for key in del_lst:
self.live_infos.delete(key)
self.load_room_info()
def judge_in(self, key):
room_lst = [i[0] for i in self.config.config['live']['room_info']]
if key not in room_lst:
return False
return True
def judge_download(self,key):
if not self.judge_in(key):
return False
live_info = self.live_infos.copy()[key]
if live_info['live_status'] != 1:
live_info['recording'] =0
self.live_infos.update(key,live_info)
return False
elif not self.check_live(key) and live_info['live_status'] == 1:
live_info['recording'] =2
self.live_infos.update(key,live_info)
return False
elif self.check_live(key) and live_info['live_status'] == 1 and live_info['need_rec'] == '0':
live_info['recording']=3
self.live_infos.update(key,live_info)
return False
elif live_info['live_status'] == 1 and live_info['need_rec'] == '1':
live_info['recording']=1
self.live_infos.update(key,live_info)
return True
else:
logger.warning('%s[RoomID:%s]进入了未知的分支呢' % (live_info['uname'],key))
live_info['recording'] =0
self.live_infos.update(key,live_info)
return False
def get_live_url(self):
'''
获取所有监听直播间的信息
'''
room_lst = [i[0] for i in self.config.config['live']['room_info']]
for id in room_lst:
info = None
while info is None:
try:
info = live.get_room_info(id, cookies=self.cookies)
time.sleep(0.3)
except:
logger.error('[RoomID:%s]获取信息失败，重新尝试' % (id))
continue
live_info = self.live_infos.copy()[id]
live_info['room_id'] = id
live_info['real_id'] = info['room_info']['room_id']
try:
if live_info['live_status'] != 1 and info['room_info']['live_status'] == 1:
logger.info('%s[RoomID:%s]开播了' % (live_info['uname'], id))
toaster = ToastNotifier()
toaster.show_toast("开播通知",
'%s[RoomID:%s]开播了' % (live_info['uname'], id),
icon_path=None,
duration=3)
except:
pass
try:
live_info['live_status'] = info['room_info']['live_status']
live_info['uid'] = info['room_info']['uid']
live_info['uname'] = info['anchor_info']['base_info']['uname']
live_info['save_name'] = '%s_%s.flv' % (
live_info['uname'], time.strftime("%Y%m%d%H%M%S", time.localtime()))
live_info['title'] = info['room_info']['title']
live_info['live_start_time'] = info['room_info']['live_start_time']
self.live_infos.update(id,live_info)
logger.debug(
'%s[RoomID:%s]直播状态\t%s' % (live_info['uname'], id, live_info['live_status']))
except Exception as e:
logger.critical(e)
logger.error('[RoomID:%s]房间信息更新失败' % (id))
logger.error(info)
# logger.info(self.live_infos.copy())
def get_stream(self, key):
'''
获取直播流
:param key: 房间显示id
:return: stream
'''
if not self.judge_in(key):
return None
live_info = self.live_infos.copy()[key]
logger.info('%s[RoomID:%s]获取直播流' % (live_info['uname'], key))
session = streamlink.Streamlink()
session.set_option("http-cookies", self.cookies)
session.set_option("http-headers", headers)
log_path = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), 'log', 'stream.log')
session.set_loglevel("debug")
session.set_logoutput(open(log_path, 'a',encoding='utf-8'))
streams = None
while streams is None:
try:
streams = session.streams('https://live.bilibili.com/%s' % key)
except:
logger.warning('%s[RoomID:%s]获取直播流失败，正在重试' % (live_info['uname'], key))
time.sleep(1)
if streams == {}:
logger.error('%s[RoomID:%s]未获取到直播流，可能是下播或者网络问题' % (live_info['uname'], key))
return None
if 'best' in streams:
logger.info('%s[RoomID:%s]获取到best直播流' % (live_info['uname'], key))
return streams['best']
elif 'source' in streams:
logger.info('%s[RoomID:%s]获取到source直播流' % (live_info['uname'], key))
return streams['source']
elif 'worst' in streams:
logger.info('%s[RoomID:%s]获取到worst直播流' % (live_info['uname'], key))
return streams['worst']
else:
logger.info('%s[RoomID:%s]未获取到直播流' % (live_info['uname'], key))
return None
def unlive(self, key, unlived):
if not self.judge_in(key):
return None
live_info = self.live_infos.copy()[key]
logger.info('%s[RoomID:%s]似乎下播了' % (live_info['uname'], key))
live_info['recording'] = 0
logger.info('%s[RoomID:%s]录制结束，录制了%.2f分钟' % (live_info['uname'], key, (
datetime.datetime.now() - datetime.datetime.strptime(
live_info['record_start_time'],
'%Y-%m-%d %H:%M:%S')).total_seconds() / 60.0))
live_info['record_start_time'] = ''
if unlived:
logger.info('%s[RoomID:%s]确认下播，加入转码上传队列' % (live_info['uname'], key))
# if live_info['need_upload'] == '1':
# live_info['filename'] = live_info['uname'] + live_info['duration'].split('-')[0].split(' ')[0]
# live_info['filepath'] = os.path.join(self.base_path, live_info['uname'], '%s_%s' % (live_info['uname'], live_info['duration'].split('-')[0].split(' ')[0]))
# if live_info['need_mask'] == '1':
# live_info['filepath'] += '_mask.mp4'
# else:
# live_info['filepath'] += '.mp4'
self.decoder.enqueue(key)
self.live_infos.update(key,live_info)
def download_live(self, key):
if not self.judge_in(key):
return None
save_path = os.path.join(self.base_path, self.live_infos.get(key)['uname'], 'recording')
logger.info('%s[RoomID:%s]准备下载直播流,保存在%s' % (self.live_infos.get(key)['uname'], key, save_path))
self.live_infos.get(key)['recording'] = 1
if not os.path.exists(save_path):
os.makedirs(save_path)
stream = self.get_stream(key)
if stream is None:
logger.error('%s[RoomID:%s]获取直播流失败' % (self.live_infos.get(key)['uname'], key))
self.live_infos.get(key)['record_start_time'] = ''
self.live_infos.get(key)['recording'] = 0
return
filename = os.path.join(save_path, self.live_infos.get(key)['save_name'])
self.live_infos.get(key)['record_start_time'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
try:
fd = stream.open()
except Exception as e:
self.unlive(key,unlived=False)
logger.critical('%s[RoomID:%s]fd open error' % (self.live_infos.get(key)['uname'], key))
logger.error(e)
return
with open(filename, 'wb') as f:
while self.judge_in(key) and self.live_infos.get(key)['live_status'] == 1 and self.live_infos.get(key)[
'need_rec'] == '1' and self.check_live(key):
try:
data = fd.read(1024 * 8)
if len(data) > 0:
f.write(data)
else:
fd.close()
logger.warning('%s[RoomID:%s]直播流断开,尝试重连' % (self.live_infos.get(key)['uname'], key))
stream = self.get_stream(key)
if stream is None:
logger.warning('%s[RoomID:%s]重连失败' % (self.live_infos.get(key)['uname'], key))
self.unlive(key, True)
return
else:
logger.info('%s[RoomID:%s]重连成功' % (self.live_infos.get(key)['uname'], key))
fd = stream.open()
except Exception as e:
fd.close()
self.unlive(key,unlived=False)
logger.critical('%s[RoomID:%s]遇到了什么问题' % (self.live_infos.get(key)['uname'], key))
logger.error(e)
return
fd.close()
self.unlive(key, True)
def run(self):
while True:
time.sleep(1)
self.load_realtime()
self.get_live_url()
live_infos = self.live_infos.copy()
for key in live_infos:
if live_infos[key]['recording'] != 1 and self.judge_download(key):
self.threadRecorder.add('download_live_%s' % (key),self.download_live,[key,],False)
time.sleep(0.2)
def start(self):
self.threadRecorder.add('display_run',self.display.run,None,False)
self.threadRecorder.add('decoder_run',self.decoder.run,None,False)
self.threadRecorder.add('uploader_run',self.uploader.run,None,False)
self.threadRecorder.add('live_run',self.run,None,False)
# -*- coding: utf-8 -*-
"""NewsCartSendgridAPI SendgridService
This module contains helper classes and methods for
interacting with the Sendgrid API
Documentation can be found here:
https://sendgrid.com/docs/Integrate/Code_Examples/v2_Mail/python.html
Code Samples found here:
https://github.com/sendgrid/sendgrid-python/blob/master/examples/helpers/mail/mail_example.py
"""
from utils.decoder import Decoder
from sendgrid.helpers.mail import Mail, Personalization, Email, Content
DECODER = Decoder()
class TemplateBuilder(object):
"""
Helper class to build Sendgrid compatible Mailer Template
"""
def __init__(self, config):
if config is None:
raise ValueError('Missing config for SendgridService Constructor')
self.template_config = config
self.template = None
def build_template(self):
"""
Builds mailer template based on given config
"""
class RNN_BLSTM(RNN):
def __init__(self, **kwargs):
super().__init__(**kwargs)
# Define encoder
with tf.variable_scope('encoder'):
cell_encoder = rnn_cell(num_units=self.config.num_units,
num_cells=self.config.num_cells,
keep_prob=self.keep_prob_ph,
add_dropout=self.add_dropout)
self.encoder = BidirectionalEncoder(
cell=cell_encoder,
parallel_iterations=self.parallel_iterations,
swap_memory=self.swap_memory)
# Define decoder
with tf.variable_scope('decoder'):
cell_decoder = rnn_cell(
num_units=2 *
self.config.num_units, # Because we use bidirectional encoder
num_cells=self.config.num_cells,
keep_prob=self.keep_prob_ph,
add_dropout=self.add_dropout)
self.decoder = Decoder(
cell=cell_decoder,
max_iterations=self.config.max_dec_seq_length + 1,
infer_token_prob=self.infer_token_prob,
use_scheduled_sampling=self.config.scheduled_sampling,
alphabet=self.alphabet,
parallel_iterations=self.parallel_iterations,
swap_memory=self.swap_memory)
# Inputs for the one-step-at-a-time decoding
self.decoder_inputs = tf.placeholder(tf.int32,
shape=[None],
name='decoder_inputs')
self.decoder_state = tf.placeholder(
tf.float32,
shape=[None, 2, self.config.num_cells, 2 * self.config.num_units],
name='decoder_state')
def setup_network(self):
# Setup character embedding
embedded_encoder_input, embedded_decoder_input, embed_func = self.setup_character_embedding(
)
# Output projection
with tf.variable_scope('alphabet_projection') as scope:
self.projection_W, self.projection_b = intialize_projections(
input_size=2 *
self.config.num_units, # Because we use bidirectional encoder
output_size=self.config.alphabet_size,
scope=scope)
# Define alphabet projection function
def project_func(output):
return projection(output,
W=self.projection_W,
b=self.projection_b)
# Encoder
with tf.variable_scope('encoder') as scope:
enc_outputs, enc_final_state = self.encoder.encode(
inputs=embedded_encoder_input,
seq_lengths=self.encoder_sequence_length,
scope=scope)
# Set decoder initial state and encoder outputs based on the binary
# mode input value
# - If `self.is_lm_mode=0` Use the passed initial state from encoder
# - If `self.is_lm_mode=1` Use the zero vector
enc_outputs, self.enc_final_state = select_decoder_inputs(
is_lm_mode=self.is_lm_mode,
enc_outputs=enc_outputs,
initial_state=enc_final_state)
# Pack state to tensor
self.enc_final_state_tensor = pack_state_tuple(self.enc_final_state)
# Define decoder
with tf.variable_scope('decoder'):
dec_outputs, dec_final_state = self.decoder.decode(
inputs=embedded_decoder_input,
initial_state=self.enc_final_state,
seq_length=self.decoder_sequence_length,
embed_func=embed_func,
project_func=project_func)
# Project output to alphabet size and reshape
dec_outputs = tf.reshape(dec_outputs,
[-1, 2 * self.config.num_units])
dec_outputs = projection(dec_outputs,
W=self.projection_W,
b=self.projection_b)
dec_outputs = tf.reshape(dec_outputs, [
-1, self.config.max_dec_seq_length + 1,
self.config.alphabet_size
])
if self.prediction_mode:
dec_outputs = self.decoder_logits
# Define loss
self.setup_losses(dec_outputs=dec_outputs,
target_chars=self.target_chars,
decoder_sequence_length=self.decoder_sequence_length)
if self.prediction_mode:
# Look up inputs
decoder_inputs_embedded = tf.nn.embedding_lookup(
self.embedding_matrix,
self.decoder_inputs,
name='decoder_input')
is_lm_mode_tensor = tf.to_float(
tf.expand_dims(self.is_lm_mode, axis=1))
decoder_inputs = tf.concat(
[decoder_inputs_embedded, is_lm_mode_tensor], axis=1)
# Unpack state
initial_state = unpack_state_tensor(self.decoder_state)
with tf.variable_scope('decoder', reuse=True):
decoder_output, decoder_final_state = self.decoder.predict(
inputs=decoder_inputs, initial_state=initial_state)
# Project output to alphabet size
self.decoder_output = projection(decoder_output,
W=self.projection_W,
b=self.projection_b,
name='decoder_output')
# Compute decayed logits
self.decoder_probs_decayed = compute_decayed_probs(
logits=self.decoder_output,
decay_parameter_ph=self.probs_decay_parameter)
# Pack state to tensor
self.decoder_final_state = pack_state_tuple(
decoder_final_state, name='decoder_final_state')
def main():
# Setup input ports
colorSensorStop = ev3.LightSensor('in1')
colorSensorLeft = ev3.ColorSensor('in2')
colorSensorRight = ev3.ColorSensor('in3')
assert colorSensorStop.connected, "LightSensorStop(ColorSensor) is not connected"
assert colorSensorLeft.connected, "LightSensorLeft(ColorSensor) is not conected"
assert colorSensorRight.connected, "LightSensorRight(ColorSensor) is not conected"
#colorSensorStop.MODE_REFLECT
#colorSensorLeft.raw
#colorSensorRight.raw
# Setup output ports
mDiff = MoveDifferential(OUTPUT_A, OUTPUT_D, EV3Tire, 15 * STUD_MM)
mDiff.left_motor.polarity = "normal"
mDiff.right_motor.polarity = "normal"
# setup gracefully shutdown
def signal_handler(sig, frame):
print('Shutting down gracefully')
mDiff.stop()
exit(0)
signal.signal(signal.SIGINT, signal_handler)
print('Press Ctrl+C to exit')
# setup decoder and define chain of actions
#string_of_actions = "urrruulldddl"
#string_of_actions = "fffblfrffrfrffb" #"ffffrfrfrfblfflffrflfb"
string_of_actions = "llll.uddllu.r.r.r.r.rdr.u.uruulld.r.rlddllu.luulld.rur.d.dull.d.rd.r.r.rdr.u.uruurrd.lul.dulld.rddlllluur.dld.r.r.rdr.u.udlllldllu.r.r.r.r.rdr.u"
#string_of_actions = "lffffrffbtffrffrfrffffffbrflflfffbrflfflfflflfffbtflffrffrflffbrfflfflflffblfrffffbtflfflffblffbrflffffbrflflfffbrflfflfflflffblfrfrffbtfrffrfrffbrflfflfffrffffrffrfrffbrflflffffbrflflfffbtfrfffflfrffrfrffffffbrflflff"
#string_of_actions = "lfffrfflf"
#string_of_actions = "l"
decoder = Decoder(string_of_actions, DEFINED_ACTIONS)
current_action = decoder.get_next_action()
while current_action != NOT_DEFINED: # current action == -1 -> no more actions to execute
if (current_action == FORWARD):
move_forward(mDiff, colorSensorStop, colorSensorLeft,
colorSensorRight)
current_action = decoder.get_next_action()
#if(current_action == FORWARD_GYRO):
# move_forward_gyro(mDiff, colorSensorStop, colorSensorFollow, gyro)
# current_action = decoder.get_next_action()
# if(current_action == FORWARD):
# move_forward_dual(mDiff, colorSensorStop, colorSensorLeft, colorSensorRight)
# current_action = decoder.get_next_action()
elif (current_action == CONTINUE):
cont_forward(mDiff, colorSensorStop, colorSensorLeft,
colorSensorRight)
current_action = decoder.get_next_action()
elif (current_action == BACKWARD):
move_backward(mDiff, colorSensorStop)
current_action = decoder.get_next_action()
elif (current_action == ONE_EIGHTY):
turn_one_eighty(mDiff)
current_action = decoder.get_next_action()
elif (current_action == LEFT):
turn_left(mDiff)
current_action = decoder.get_next_action()
elif (current_action == RIGHT):
turn_right(mDiff)
current_action = decoder.get_next_action()
else:
raise TypeError(
"No switch case implemented for action/behaviour: {}".format(
current_action))
BATCH_SIZE = 64
dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train)).shuffle(BATCH_SIZE)
dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)
source_batch, target_batch = next(iter(dataset))
BUFFER_SIZE = len(x_train)
steps_per_epoch = BUFFER_SIZE//BATCH_SIZE
embedding_dim = 256
units = 300
#creating encoder, attention, decoder layers objects
encoder = Encoder(english_vocab, embedding_dim, units, BATCH_SIZE)
attention_layer = Attention(10)
decoder = Decoder(hindi_vocab, embedding_dim, units, BATCH_SIZE)
optimizer = Adam()
loss = SparseCategoricalCrossentropy(from_logits=True, reduction='nonTrue')
def loss_function(y_true, y_pred):
mask = tf.math.logical_not(tf.math.equal(y_true, 0))
loss_ = loss(y_true, y_pred)
mask = tf.cast(mask, dtype=loss_.dtype)
loss_ = loss_*mask
return tf.reduce_mean(loss_)
#TRAIN
def train(source, target, encoder_hidden):
loss = 0
with tf.GradientTape() as tape:
class RNN_LM(RNN):
def __init__(self, **kwargs):
super().__init__(**kwargs)
# Define decoder
with tf.variable_scope('decoder'):
cell_decoder = rnn_cell(num_units=self.config.num_units,
num_cells=self.config.num_cells,
keep_prob=self.keep_prob_ph,
add_dropout=self.add_dropout)
self.decoder = Decoder(
cell=cell_decoder,
max_iterations=self.config.max_dec_seq_length + 1,
infer_token_prob=self.infer_token_prob,
alphabet=self.alphabet,
use_scheduled_sampling=False,
check_seq_lengths=True,
parallel_iterations=self.parallel_iterations,
swap_memory=self.swap_memory)
# Inputs for the one-step-at-a-time decoding
self.decoder_inputs = tf.placeholder(tf.int32,
shape=[None],
name='decoder_inputs')
self.decoder_state = tf.placeholder(
tf.float32,
shape=[None, 2, self.config.num_cells, self.config.num_units],
name='decoder_state')
def setup_network(self):
# Setup character embedding (defines `self.embedding_matrix`)
with tf.device('/cpu:0'), tf.variable_scope(name_or_scope='embedding'):
self.embedding_matrix = tf.get_variable(
shape=[self.config.alphabet_size, self.config.embedding_size],
initializer=tf.contrib.layers.xavier_initializer(),
name='W')
# Gather slices from `params` according to `indices`
embedded_decoder_input = tf.nn.embedding_lookup(
self.embedding_matrix,
self.decoder_input_chars,
name='dec_input')
def embed_func(input_chars):
return tf.gather(self.embedding_matrix, input_chars)
# Output projection
with tf.variable_scope('alphabet_projection') as scope:
self.projection_W, self.projection_b = intialize_projections(
input_size=self.config.num_units,
output_size=self.config.alphabet_size,
scope=scope)
# Define alphabet projection function
def project_func(output):
return projection(output,
W=self.projection_W,
b=self.projection_b)
# Define initial state as zero states
self.enc_final_state = self.decoder.cell.zero_state(
batch_size=tf.shape(embedded_decoder_input)[0], dtype=tf.float32)
# Define decoder
with tf.variable_scope('decoder'):
dec_outputs, dec_final_state = self.decoder.decode(
inputs=embedded_decoder_input,
initial_state=self.enc_final_state,
seq_length=self.decoder_sequence_length,
embed_func=embed_func,
project_func=project_func)
# Project output to alphabet size and reshape
dec_outputs = tf.reshape(dec_outputs, [-1, self.config.num_units])
dec_outputs = projection(dec_outputs,
W=self.projection_W,
b=self.projection_b)
dec_outputs = tf.reshape(dec_outputs, [
-1, self.config.max_dec_seq_length + 1,
self.config.alphabet_size
])
# self.packed_dec_final_state = pack_state_tuple(dec_final_state)
if self.prediction_mode:
dec_outputs = self.decoder_logits
# Define loss
self.setup_losses(dec_outputs=dec_outputs,
target_chars=self.target_chars,
decoder_sequence_length=self.decoder_sequence_length)
if self.prediction_mode:
# Pack state to tensor
self.enc_final_state_tensor = pack_state_tuple(
self.enc_final_state)
# Look up inputs
decoder_inputs_embedded = tf.nn.embedding_lookup(
self.embedding_matrix,
self.decoder_inputs,
name='decoder_input')
# Unpack state
initial_state = unpack_state_tensor(self.decoder_state)
with tf.variable_scope('decoder', reuse=True):
decoder_output, decoder_final_state = self.decoder.predict(
inputs=decoder_inputs_embedded,
initial_state=initial_state)
# Project output to alphabet size
self.decoder_output = projection(decoder_output,
W=self.projection_W,
b=self.projection_b,
name='decoder_output')
self.decoder_probs = tf.nn.softmax(self.decoder_output,
name='decoder_probs')
self.probs_decay_parameter = tf.placeholder(
tf.float64, shape=(), name='probs_decay_parameter')
self.decoder_probs_decayed = tf.pow(
tf.cast(self.decoder_probs, tf.float64),
self.probs_decay_parameter)
decoder_probs_sum = tf.expand_dims(tf.reduce_sum(
self.decoder_probs_decayed, axis=1),
axis=1)
decoder_probs_sum = tf.tile(decoder_probs_sum,
[1, self.config.alphabet_size])
self.decoder_probs_decayed = self.decoder_probs_decayed / decoder_probs_sum
# Pack state to tensor
self.decoder_final_state = pack_state_tuple(
decoder_final_state, name='decoder_final_state')
def setup_optimizer(self):
# Get trainable variables
params = tf.trainable_variables()
# Define optimizer algorithm
# optimizer = tf.train.AdamOptimizer(self.learning_rate)
optimizer = tf.train.MomentumOptimizer(
learning_rate=self.learning_rate,
momentum=0.99,
use_locking=False,
name='Momentum',
use_nesterov=False)
# Get and clip gradients
self.gradients = tf.gradients(self.losses, params)
clipped_gradients, norm = tf.clip_by_global_norm(
t_list=self.gradients,
clip_norm=self.config.max_gradient_norm,
name='gradient-clipping')
if self.create_summary:
for variable in params:
self.train_summaries.append(
tf.summary.histogram(variable.name, variable))
for gradient in self.gradients:
if gradient is not None:
self.train_summaries.append(
tf.summary.histogram(gradient.name, gradient))
# Define update operation
self.update_op = optimizer.apply_gradients(
grads_and_vars=zip(clipped_gradients, params),
global_step=self.global_step)
def train_op(self, session, enc_input, dec_input, dec_target,
enc_input_length, dec_input_length, is_lm_mode, **kwargs):
assert not self.prediction_mode
_, mean_loss, mean_prob_x, summary, global_step = session.run(
[
self.update_op, self.mean_loss, self.mean_prob_x,
self.train_summary, self.global_step
], {
self.decoder_input_chars: dec_input,
self.target_chars: dec_target,
self.decoder_sequence_length: dec_input_length,
self.keep_prob_ph: self.config.keep_prob,
})
return {
'mean_loss': mean_loss,
'mean_prob_x': mean_prob_x,
'summary': summary,
'global_step': global_step
}
def val_op(self, session, enc_input, dec_input, dec_target,
enc_input_length, dec_input_length, is_lm_mode, **kwargs):
assert not self.prediction_mode
mean_loss, mean_prob_x, summary, global_step = session.run(
[
self.mean_loss, self.mean_prob_x, self.val_summary,
self.global_step
], {
self.decoder_input_chars: dec_input,
self.target_chars: dec_target,
self.decoder_sequence_length: dec_input_length,
self.keep_prob_ph: 1.0,
})
return {
'mean_loss': mean_loss,
'mean_prob_x': mean_prob_x,
'summary': summary,
'global_step': global_step
}
def predict(self, session, lm_predict_func=None, **kwargs):
assert self.prediction_mode
def decode_func(inputs, state):
output, probs, state = session.run(fetches=[
self.decoder_output, self.decoder_probs,
self.decoder_final_state
],
feed_dict={
self.decoder_inputs: inputs,
self.decoder_state: state
})
if lm_predict_func is not None:
lm_output, lm_probs, lm_state = lm_predict_func(inputs, state)
return {'output': output, 'probs': probs, 'state': state}
def loss_func(logits, targets, input_length):
return session.run(
fetches=[self.mean_loss_batch, self.mean_prob_x_batch],
feed_dict={
self.decoder_logits: logits,
self.target_chars: targets,
self.decoder_sequence_length: input_length
})
# Construct vector of <GO_ID> tokens as initial input
batch_size = kwargs['enc_input'].shape[0]
dec_target = kwargs['dec_target']
dec_input_length = kwargs['dec_input_length']
initial_inputs = np.full(shape=(batch_size, ),
fill_value=self.alphabet.GO_ID,
dtype=np.float32)
max_iterations = self.config.max_dec_seq_length + 1
# Define initial state
initial_state = self.decoder.cell.zero_state(batch_size,
dtype=tf.float32)
initial_state = pack_state_tuple(initial_state)
initial_state = session.run(initial_state)
extra_features = {}
# Initialize predictor
if self.sample_type == 'beam':
predictor = BeamSearchPredictor(batch_size=batch_size,
max_length=max_iterations,
alphabet=self.alphabet,
decode_func=decode_func,
loss_func=loss_func,
beam_size=self.beam_size)
elif self.sample_type == 'sample':
predictor = SamplingPredictor(
batch_size=batch_size,
max_length=max_iterations,
alphabet=self.alphabet,
decode_func=decode_func,
loss_func=loss_func,
num_samples=self.beam_size,
)
else:
raise KeyError('Invalid sample_type provided!')
# Predict sequence candidates
final_candidates, final_logits, loss_candidates, prob_x_candidates = predictor.predict_sequences(
initial_state=initial_state,
target=dec_target,
input_length=dec_input_length,
features=extra_features)
# Remove predictions after the `<EOS>` id
for i, j, k in zip(*np.where(
final_candidates == self.alphabet.EOS_ID)):
final_candidates[i, j, k + 1:] = 0
return {
'candidates': final_candidates,
'loss_candidates': loss_candidates,
'prob_x_candidates': prob_x_candidates
}
def compute_probabilities(self, session, inputs, state):
assert self.prediction_mode
output, probs, state = session.run(fetches=[
self.decoder_output, self.decoder_probs, self.decoder_final_state
],
feed_dict={
self.decoder_inputs: inputs,
self.decoder_state: state
})
return {'output': output, 'probs': probs, 'state': state}
default=-1,
help='gpu id, set to -1 if use cpu mode')
parser.add_argument('--inputpath')
parser.add_argument('--outputpath')
parser.add_argument('--modeltype',
choices=[
'bilstm-crf', 'bilstm-partial-crf',
'bert-finetune', 'bert-featurebased'
])
args = parser.parse_args()
if args.gpu >= 0:
use_cuda = True
torch.cuda.set_device(args.gpu)
else:
use_cuda = False
decoder = Decoder(save_model_path=args.save_model_path,
batch_size=args.batch_size,
if_decode_on_gpu=use_cuda,
data_path=args.save_modelinfo_path,
model_type=args.modeltype)
start_time = time.time()
print("Decode Input:", args.inputpath)
print("Decode Output:", args.outputpath)
print("Using GPU:", args.gpu)
decoder.decode_file_writeoutput(args.inputpath, args.outputpath)
end_time = time.time()
print("Decode complete, time: %.2fs" % (end_time - start_time))