def __init__(self,
truncate_times=10,
rnn_size=32,
rnn_layer=2,
share_vocab=True,
src_vocab_size=5000,
tgt_vocab_size=5000,
src_embed_size=128,
tgt_embed_size=128,
use_attention=False,
inference=False,
learning_rate=1e-3,
batch_size=64,
epochs=100,
model_path='MODEL',
log_dir='log_train'):
"""
:param rnn_size: # rnn 维度，即 num_units
:param rnn_layer: # rnn 网络的层数
:param share_vocab:
:param src_vocab_size:
:param tgt_vocab_size:
:param src_embed_size:
:param tgt_embed_size:
:param use_attention:
:param inference:
:param learning_rate:
:param batch_size:
:param epochs:
"""
self.truncate_times = truncate_times
self.rnn_size = rnn_size
self.rnn_layer = rnn_layer
self.share_vocab = share_vocab
self.src_vocab_size = src_vocab_size
self.tgt_vocab_size = tgt_vocab_size
self.src_embed_size = src_embed_size
self.tgt_embed_size = tgt_embed_size
self.use_attention = use_attention
self.inference = inference
self.learning_rate = learning_rate
self.batch_size = batch_size
self.epochs = epochs
self.model_path = model_path
self.logger = Logger(log_file_name=log_dir,
log_level=logging.DEBUG,
logger_name='train').get_log()
self.logger.info('### 建立网络模型……')
self._build_placeholder()
self._build_embedding()
self._build_encoder()
self._build_decoder()
self._build_loss()
from webserver.jerkWebServer import JerkWebServer
from utils.logs import Logger
logger = Logger()
logger.changeLogLevel(7) # 7 is default loglevel
obj = JerkWebServer(logger)
f = open('../examples/_example-001__TEST.json', 'r')
jblock = f.read()
f.close()
obj.parseJSON(jblock)
obj.run()
from utils.cache import Redis
from utils.logs import Logger
logger = Logger().get_logger()
class PopularProductRecommendation:
def __init__(self):
self.name = "recommendaion:popular_product"
def response(self, event, context, **resp):
logger.debug(f'Event: {event}')
resp_body = {'request_id': context.aws_request_id}
product_id = event.get('queryStringParameters', {}).get('product_id')
redis = Redis()
connection = redis.connect()
data = redis.getfrom_hash(connection, self.name, product_id)
if data:
status_code = 200
resp_body.update({
'message': data,
})
else:
status_code = 401
resp_body.update({
'message': "{}"
})
resp['status_code'] = status_code
resp['body'] = resp_body
return resp
def main(config):
if config.channels == 1:
mean = [0.467]
std = [0.271]
elif config.channels == 3:
mean = [0.467, 0.467, 0.467]
std = [0.271, 0.271, 0.271]
if config.device == -1:
device = torch.device('cpu')
else:
device = torch.device('cuda:{:d}'.format(config.device))
torch.backends.cudnn.enabled = True
torch.backends.cudnn.benchmark = True
random.seed(42)
np.random.seed(42)
torch.manual_seed(42)
torch.cuda.manual_seed(42)
train_tfms = Compose([
ShiftScaleRotate(rotate_limit=15, interpolation=cv2.INTER_CUBIC),
GaussianBlur(),
GaussNoise(),
HorizontalFlip(),
RandomBrightnessContrast(),
Normalize(
mean=mean,
std=std,
),
ToTensor()
])
val_tfms = Compose([Normalize(
mean=mean,
std=std,
), ToTensor()])
SAVEPATH = Path(config.root_dir)
#Depending on the stage we either create train/validation or test dataset
if config.stage == 'train':
train_ds = EdsDS(fldr=SAVEPATH / config.train_dir,
channels=config.channels,
transform=train_tfms)
val_ds = EdsDS(fldr=SAVEPATH / config.valid_dir,
channels=config.channels,
transform=val_tfms)
train_loader = DataLoader(train_ds,
batch_size=config.bs,
shuffle=(train_sampler is None),
num_workers=workers,
pin_memory=True,
sampler=train_sampler)
checkpoint = 'logger'
if not os.path.exists(checkpoint):
os.makedirs(checkpoint)
arch = 'segnet_'
title = 'Eye_' + arch + 'fast_fd_g{}_'.format(config.gamma)
logger = Logger(os.path.join(
checkpoint, '{}e{:d}_lr{:.4f}.txt'.format(title, config.ep,
config.lr)),
title=title)
logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Dice'])
elif config.stage == 'test':
val_ds = EdsDS(fldr=SAVEPATH / config.test_dir,
channels=config.channels,
mask=False,
transform=val_tfms)
val_loader = DataLoader(val_ds,
batch_size=config.bs * 2,
shuffle=False,
num_workers=workers,
pin_memory=True)
model = SegNet(channels=config.channels).to(device)
criterion = DiceFocalWithLogitsLoss(gamma=config.gamma).to(device)
optimizer = AdamW(model.parameters(),
lr=start_lr,
betas=(max_mom, 0.999),
weight_decay=wd)
if config.stage == 'train':
steps = len(train_loader) * config.ep
schs = []
schs.append(
SchedulerCosine(optimizer, start_lr, config.lr, lr_mult,
int(steps * warmup_part), max_mom, min_mom))
schs.append(
SchedulerCosine(optimizer, config.lr, finish_lr, lr_mult,
steps - int(steps * warmup_part), min_mom,
max_mom))
lr_scheduler = LR_Scheduler(schs)
max_dice = 0
for epoch in range(config.ep):
print('\nEpoch: [{:d} | {:d}] LR: {:.10f}|{:.10f}'.format(
epoch + 1, config.ep, get_lr(optimizer, -1),
get_lr(optimizer, 0)))
# train for one epoch
train_loss = train(train_loader, model, criterion, optimizer,
lr_scheduler, device, config)
# evaluate on validation set
valid_loss, dice = validate(val_loader, model, criterion, device,
config)
# append logger file
logger.append(
[get_lr(optimizer, -1), train_loss, valid_loss, dice])
if dice > max_dice:
max_dice = dice
model_state = {
'epoch': epoch + 1,
'arch': arch,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
}
save_model = '{}e{:d}_lr_{:.3f}_max_dice.pth.tar'.format(
title, config.ep, config.lr)
torch.save(model_state, save_model)
elif config.stage == 'test':
checkpoint = torch.load(config.saved_model)
model.load_state_dict(checkpoint['state_dict'])
logits = validate(val_loader, model, criterion, device, config)
preds = np.concatenate([torch.argmax(l, 1).numpy()
for l in logits]).astype(np.uint8)
leng = len(preds)
data = {}
data['num_model_params'] = NUM_MODEL_PARAMS
data['number_of_samples'] = leng
data['labels'] = {}
for i in range(leng):
data['labels'][val_ds.img_paths[i].stem] = np_to_base64_utf8_str(
preds[i])
with open(SAVEPATH / '{}.json'.format(config.filename), 'w') as f:
json.dump(data, f)
with zipfile.ZipFile(SAVEPATH / '{}.zip'.format(config.filename),
"w",
compression=zipfile.ZIP_DEFLATED) as zf:
zf.write(SAVEPATH / '{}.json'.format(config.filename))
os.remove(SAVEPATH / '{}.json'.format(config.filename))
from utils.data_helper import input_data
from utils.logs import Logger
import logging
from model import Seq2Seq
from utils.data_helper import special_tokens
if __name__ == '__main__':
data = input_data(special_tokens=special_tokens,
preview=True,
keep_rate=1.0,
share_vocab=False)
_, _, _, src_vocab_table, tgt_vocab_table, src_vocab_table_len, tgt_vocab_table_len = \
data.load_data(src_data_dir='./data/chinese.txt', tgt_data_dir='./data/english.txt')
logger = Logger(log_file_name='./log_inference.txt',
log_level=logging.DEBUG,
logger_name="test").get_log()
# print("Please type a Chinese sentence with one space in each phrase: \n")
# while True:
sentences = [
["我 来自 中国 , 我 是 中国人 ."],
[
"我国 自行 研制 了 功能 强大 的 机群 操作系统 , 配 有 多种 流行 的 高级 程序 语言 , 主流 并行 编程 环境 和 工具 ."
]
]
s = [item[0].split() for item in sentences]
index = data._data_transform_to_index(s, src_vocab_table)
source_input, source_length = data._padding(index, src_vocab_table)
model = Seq2Seq(src_vocab_size=src_vocab_table_len,
tgt_vocab_size=tgt_vocab_table_len,
logger=logger,
# @Email : [email protected] # @File : train.py # package version: # python 3.6 # sklearn 0.20.0 # numpy 1.15.2 # tensorflow 1.5.0 from utils.data_helper import special_tokens from utils.data_helper import input_data from utils.logs import Logger import logging from model import Seq2Seq if __name__ == '__main__': logger = Logger(log_file_name='./log_train.txt', log_level=logging.DEBUG, logger_name="test").get_log() data = input_data(special_tokens=special_tokens, preview=True, keep_rate=1.0, share_vocab=False) source_input, target_input, target_output, src_vocab_table, tgt_vocab_table, src_vocab_table_len, tgt_vocab_table_len = \ data.load_data(src_data_dir='./data/chinese.txt', tgt_data_dir='./data/english.txt') model = Seq2Seq(src_vocab_size=src_vocab_table_len, tgt_vocab_size=tgt_vocab_table_len, logger=logger, model_path='./MODEL', batch_size=32, use_attention=True) model.train(source_input, target_input, target_output, src_vocab_table, tgt_vocab_table, data.gen_batch)