import re
from itertools import chain
from collections import Counter
import numpy as np
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import TensorBoard, ModelCheckpoint
sys.path.append('../../..')
from ml import get_model_dir, get_log_dir # noqa
from ml.datasets import get_data_path # noqa
wv_path = get_data_path('tencent_embedding_100k')
poem_path = get_data_path('poem')
model_name = 'rnn'
experiment_name = 'poem'
model_dir = get_model_dir()
log_dir = get_log_dir(model_name, experiment_name)
model_file_name = model_name
if experiment_name:
model_file_name += '_' + experiment_name
model_file_name += '.h5'
model_path = model_dir / model_file_name
NUM_VOCAB = 100000
EMBEDDING_DIM = 200
# import csv
# import os
# import modeling
# import optimization
# import tokenization
import tensorflow as tf
sys.path.append('../../..')
from ml import get_model_dir, get_log_dir # noqa
from ml.datasets import get_data_path # noqa
#################################
# CONFIG
#################################
weibo_path = get_data_path('weibo')
model_name = 'bert'
experiment_name = 'weibo'
model_dir = get_model_dir()
log_dir = get_log_dir(model_name, experiment_name)
model_path = model_dir / f'{model_name}_{experiment_name}.h5'
validation_rate = 0.1
# other param
bert_config_file = None
vocab_file = None
output_dir = None
init_checkpoint = None
max_seq_length = 128
do_train = False
model_file_name = model_name
if experiment_name:
model_file_name += '_' + experiment_name
model_file_name += '.h5'
model_path = model_dir / model_file_name
BATCH_SIZE = 512
EPOCHS = 200
num_classes = 120
##########
# Data
##########
data_path = get_data_path('fruits')
train_path, test_path = data_path / 'Training', data_path / 'Test'
train_gen = ImageDataGenerator(rescale=1 / 255.,
width_shift_range=0.125,
height_shift_range=0.125,
fill_mode='constant',
cval=0.,
horizontal_flip=True,
dtype='float32')
train_data = train_gen.flow_from_directory(train_path,
target_size=(100, 100),
class_mode='categorical',
batch_size=BATCH_SIZE)
test_gen = ImageDataGenerator(rescale=1 / 255., dtype='float32')
import sys
import jieba
import tensorflow as tf
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras import layers
import pickle
from pathlib import Path
from itertools import chain
sys.path.append('../../..')
from ml import get_model_dir, get_log_dir # noqa
from ml.datasets import get_data_path # noqa
data_path = get_data_path('shuihu')
model_name = 'rnn'
experiment_name = 'shuihu'
model_dir = get_model_dir()
log_dir = get_log_dir(model_name, experiment_name)
model_file_name = model_name
if experiment_name:
model_file_name += '_' + experiment_name
model_file_name += '.h5'
model_path = model_dir / model_file_name
NUM_VOCAB = 56131
EMBEDDING_DIM = 256
SEQ_LENGTH = 100
BATCH_SIZE = 64