def __init__(self):
################ Model parameter parsing #################
cur_dir = os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe())))
config_dir = os.path.join(cur_dir, 'config')
config = load_config(config_dir)
self.conv_layer_total_num = config['model_definition'][
'conv_layer_total_num']
self.conv_layer_1_filter_height = config['model_definition'][
'conv_layer_1_filter_height']
self.conv_layer_1_filter_width = config['model_definition'][
'conv_layer_1_filter_width']
self.conv_layer_1_neuron_num = config['model_definition'][
'oonv_layer_1_neuron_num']
self.conv_layer_2_1_filter_height = config['model_definition'][
'conv_layer_2_1_filter_height']
self.conv_layer_2_1_filter_width = config['model_definition'][
'conv_layer_2_1_filter_width']
self.conv_layer_2_1_neuron_num = config['model_definition'][
'conv_layer_2_1_neuron_num']
self.conv_layer_2_2_filter_height = config['model_definition'][
'conv_layer_2_2_filter_height']
self.conv_layer_2_2_filter_width = config['model_definition'][
'conv_layer_2_2_filter_width']
self.conv_layer_2_2_neuron_num = config['model_definition'][
'conv_layer_2_2_neuron_num']
self.conv_layer_3_1_filter_height = config['model_definition'][
'conv_layer_3_1_filter_height']
self.conv_layer_3_1_filter_width = config['model_definition'][
'conv_layer_3_1_filter_width']
self.conv_layer_3_1_neuron_num = config['model_definition'][
'conv_layer_3_1_neuron_num']
self.conv_layer_3_2_filter_height = config['model_definition'][
'conv_layer_3_2_filter_height']
self.conv_layer_3_2_filter_width = config['model_definition'][
'conv_layer_3_2_filter_width']
self.conv_layer_3_2_neuron_num = config['model_definition'][
'conv_layer_3_2_neuron_num']
self.dense_layer_1_neuron_num = config['model_definition'][
'dense_layer_1_neuron_num']
self.dense_layer_2_neuron_num = config['model_definition'][
'dense_layer_2_neuron_num']
def run_main():
print("Start to process......")
params = load_config("./config.conf")
params = namedtuple("params", params.keys())(**params)
# load data
ld = LoadData(params)
df_train, df_test, train_X, train_y, \
test_X, test_ids, cat_features_indices = ld.process()
# folds
# StratifiedKFold分层采样，用于交叉验证。根据标签中不同类别的占比来拆分数据。
# 将数据及划分成n_splits个互斥的数据集。
folds = list(
StratifiedKFold(n_splits=params.num_splits,
shuffle=True,
random_state=params.random_seed).split(
train_X, train_y))
# Deep - FM
train_dfm_y, test_dfm_y = run_base_model_dfm(df_train, df_test, folds,
params)
return train_dfm_y, test_dfm_y
if not args.weights:
print('you must specify either --config or --weights')
sys.exit()
# f'{config.version}_f{args.fold}_e{epoch:02d}_{score:.04f}.pth')
m = re.match(r'(.*)_f(\d)_e(\d+)_([.0-9]+)\.pth', os.path.basename(args.weights))
if not m:
print('could not parse model name', os.path.basename(args.weights))
assert False
args.config = f'config/{m.group(1)}.yml'
args.fold = int(m.group(2))
print(f'detected config={args.config} fold={args.fold}')
config = load_config(args.config, args.fold)
if args.num_epochs:
config.train.num_epochs = args.num_epochs
if args.num_ttas:
config.test.num_ttas = args.num_ttas
# config.test.batch_size //= args.num_ttas # ideally, I'd like to use big batches
if not os.path.exists(config.experiment_dir):
os.makedirs(config.experiment_dir)
threshold_files = {os.path.basename(path): path for path in glob(THRESHOLDS_PATH + '*.yml')}
assert len(threshold_files)
log_filename = 'log_predict.txt' if args.predict_oof or args.predict_test \
parser.add_argument('path', help='models path', type=str)
args = parser.parse_args()
files = glob(os.path.join(args.path, '*.pth'))
files.sort(key=lambda path: parse_model_name(path)[2])
files = list(filter(lambda model: get_score(model) >= 0.59, files))
print(np.array(files))
assert files
if len(files) < 2:
print('nothing to do here')
sys.exit()
model_name, fold, _, __ = parse_model_name(files[0])
print(f'model {model_name}, fold {fold}')
config = load_config(f'config/{model_name}.yml', 0)
avg_model = create_model(config, pretrained=False)
cur_model = create_model(config, pretrained=False)
data_loader = load_data(fold)
current_best_file = None
best_score = get_best_score(files)
dprint(best_score)
for weight in [0.3, 0.4, 0.5]:
print('-' * 80)
score = apply_swa(files, weight)
if score > best_score:
best_score = score
# py model_train.py
# The script will automatically download the MNIST data set and start training.
# The checkpoint will be store in "project_direcotry/model/final"
#########################################
import tensorflow as tf
import os
import sys
from tensorflow.examples.tutorials.mnist import input_data
from model_def import model_def
sys.path.append(os.path.join('config'))
from parse_config import load_config
################ Parse the config to set up the parameters #################
config = load_config(os.path.join('config'))
image_height = config['model_definition']['image_height']
image_width = config['model_definition']['image_width']
label_dimention = config['model_definition']['label_dimention']
################ Import the model ###################
x = tf.placeholder(tf.float32, shape=[None, image_height * image_width])
y_ = tf.placeholder(tf.float32, shape=[None, label_dimention])
drop_out_prob = tf.placeholder(
tf.float32) # drop_out_prob = 1.0 means there will be NO drop out
model_definition = model_def()
model = model_definition.model_init(data=x,
image_height=image_height,
image_width=image_width,
label_dimention=label_dimention,
drop_out_prob=drop_out_prob)
