def confusion_matrix(self, name=None):
predictions = self.model.predict(self.valX)
labels = list(set(get_labels(self.valY)))
print(labels)
target_names = ["N", "P"]
print("Classification report for " + FOLDER + " ---> " +self.model.name)
# print(precision_recall_fscore_support(np.argmax(predictions, axis=1), np.argmax(self.valY, axis=1)))
print("F1 SCORE:")
print(f1_score(np.argmax(self.valY, axis=1), np.argmax(predictions, axis=1)))
print("RECALL:")
print(recall_score(np.argmax(self.valY, axis=1), np.argmax(predictions, axis=1)))
print("PRECISION:")
print(precision_score(np.argmax(self.valY, axis=1), np.argmax(predictions, axis=1)))
print("SPECIFICITY:")
self.fpr, self.tpr, _ = roc_curve(np.argmax(self.valY, axis=1),predictions[:,1])
self.auc = roc_auc_score(np.argmax(self.valY, axis=1), np.argmax(predictions, axis=1))
cm = confusion_matrix(get_labels(self.valY),get_labels(predictions))
tn, fp, fn, tp = confusion_matrix(get_labels(self.valY),get_labels(predictions)).ravel()
print("True Positive {} False Positive {} False Negative {} True Positive {}".format(tn, fp, fn, tp))
print("TN {}".format(cm[0][0]))
print("FP {}".format(cm[0][1]))
print("FN {}".format(cm[1][0]))
print("TP {}".format(cm[1][1]))
specificity = cm[0][0] / (cm[0][0] + cm[0][1])
print(specificity)
print("Confusion Matrix {}\n".format(cm))
plot_confusion_matrix(cm, labels, title=name if not None else self.model.name+FOLDER)
def get_results(self, models):
for model in models:
model.compile(loss='categorical_crossentropy', optimizer=self.optimizer(),metrics=['accuracy'])
aug = ImageDataGenerator(
# rotation_range=90,
# zoom_range=0.15,
# width_shift_range=0.2,
# height_shift_range=0.2,
shear_range=0.25,
horizontal_flip=True,
vertical_flip=True,
fill_mode="nearest"
)
if(K.image_dim_ordering() == 'th'):
self.x = np.moveaxis(self.x, -1, 1)
self.valX = np.moveaxis(self.valX, -1, 1)
if(os.path.exists('../models/'+model.name+FOLDER+'.h5')):
model.load_weights('../models/'+model.name+FOLDER+'.h5')
else:
# self.history = self.model.fit_generator(self.trainGen,
# epochs=self.epochs, verbose=1, shuffle=True,
# validation_data=self.valGen, workers=2, use_multiprocessing=False)
self.history = model.fit_generator(aug.flow(self.trainX,self.trainY, batch_size=self.batch_size, shuffle=True, seed=1000),
steps_per_epoch=len(self.trainX)/self.batch_size ,epochs=self.epochs, verbose=1,
validation_data=(self.valX, self.valY))
predictions = model.predict(self.valX)
labels = list(set(get_labels(self.valY)))
print(labels)
target_names = ["N", "P"]
print("Classification report for " + FOLDER + " ---> " +model.name)
print("\n====================================================================================================================================================")
# print(precision_recall_fscore_support(np.argmax(predictions, axis=1), np.argmax(self.valY, axis=1)))
print("F1 SCORE:")
print(f1_score(np.argmax(self.valY, axis=1), np.argmax(predictions, axis=1)))
print("RECALL:")
print(recall_score(np.argmax(self.valY, axis=1), np.argmax(predictions, axis=1)))
print("PRECISION:")
print(precision_score(np.argmax(self.valY, axis=1), np.argmax(predictions, axis=1)))
print("SPECIFICITY:")
self.fpr, self.tpr, _ = roc_curve(np.argmax(self.valY, axis=1),predictions[:,1])
self.auc = roc_auc_score(np.argmax(self.valY, axis=1), np.argmax(predictions, axis=1))
cm = confusion_matrix(get_labels(self.valY),get_labels(predictions))
tn, fp, fn, tp = confusion_matrix(get_labels(self.valY),get_labels(predictions)).ravel()
print("True Positive {} False Positive {} False Negative {} True Positive {}".format(tn, fp, fn, tp))
print("TN {}".format(cm[0][0]))
print("FP {}".format(cm[0][1]))
print("FN {}".format(cm[1][0]))
print("TP {}".format(cm[1][1]))
specificity = cm[0][0] / (cm[0][0] + cm[0][1])
print(specificity)
print("Confusion Matrix {}\n".format(cm))
plot_confusion_matrix(cm, labels, title=model.name+FOLDER)
def confusion_matrix(self, title=None):
if title == None:
title = self.model.name + FOLDER
predictions = self.model.predict(self.valX)
class_counts = onehot_to_cat(self.valY)
print("Class counts")
print(np.unique(class_counts, return_counts=True))
print("=================================================")
labels = list(set(get_labels(self.valY)))
cm = confusion_matrix(get_labels(self.valY), get_labels(predictions))
print("Confusion Matrix {}".format(cm))
plot_confusion_matrix(cm, labels, title=title)
def srv_get_training_data(self, params):
data_dir = os.path.join(self.data_root, self.ns)
labels = get_labels(data_dir)
counts = [
len(os.listdir(os.path.join(data_dir, label))) for label in labels
]
return GetPredictionsResponse(labels, counts)
def __init__(self, threshold=0.5):
args = utils.get_arguments()
self.weight_path = args.weight_path
self.cfg_path = args.config_path
self.labels = utils.get_labels(args.classes_path)
self.threshold = threshold
# Tải mô hình YOLOv3 Tiny cho phát hiện vùng biển số
self.model = cv2.dnn.readNet(model=self.weight_path,
config=self.cfg_path)
def __init__(self, threshold=0.5):
args = utils.get_arguments()
self.weight_path = args.weight_path
self.cfg_path = args.config_path
self.labels = utils.get_labels(args.classes_path)
self.threshold = threshold
# Load model
self.model = cv2.dnn.readNet(model=self.weight_path,
config=self.cfg_path)
def __init__(self, threshold=0.5):
print("------- initial detectNumberPlate")
try:
self.weight_path = "./weights/yolov3-tiny.weights"
self.cfg_path = "./cfg/yolov3-tiny.cfg"
self.labels = utils.get_labels("./cfg/coco.names")
self.threshold = threshold
print("------- before DetectVehicle.load_model_readNet")
# Load model
self.model = cv2.dnn.readNet(model=self.weight_path,
config=self.cfg_path)
print("------- after DetectVehicle.load_model_readNet")
except Exception as ex:
print("############## Error: {} ##############".format(str(ex)))
def validate():
data_dir = ".\\data\\"
label_names = get_labels(data_dir)
model = load_model('asr_model.h5') # 加载训练模型
features, labels = loadFromPickle()
features, labels = shuffle(features, labels)
features=features.reshape(features.shape[0],20,32,1)
labels=prepress_labels(labels)
train_x, test_x, train_y, test_y = train_test_split(features, labels, random_state=0,
test_size=0.3)
print_summary(model)
# 开始评估模型效果 # verbose=0为不输出日志信息
score = model.evaluate(test_x, test_y, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1]) # 准确度
def predict(self, cv_img=None, path=None):
with self.session.as_default():
with self.graph.as_default():
data_dir = os.path.join(self.data_root, self.ns)
if path is not None:
cv_img = cv2.imread(path)
label_names = get_labels(data_dir)
rs_img_f32 = cv2.resize(
cv_img,
(self.IMAGE_SIZE, self.IMAGE_SIZE)).astype('float32')
input_data = rs_img_f32.reshape(-1, self.IMAGE_SIZE,
self.IMAGE_SIZE, 3)
if self.model is not None:
result = self.model.predict(input_data, steps=1)
print(label_names)
print("result:", result)
return label_names, result[0]
else:
print('your model is not ready')
return None
def __init__(self, threshold=0.5):
print("------- initial detectNumberPlate")
try:
# args = utils.get_arguments()
# print("------- args: ", args)
# self.weight_path = args.weight_path
# self.cfg_path = args.config_path
# self.labels = utils.get_labels(args.classes_path)
# self.threshold = threshold
self.weight_path = "./weights/yolov3-tiny_15000.weights"
self.cfg_path = "./cfg/yolov3-tiny.cfg"
self.labels = utils.get_labels("./cfg/yolo.names")
self.threshold = threshold
print("------- before detectNumberPlate.load_model_readNet")
# Load model
self.model = cv2.dnn.readNet(model=self.weight_path,
config=self.cfg_path)
print("------- after detectNumberPlate.load_model_readNet")
except Exception as ex:
print("############## Error: {} ##############".format(str(ex)))
def main():
parser = argparse.ArgumentParser()
# Required parameters
parser.add_argument(
"--data_dir",
default=None,
type=str,
required=True,
help="输入数据目录。应该包含CoNLL-2003 NER任务的训练文件",
)
parser.add_argument(
"--model_type",
default=None,
type=str,
required=True,
help="列表中选择的模型类型: " + ", ".join(MODEL_CLASSES.keys()),
)
parser.add_argument(
"--model_name_or_path",
default=None,
type=str,
required=True,
help="列表中选择的预训练模型或快捷方式名称的路径: " + ", ".join(ALL_MODELS),
)
parser.add_argument(
"--output_dir",
default=None,
type=str,
required=True,
help="输出目录, 将在其中写入模型预测和checkpoint",
)
# Other parameters
parser.add_argument("--config_name",
default="",
type=str,
help="预训练的配置名称或路径(如果与model_name不同)")
parser.add_argument(
"--tokenizer_name",
default="",
type=str,
help="预训练的tokenizer名称或路径(如果与model_name不同)",
)
parser.add_argument(
"--cache_dir",
default="",
type=str,
help="您想在哪里存储从s3下载的预训练模型",
)
parser.add_argument(
"--max_seq_length",
default=128,
type=int,
help="tokenization后的最大总输入序列长度。长度大于此长度的序列将被截断，较短的序列将被填充。",
)
parser.add_argument("--do_train",
action="store_true",
help="Whether to run training.")
parser.add_argument("--do_eval",
action="store_true",
help="Whether to run eval on the dev set.")
parser.add_argument("--do_predict",
action="store_true",
help="Whether to run predictions on the test set.")
parser.add_argument(
"--evaluate_during_training",
action="store_true",
help="是否在每个日志记录step的训练期间进行评估.",
)
parser.add_argument("--do_lower_case",
action="store_true",
help="如果使用的是uncased的模型，请设置此标志")
parser.add_argument("--per_gpu_train_batch_size",
default=8,
type=int,
help="训练时每个GPU / CPU的批次大小。")
parser.add_argument("--per_gpu_eval_batch_size",
default=8,
type=int,
help="评估时每个GPU / CPU的批次大小。")
parser.add_argument(
"--gradient_accumulation_steps",
type=int,
default=1,
help="在执行向后/更新过程之前要梯度累积的更新步骤数。",
)
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--weight_decay",
default=0.0,
type=float,
help="Weight decay if we apply some.")
parser.add_argument("--adam_epsilon",
default=1e-8,
type=float,
help="Epsilon for Adam optimizer.")
parser.add_argument("--adam_beta1",
default=0.9,
type=float,
help="BETA1 for Adam optimizer.")
parser.add_argument("--adam_beta2",
default=0.999,
type=float,
help="BETA2 for Adam optimizer.")
parser.add_argument("--max_grad_norm",
default=1.0,
type=float,
help="Max gradient norm.")
parser.add_argument("--num_train_epochs",
default=3.0,
type=float,
help="要执行的训练epoch总数。")
parser.add_argument(
"--max_steps",
default=-1,
type=int,
help="If > 0: 设置要执行的训练步骤总数。覆盖num_train_epochs。",
)
parser.add_argument("--warmup_steps",
default=0,
type=int,
help="Linear warmup over warmup_steps.")
parser.add_argument("--logging_steps",
type=int,
default=50,
help="Log every X updates steps.")
parser.add_argument("--save_steps",
type=int,
default=50,
help="Save checkpoint every X updates steps.")
parser.add_argument(
"--eval_all_checkpoints",
action="store_true",
help=
"Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number",
)
parser.add_argument("--no_cuda",
action="store_true",
help="Avoid using CUDA when available")
parser.add_argument("--overwrite_output_dir",
action="store_true",
help="Overwrite the content of the output directory")
parser.add_argument(
"--overwrite_cache",
action="store_true",
help="Overwrite the cached training and evaluation sets")
parser.add_argument("--seed",
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
"--fp16",
action="store_true",
help=
"Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit",
)
parser.add_argument(
"--fp16_opt_level",
type=str,
default="O1",
help=
"For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
"See details at https://nvidia.github.io/apex/amp.html",
)
parser.add_argument("--local_rank",
type=int,
default=-1,
help="For distributed training: local_rank")
parser.add_argument("--server_ip",
type=str,
default="",
help="For distant debugging.")
parser.add_argument("--server_port",
type=str,
default="",
help="For distant debugging.")
# mean teacher
parser.add_argument('--mt',
type=int,
default=0,
help='mean teacher. 是否使用mean teacher')
parser.add_argument('--mt_updatefreq',
type=int,
default=1,
help='mean teacher update frequency')
parser.add_argument(
'--mt_class',
type=str,
default="kl",
help=
'mean teacher class, choices:[smart, prob, logit, kl(default), distill].'
)
parser.add_argument('--mt_lambda',
type=float,
default=1,
help="trade off parameter of the consistent loss.")
parser.add_argument('--mt_rampup',
type=int,
default=300,
help="rampup iteration.")
parser.add_argument(
'--mt_alpha1',
default=0.99,
type=float,
help=
"moving average parameter of mean teacher (for the exponential moving average)."
)
parser.add_argument(
'--mt_alpha2',
default=0.995,
type=float,
help=
"moving average parameter of mean teacher (for the exponential moving average)."
)
parser.add_argument('--mt_beta',
default=10,
type=float,
help="coefficient of mt_loss term.")
parser.add_argument(
'--mt_avg',
default="exponential",
type=str,
help=
"moving average method, choices:[exponentail(default), simple, double_ema]."
)
parser.add_argument(
'--mt_loss_type',
default="logits",
type=str,
help="subject to 衡量模型差异, choices:[embeds, logits(default)].")
# virtual adversarial training
parser.add_argument('--vat',
type=int,
default=0,
help='virtual adversarial training.')
parser.add_argument(
'--vat_eps',
type=float,
default=1e-3,
help='perturbation size for virtual adversarial training.')
parser.add_argument(
'--vat_lambda',
type=float,
default=1,
help='trade off parameter for virtual adversarial training.')
parser.add_argument(
'--vat_beta',
type=float,
default=1,
help='coefficient of the virtual adversarial training loss term.')
parser.add_argument(
'--vat_loss_type',
default="logits",
type=str,
help=
"subject to measure model difference, choices = [embeds, logits(default)]."
)
# self-training
parser.add_argument(
'--self_training_reinit',
type=int,
default=0,
help='如果teacher模型已更新，是否重新初始化student模型。0表示重启重新初始化，1表示不初始化')
parser.add_argument(
'--self_training_begin_step',
type=int,
default=900,
help='开始步骤(通常在第一个epoch之后)开始self-training。对应论文中的第一阶段早停策略')
parser.add_argument(
'--self_training_label_mode',
type=str,
default="hard",
help=
'伪标签类型. choices:[hard(default), soft]. 软标签是一个teacher模型预测出来的，类似logits的概率值，是浮点数，硬标签直接就是整数，就是对应概率最大的位置的索引，例如soft是0.82, hard就是1'
)
parser.add_argument(
'--self_training_period',
type=int,
default=878,
help='the self-training period., 每训练多少个step后，更新一下teacher模型')
parser.add_argument('--self_training_hp_label',
type=float,
default=0,
help='是否使用高置信度标签重新加权软标签')
parser.add_argument('--self_training_ensemble_label',
type=int,
default=0,
help='use ensemble label.')
args = parser.parse_args()
# 决定是否覆盖已有的output目录
if (os.path.exists(args.output_dir) and os.listdir(args.output_dir)
and args.do_train and not args.overwrite_output_dir):
raise ValueError(
"Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome."
.format(args.output_dir))
# 如果outputs目录不存在，那么创建
if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
os.makedirs(args.output_dir)
# Setup distant debugging if needed
if args.server_ip and args.server_port:
# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
import ptvsd
print("Waiting for debugger attach")
ptvsd.enable_attach(address=(args.server_ip, args.server_port),
redirect_output=True)
ptvsd.wait_for_attach()
# Setup CUDA, GPU & distributed training
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
args.n_gpu = torch.cuda.device_count()
else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
torch.distributed.init_process_group(backend="nccl")
args.n_gpu = 1
args.device = device
# Setup logging
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN,
)
logging_fh = logging.FileHandler(os.path.join(args.output_dir, 'log.txt'))
logging_fh.setLevel(logging.DEBUG)
logger.addHandler(logging_fh)
logger.warning(
"处理的 rank: %s, device: %s, n_gpu: %s, 是否分布式训练: %s, 是否 16-bits 训练 : %s",
args.local_rank,
device,
args.n_gpu,
bool(args.local_rank != -1),
args.fp16,
)
# Set seed
set_seed(args)
# 获取这个数据的所有labels. eg: ['O', 'B-LOC', 'B-ORG', 'B-PER', 'B-MISC', 'I-PER', 'I-MISC', 'I-ORG', 'I-LOC', '<START>', '<STOP>']
labels = get_labels(args.data_dir)
num_labels = len(labels)
# 使用交叉熵, 忽略索引作为padding label ID，以便以后真实标签ID去计算损失, eg: pad_token_label_id = -100
pad_token_label_id = CrossEntropyLoss().ignore_index
# 加载预训练模型
if args.local_rank not in [-1, 0]:
torch.distributed.barrier(
) # Make sure only the first process in distributed training will download model & vocab
args.model_type = args.model_type.lower()
config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
config = config_class.from_pretrained(
args.config_name if args.config_name else args.model_name_or_path,
num_labels=num_labels,
cache_dir=args.cache_dir if args.cache_dir else None,
)
tokenizer = tokenizer_class.from_pretrained(
args.tokenizer_name
if args.tokenizer_name else args.model_name_or_path,
do_lower_case=args.do_lower_case,
cache_dir=args.cache_dir if args.cache_dir else None,
)
if args.local_rank == 0:
torch.distributed.barrier(
) # Make sure only the first process in distributed training will download model & vocab
logger.info("训练/评估 参数 %s", args)
# 开始训练
if args.do_train:
#加载数据集
train_dataset = load_and_cache_examples(args,
tokenizer,
labels,
pad_token_label_id,
mode="train")
#开始训练模型
model, global_step, tr_loss, best_dev, best_test = train(
args, train_dataset, model_class, config, tokenizer, labels,
pad_token_label_id)
#打印日志
logger.info(" global_step = %s, average loss = %s", global_step,
tr_loss)
# 保存last-practice：如果您使用模型的默认名称，则可以使用from_pretrained()重新加载它
if args.do_train and (args.local_rank == -1
or torch.distributed.get_rank() == 0):
logger.info("保存模型 checkpoint to %s", args.output_dir)
model_to_save = (model.module if hasattr(model, "module") else model
) # Take care of distributed/parallel training
model_to_save.save_pretrained(args.output_dir)
tokenizer.save_pretrained(args.output_dir)
torch.save(args, os.path.join(args.output_dir, "training_args.bin"))
torch.save(model.state_dict(), os.path.join(args.output_dir,
"model.pt"))
#评估模型
results = {}
if args.do_eval and args.local_rank in [-1, 0]:
tokenizer = tokenizer_class.from_pretrained(
args.output_dir, do_lower_case=args.do_lower_case)
checkpoints = [args.output_dir]
if args.eval_all_checkpoints:
checkpoints = list(
os.path.dirname(c) for c in sorted(
glob.glob(args.output_dir + "/**/" + WEIGHTS_NAME,
recursive=True)))
logging.getLogger("pytorch_transformers.modeling_utils").setLevel(
logging.WARN) # Reduce logging
logger.info("评估如下 checkpoints: %s", checkpoints)
if not best_dev:
best_dev = [0, 0, 0]
for checkpoint in checkpoints:
global_step = checkpoint.split(
"-")[-1] if len(checkpoints) > 1 else ""
model = model_class.from_pretrained(checkpoint)
model.to(args.device)
result, _, best_dev, _ = evaluate(args,
model,
tokenizer,
labels,
pad_token_label_id,
best=best_dev,
mode="dev",
prefix=global_step)
if global_step:
result = {
"{}_{}".format(global_step, k): v
for k, v in result.items()
}
results.update(result)
output_eval_file = os.path.join(args.output_dir, "eval_results.txt")
with open(output_eval_file, "w") as writer:
for key in sorted(results.keys()):
writer.write("{} = {}\n".format(key, str(results[key])))
if args.do_predict and args.local_rank in [-1, 0]:
tokenizer = tokenizer_class.from_pretrained(
args.output_dir, do_lower_case=args.do_lower_case)
model = model_class.from_pretrained(args.output_dir)
model.to(args.device)
if not best_test:
best_test = [0, 0, 0]
result, predictions, _, _ = evaluate(args,
model,
tokenizer,
labels,
pad_token_label_id,
best=best_test,
mode="test")
# Save results
output_test_results_file = os.path.join(args.output_dir,
"test_results.txt")
with open(output_test_results_file, "w") as writer:
for key in sorted(result.keys()):
writer.write("{} = {}\n".format(key, str(result[key])))
# Save predictions
output_test_predictions_file = os.path.join(args.output_dir,
"test_predictions.txt")
with open(output_test_predictions_file, "w") as writer:
with open(os.path.join(args.data_dir, "test.json"), "r") as f:
example_id = 0
data = json.load(f)
for item in data:
output_line = str(
item["str_words"]) + " " + predictions[example_id].pop(
0) + "\n"
writer.write(output_line)
example_id += 1
return results
def train(self, data_dir, epochs=3, callback=None):
if self.busy:
return 1
self.busy = True
label_names = get_labels(data_dir)
self.NUM_CLASSES = len(label_names)
from keras.applications.vgg16 import VGG16, preprocess_input
# 采用VGG16为基本模型，include_top为False，表示FC层是可自定义的，抛弃模型中的FC层；该模型会在~/.keras/models下载基本模型
base_model = VGG16(input_shape=(self.IMAGE_SIZE, self.IMAGE_SIZE, 3),
include_top=False,
weights='imagenet')
x_data, y_label = load_img_from_dir(data_dir,
target_size=(self.IMAGE_SIZE,
self.IMAGE_SIZE),
max_num=30)
for i in range(x_data.shape[0]):
x_data[i] = preprocess_input(x_data[i])
print(x_data.shape)
print(x_data[0].shape)
x_data = x_data.reshape(x_data.shape[0], self.IMAGE_SIZE,
self.IMAGE_SIZE, 3)
y_label_one_hot = prepress_labels(y_label)
# 验证应该使用从未见过的图片
train_x, test_x, train_y, test_y = train_test_split(x_data,
y_label_one_hot,
random_state=0,
test_size=0.3)
# 自定义FC层以基本模型的输入为卷积层的最后一层
x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dense(self.FC_NUMS, activation='relu')(x)
prediction = Dense(self.NUM_CLASSES, activation='softmax')(x)
# 构造完新的FC层，加入custom层
model = Model(inputs=base_model.input, outputs=prediction)
# 获取模型的层数
print("layer nums:", len(model.layers))
# 除了FC层，靠近FC层的一部分卷积层可参与参数训练，
# 一般来说，模型结构已经标明一个卷积块包含的层数，
# 在这里我们选择FREEZE_LAYERS为17，表示最后一个卷积块和FC层要参与参数训练
for layer in model.layers[:self.FREEZE_LAYERS]:
layer.trainable = False
for layer in model.layers[self.FREEZE_LAYERS:]:
layer.trainable = True
for layer in model.layers:
print("layer.trainable:", layer.trainable)
# 预编译模型
model.compile(optimizer=SGD(lr=0.0001, momentum=0.9),
loss='categorical_crossentropy',
metrics=['accuracy'])
model.summary()
model.fit(
train_x,
train_y,
validation_data=(test_x, test_y),
# model.fit(x_data, y_label_one_hot,
# validation_split=0.4,
callbacks=[AccuracyLogger(callback)],
epochs=epochs,
batch_size=4,
# steps_per_epoch=1,validation_steps =1 ,
verbose=1,
shuffle=True)
self.model = model
model.save(os.path.join(data_dir, 'model.h5'))
self.dump_label_name(label_names)
self.session = K.get_session()
self.graph = tf.get_default_graph()
self.busy = False
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument("--data_dir", default=None, type=str, required=True,
help="The input data dir. Should contain the training files for the CoNLL-2003 NER task.")
parser.add_argument("--model_type", default=None, type=str, required=True,
help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys()))
parser.add_argument("--model_name_or_path", default=None, type=str, required=True,
help="Path to pre-trained model or shortcut name selected in the list: " + ", ".join(ALL_MODELS))
parser.add_argument("--output_dir", default=None, type=str, required=True,
help="The output directory where the model predictions and checkpoints will be written.")
## Other parameters
parser.add_argument("--labels", default="", type=str,
help="Path to a file containing all labels. If not specified, CoNLL-2003 labels are used.")
parser.add_argument("--config_name", default="", type=str,
help="Pretrained config name or path if not the same as model_name")
parser.add_argument("--tokenizer_name", default="", type=str,
help="Pretrained tokenizer name or path if not the same as model_name")
parser.add_argument("--cache_dir", default="", type=str,
help="Where do you want to store the pre-trained models downloaded from s3")
parser.add_argument("--max_seq_length", default=128, type=int,
help="The maximum total input sequence length after tokenization. Sequences longer "
"than this will be truncated, sequences shorter will be padded.")
parser.add_argument("--do_train", action="store_true",
help="Whether to run training.")
parser.add_argument("--do_eval", action="store_true",
help="Whether to run eval on the dev set.")
parser.add_argument("--do_predict", action="store_true",
help="Whether to run predictions on the test set.")
parser.add_argument("--evaluate_during_training", action="store_true",
help="Whether to run evaluation during training at each logging step.")
parser.add_argument("--do_lower_case", action="store_true",
help="Set this flag if you are using an uncased model.")
parser.add_argument("--per_gpu_train_batch_size", default=16, type=int,
help="Batch size per GPU/CPU for training.")
parser.add_argument("--per_gpu_eval_batch_size", default=16, type=int,
help="Batch size per GPU/CPU for evaluation.")
parser.add_argument("--gradient_accumulation_steps", type=int, default=1,
help="Number of updates steps to accumulate before performing a backward/update pass.")
parser.add_argument("--learning_rate", default=5e-5, type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--weight_decay", default=0.0, type=float,
help="Weight decay if we apply some.")
parser.add_argument("--adam_epsilon", default=1e-8, type=float,
help="Epsilon for Adam optimizer.")
parser.add_argument("--max_grad_norm", default=1.0, type=float,
help="Max gradient norm.")
parser.add_argument("--num_train_epochs", default=1.0, type=float,
help="Total number of training epochs to perform.")
parser.add_argument("--max_steps", default=-1, type=int,
help="If > 0: set total number of training steps to perform. Override num_train_epochs.")
parser.add_argument("--warmup_steps", default=0, type=int,
help="Linear warmup over warmup_steps.")
parser.add_argument("--logging_steps", type=int, default=50,
help="Log every X updates steps.")
parser.add_argument("--save_steps", type=int, default=50,
help="Save checkpoint every X updates steps.")
parser.add_argument("--eval_all_checkpoints", action="store_true",
help="Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number")
parser.add_argument("--no_cuda", action="store_true",
help="Avoid using CUDA when available")
parser.add_argument("--overwrite_output_dir", action="store_true",
help="Overwrite the content of the output directory")
parser.add_argument("--overwrite_cache", action="store_true",
help="Overwrite the cached training and evaluation sets")
parser.add_argument("--seed", type=int, default=42,
help="random seed for initialization")
parser.add_argument("--fp16", action="store_true",
help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit")
parser.add_argument("--fp16_opt_level", type=str, default="O1",
help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
"See details at https://nvidia.github.io/apex/amp.html")
parser.add_argument("--local_rank", type=int, default=-1,
help="For distributed training: local_rank")
parser.add_argument("--server_ip", type=str, default="", help="For distant debugging.")
parser.add_argument("--server_port", type=str, default="", help="For distant debugging.")
parser.add_argument("--method", default='finetune', type=str, help="which method to use")
# self-training
parser.add_argument('--soft_label', type = int, default = 1, help = 'whether soft label (0 for hard, 1 for soft)')
parser.add_argument("--soft_label_weight", default=1.0, type=float, help="iters for pretrains")
parser.add_argument('--self_training_eps', type = float, default = 0.8, help = 'threshold for confidence')
parser.add_argument('--self_training_power', type = float, default = 2, help = 'power of pred score')
parser.add_argument('--self_training_confreg', type = float, default = 0, help = 'confidence smooth power')
parser.add_argument('--self_training_contrastive_weight', type = float, default = 0, help = 'contrastive learning weight')
parser.add_argument('--self_training_max_step', type = int, default = 10000, help = 'the maximum step (usually after the first epoch) for self training')
parser.add_argument('--distmetric', type = str, default = "l2", help = 'distance type. Choices = [cos, l2]')
parser.add_argument('--self_training_label_mode', type = str, default = "hard", help = 'pseudo label type. Choices = [hard, soft]')
parser.add_argument('--self_training_update_period', type = int, default = 100, help = 'update period')
args = parser.parse_args()
if os.path.exists(args.output_dir) and os.listdir(
args.output_dir) and args.do_train and not args.overwrite_output_dir:
raise ValueError(
"Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(
args.output_dir))
# Setup distant debugging if needed
if args.server_ip and args.server_port:
# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
import ptvsd
print("Waiting for debugger attach")
ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True)
ptvsd.wait_for_attach()
# Setup CUDA, GPU & distributed training
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
args.n_gpu = torch.cuda.device_count()
else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
torch.distributed.init_process_group(backend="nccl")
args.n_gpu = 1
args.device = device
# Setup logging
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN)
logger.warning("Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
args.local_rank, device, args.n_gpu, bool(args.local_rank != -1), args.fp16)
# Set seed
set_seed(args)
# Prepare NER task
labels = get_labels(args.labels)
num_labels = len(labels)
# Use cross entropy ignore index as padding label id so that only real label ids contribute to the loss later
pad_token_label_id = CrossEntropyLoss().ignore_index
# Load pretrained model and tokenizer
if args.local_rank not in [-1, 0]:
torch.distributed.barrier() # Make sure only the first process in distributed training will download model & vocab
args.model_type = args.model_type.lower()
config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path,
num_labels=num_labels)
tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
do_lower_case=args.do_lower_case)
model = model_class.from_pretrained(args.model_name_or_path, from_tf=bool(".ckpt" in args.model_name_or_path),
config=config)
if args.local_rank == 0:
torch.distributed.barrier() # Make sure only the first process in distributed training will download model & vocab
model.to(args.device)
logger.info("Training/evaluation parameters %s", args)
# Training
if args.do_train:
if args.method == 'finetune':
train_dataset = load_and_cache_examples(args, tokenizer, labels, pad_token_label_id, mode="train")
global_step, tr_loss = train(args, train_dataset, model, tokenizer, labels, pad_token_label_id)
logger.info(" global_step = %s, average loss = %s", global_step, tr_loss)
if args.method == 'selftrain':
args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu)
logger.info("***selftrain starts here***")
unlabeled_dataset = load_and_cache_examples(args, tokenizer, labels, pad_token_label_id, mode="unlabeled")
train_dataset = load_and_cache_examples(args, tokenizer, labels, pad_token_label_id, mode="train")
selftrain_dataset = ConcatDataset([train_dataset, unlabeled_dataset])
##train step
self_train(args, selftrain_dataset, model, tokenizer, labels, pad_token_label_id)
exit()
# Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()
if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0):
# Create output directory if needed
if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
os.makedirs(args.output_dir)
logger.info("Saving model checkpoint to %s", args.output_dir)
# Save a trained model, configuration and tokenizer using `save_pretrained()`.
# They can then be reloaded using `from_pretrained()`
model_to_save = model.module if hasattr(model, "module") else model # Take care of distributed/parallel training
model_to_save.save_pretrained(args.output_dir)
tokenizer.save_pretrained(args.output_dir)
# Good practice: save your training arguments together with the trained model
torch.save(args, os.path.join(args.output_dir, "training_args.bin"))
# Evaluation
results = {}
if args.do_eval and args.local_rank in [-1, 0]:
tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
checkpoints = [args.output_dir]
if args.eval_all_checkpoints:
checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + "/**/" + WEIGHTS_NAME, recursive=True)))
logging.getLogger("pytorch_transformers.modeling_utils").setLevel(logging.WARN) # Reduce logging
logger.info("Evaluate the following checkpoints: %s", checkpoints)
for checkpoint in checkpoints:
global_step = checkpoint.split("-")[-1] if len(checkpoints) > 1 else ""
model = model_class.from_pretrained(checkpoint)
model.to(args.device)
result, _ = evaluate(args, model, tokenizer, labels, pad_token_label_id, mode="dev", prefix=global_step)
if global_step:
result = {"{}_{}".format(global_step, k): v for k, v in result.items()}
results.update(result)
output_eval_file = os.path.join(args.output_dir, "eval_results.txt")
with open(output_eval_file, "w") as writer:
for key in sorted(results.keys()):
writer.write("{} = {}\n".format(key, str(results[key])))
if args.do_predict and args.local_rank in [-1, 0]:
tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
model = model_class.from_pretrained(args.output_dir)
model.to(args.device)
result, predictions = evaluate(args, model, tokenizer, labels, pad_token_label_id, mode="test")
# Save results
output_test_results_file = os.path.join(args.output_dir, "test_results.txt")
with open(output_test_results_file, "w") as writer:
for key in sorted(result.keys()):
writer.write("{} = {}\n".format(key, str(result[key])))
# Save predictions
output_test_predictions_file = os.path.join(args.output_dir, "test_predictions.txt")
with open(output_test_predictions_file, "w", encoding='utf-8') as writer:
with open(os.path.join(args.data_dir, "test.txt"), "r", encoding='utf-8') as f:
example_id = 0
for line in f:
if line.startswith("-DOCSTART-") or line == "" or line == "\n":
writer.write(line)
if not predictions[example_id]:
example_id += 1
elif predictions[example_id]:
output_line = line.split()[0] + " " + predictions[example_id].pop(0) + "\n"
writer.write(output_line)
else:
logger.warning("Maximum sequence length exceeded: No prediction for '%s'.", line.split()[0])
return results
from PIL import Image
import data_utils as du
import tensorflow as tf
import matplotlib.pyplot as plt
from skimage.transform import resize
from collections import namedtuple
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
import fileinput
coords, chips, classes = du.get_labels(
'/Users/nanboliu/Desktop/xview/data/xView_train.geojson')
img_dict = {}
for i in range(chips.shape[0]):
if chips[i] not in ['24.tif']:
if chips[i] not in img_dict:
img_dict[chips[i]] = set()
img_dict[chips[i]].add(i)
#create a label dictionary
labels = {}
for line in fileinput.input(
'/Users/nanboliu/Desktop/xview/data/class_labels.txt'):
labels[int(line.split(":")[0])] = line.split(":")[1].rstrip('\n')
class_vector = [keys for keys in labels]
n_class = len(class_vector)
resized_image = (32, 32)
dataset = namedtuple('Dataset', ['X', 'y'])
path = '/Volumes/Nanbo/xview_data/train_images/'
from keras.models import load_model
import tensorflow as tf
import base64
import numpy as np
import cv2
import os
import json
from data_utils import get_labels
app = Flask(__name__)
graph = tf.get_default_graph()
model = load_model('QuickDraw_conv82.h5') # 加载训练模型
data_dir = ".\\npy_data\\"
label_names = get_labels(data_dir)
@app.route('/labels')
def get_labels():
# return json.dumps(label_names)
return Response(json.dumps(label_names), mimetype='application/json')
@app.route('/', methods=['GET', 'POST'])
def index():
if request.method == 'POST':
# print(request.form)
img_b64encode = request.form.get("base64img", "")
# img_b64encode = "data:image/jpeg;base64,/9j/4AAQS......."
img_b64decode = base64.b64decode(img_b64encode[23:]) # base64解码
def main():
parser = argparse.ArgumentParser()
# Required parameters
parser.add_argument(
"--data_dir",
default=None,
type=str,
required=True,
help="The input data dir. Should contain the training files for the CoNLL-2003 NER task.",
)
parser.add_argument(
"--model_type",
default=None,
type=str,
required=True,
help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys()),
)
parser.add_argument(
"--model_name_or_path",
default=None,
type=str,
required=True,
help="Path to pre-trained model or shortcut name selected in the list: " + ", ".join(ALL_MODELS),
)
parser.add_argument(
"--output_dir",
default=None,
type=str,
required=True,
help="The output directory where the model predictions and checkpoints will be written.",
)
parser.add_argument(
"--config_name", default="", type=str, help="Pretrained config name or path if not the same as model_name"
)
parser.add_argument(
"--tokenizer_name",
default="",
type=str,
help="Pretrained tokenizer name or path if not the same as model_name",
)
parser.add_argument(
"--cache_dir",
default="",
type=str,
help="Where do you want to store the pre-trained models downloaded from s3",
)
parser.add_argument(
"--max_seq_length",
default=128,
type=int,
help="The maximum total input sequence length after tokenization. Sequences longer "
"than this will be truncated, sequences shorter will be padded.",
)
parser.add_argument("--do_train", action="store_true", help="Whether to run training.")
parser.add_argument("--do_eval", action="store_true", help="Whether to run eval on the dev set.")
parser.add_argument("--do_predict", action="store_true", help="Whether to run predictions on the test set.")
parser.add_argument(
"--evaluate_during_training",
action="store_true",
help="Whether to run evaluation during training at each logging step.",
)
parser.add_argument(
"--do_lower_case", action="store_true", help="Set this flag if you are using an uncased model."
)
parser.add_argument("--per_gpu_train_batch_size", default=8, type=int, help="Batch size per GPU/CPU for training.")
parser.add_argument(
"--per_gpu_eval_batch_size", default=8, type=int, help="Batch size per GPU/CPU for evaluation."
)
parser.add_argument(
"--gradient_accumulation_steps",
type=int,
default=1,
help="Number of updates steps to accumulate before performing a backward/update pass.",
)
## Other parameters
parser.add_argument("--labels", default="", type=str,
help="Path to a file containing all labels. If not specified, CoNLL-2003 labels are used.")
parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.")
parser.add_argument("--weight_decay", default=0.0, type=float, help="Weight decay if we apply some.")
parser.add_argument("--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.")
parser.add_argument("--adam_beta1", default=0.9, type=float, help="BETA1 for Adam optimizer.")
parser.add_argument("--adam_beta2", default=0.999, type=float, help="BETA2 for Adam optimizer.")
parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.")
parser.add_argument(
"--num_train_epochs", default=3.0, type=float, help="Total number of training epochs to perform."
)
parser.add_argument(
"--max_steps",
default=-1,
type=int,
help="If > 0: set total number of training steps to perform. Override num_train_epochs.",
)
parser.add_argument("--warmup_steps", default=0, type=int, help="Linear warmup over warmup_steps.")
parser.add_argument("--logging_steps", type=int, default=50, help="Log every X updates steps.")
parser.add_argument("--save_steps", type=int, default=50, help="Save checkpoint every X updates steps.")
parser.add_argument(
"--eval_all_checkpoints",
action="store_true",
help="Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number",
)
parser.add_argument("--no_cuda", action="store_true", help="Avoid using CUDA when available")
parser.add_argument(
"--overwrite_output_dir", action="store_true", help="Overwrite the content of the output directory"
)
parser.add_argument(
"--overwrite_cache", action="store_true", help="Overwrite the cached training and evaluation sets"
)
parser.add_argument("--seed", type=int, default=42, help="random seed for initialization")
parser.add_argument(
"--fp16",
action="store_true",
help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit",
)
parser.add_argument(
"--fp16_opt_level",
type=str,
default="O1",
help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
"See details at https://nvidia.github.io/apex/amp.html",
)
parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank")
parser.add_argument("--server_ip", type=str, default="", help="For distant debugging.")
parser.add_argument("--server_port", type=str, default="", help="For distant debugging.")
# mean teacher
parser.add_argument('--mt', type = int, default = 0, help = 'mean teacher.')
parser.add_argument('--mt_updatefreq', type=int, default=1, help = 'mean teacher update frequency')
parser.add_argument('--mt_class', type=str, default="kl", help = 'mean teacher class, choices:[smart, prob, logit, kl(default), distill].')
parser.add_argument('--mt_lambda', type=float, default=1, help= "trade off parameter of the consistent loss.")
parser.add_argument('--mt_rampup', type=int, default=300, help="rampup iteration.")
parser.add_argument('--mt_alpha1', default=0.99, type=float, help="moving average parameter of mean teacher (for the exponential moving average).")
parser.add_argument('--mt_alpha2', default=0.995, type=float, help="moving average parameter of mean teacher (for the exponential moving average).")
parser.add_argument('--mt_beta', default=10, type=float, help="coefficient of mt_loss term.")
parser.add_argument('--mt_avg', default="exponential", type=str, help="moving average method, choices:[exponentail(default), simple, double_ema].")
parser.add_argument('--mt_loss_type', default="logits", type=str, help="subject to measure model difference, choices:[embeds, logits(default)].")
# virtual adversarial training
parser.add_argument('--vat', type = int, default = 0, help = 'virtual adversarial training.')
parser.add_argument('--vat_eps', type = float, default = 1e-3, help = 'perturbation size for virtual adversarial training.')
parser.add_argument('--vat_lambda', type = float, default = 1, help = 'trade off parameter for virtual adversarial training.')
parser.add_argument('--vat_beta', type = float, default = 1, help = 'coefficient of the virtual adversarial training loss term.')
parser.add_argument('--vat_loss_type', default="logits", type=str, help="subject to measure model difference, choices = [embeds, logits(default)].")
args = parser.parse_args()
# Prepare NER task
labels = get_labels(args.labels)
num_labels = len(labels)
if (
os.path.exists(args.output_dir)
and os.listdir(args.output_dir)
and args.do_train
and not args.overwrite_output_dir
):
raise ValueError(
"Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(
args.output_dir
)
)
# Create output directory if needed
if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
os.makedirs(args.output_dir)
# Setup distant debugging if needed
if args.server_ip and args.server_port:
# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
import ptvsd
print("Waiting for debugger attach")
ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True)
ptvsd.wait_for_attach()
# Setup CUDA, GPU & distributed training
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
args.n_gpu = torch.cuda.device_count()
else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
torch.distributed.init_process_group(backend="nccl")
args.n_gpu = 1
args.device = device
# Setup logging
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN,
)
logging_fh = logging.FileHandler(os.path.join(args.output_dir, 'log.txt'))
logging_fh.setLevel(logging.DEBUG)
logger.addHandler(logging_fh)
logger.warning(
"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
args.local_rank,
device,
args.n_gpu,
bool(args.local_rank != -1),
args.fp16,
)
# Set seed
set_seed(args)
labels = get_labels(args.data_dir)
num_labels = len(labels)
# Use cross entropy ignore index as padding label id so that only real label ids contribute to the loss later
pad_token_label_id = CrossEntropyLoss().ignore_index
# Load pretrained model and tokenizer
if args.local_rank not in [-1, 0]:
torch.distributed.barrier() # Make sure only the first process in distributed training will download model & vocab
args.model_type = args.model_type.lower()
config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
config = config_class.from_pretrained(
args.config_name if args.config_name else args.model_name_or_path,
num_labels=num_labels,
cache_dir=args.cache_dir if args.cache_dir else None,
)
tokenizer = tokenizer_class.from_pretrained(
args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
do_lower_case=args.do_lower_case,
cache_dir=args.cache_dir if args.cache_dir else None,
)
model = model_class.from_pretrained(
args.model_name_or_path,
from_tf=bool(".ckpt" in args.model_name_or_path),
config=config,
cache_dir=args.cache_dir if args.cache_dir else None,
)
if args.local_rank == 0:
torch.distributed.barrier() # Make sure only the first process in distributed training will download model & vocab
model.to(args.device)
logger.info("Training/evaluation parameters %s", args)
# Training
if args.do_train:
train_dataset = load_and_cache_examples(args, tokenizer, labels, pad_token_label_id, mode="train")
global_step, tr_loss, best_dev, best_test = train(args, train_dataset, model, tokenizer, labels, pad_token_label_id)
logger.info(" global_step = %s, average loss = %s", global_step, tr_loss)
# Saving last-practice: if you use defaults names for the model, you can reload it using from_pretrained()
if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0):
logger.info("Saving model checkpoint to %s", args.output_dir)
model_to_save = (
model.module if hasattr(model, "module") else model
) # Take care of distributed/parallel training
model_to_save.save_pretrained(args.output_dir)
tokenizer.save_pretrained(args.output_dir)
torch.save(args, os.path.join(args.output_dir, "training_args.bin"))
torch.save(model.state_dict(), os.path.join(args.output_dir, "model.pt"))
# Evaluation
results = {}
if args.do_eval and args.local_rank in [-1, 0]:
tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
checkpoints = [args.output_dir]
if args.eval_all_checkpoints:
checkpoints = list(
os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + "/**/" + WEIGHTS_NAME, recursive=True))
)
logging.getLogger("pytorch_transformers.modeling_utils").setLevel(logging.WARN) # Reduce logging
logger.info("Evaluate the following checkpoints: %s", checkpoints)
if not best_dev:
best_dev = [0, 0, 0]
for checkpoint in checkpoints:
global_step = checkpoint.split("-")[-1] if len(checkpoints) > 1 else ""
model = model_class.from_pretrained(checkpoint)
model.to(args.device)
result, _, best_dev, _ = evaluate(args, model, tokenizer, labels, pad_token_label_id, best=best_dev, mode="dev", prefix=global_step)
if global_step:
result = {"{}_{}".format(global_step, k): v for k, v in result.items()}
results.update(result)
output_eval_file = os.path.join(args.output_dir, "eval_results.txt")
with open(output_eval_file, "w") as writer:
for key in sorted(results.keys()):
writer.write("{} = {}\n".format(key, str(results[key])))
if args.do_predict and args.local_rank in [-1, 0]:
tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
model = model_class.from_pretrained(args.output_dir)
model.to(args.device)
if not best_test:
best_test = [0, 0, 0]
result, predictions, _, _ = evaluate(args, model, tokenizer, labels, pad_token_label_id, best=best_test, mode="test")
# Save results
output_test_results_file = os.path.join(args.output_dir, "test_results.txt")
with open(output_test_results_file, "w") as writer:
for key in sorted(result.keys()):
writer.write("{} = {}\n".format(key, str(result[key])))
# Save predictions
output_test_predictions_file = os.path.join(args.output_dir, "test_predictions.txt")
with open(output_test_predictions_file, "w") as writer:
with open(os.path.join(args.data_dir, "test.json"), "r") as f:
example_id = 0
data = json.load(f)
for item in data:
output_line = str(item["str_words"]) + " " + predictions[example_id].pop(0) + "\n"
writer.write(output_line)
example_id += 1
return results
def train(self, data_dir, epochs=3, callback=None, model_name='vgg16'):
"""
train 函数, 训练模型
Keyword arguments::
epochs: int 训练次数
"""
if self.busy:
return 1
self.busy = True
label_names = get_labels(data_dir)
self.NUM_CLASSES = len(label_names)
base_model = self.get_base_model(model_name)
x_data, y_label = load_img_from_dir(data_dir,
target_size=(self.IMAGE_SIZE,
self.IMAGE_SIZE),
max_num=30)
for i in range(x_data.shape[0]):
x_data[i] = self.preprocess_input(x_data[i])
print(x_data.shape)
print(x_data[0].shape)
x_data = x_data.reshape(x_data.shape[0], self.IMAGE_SIZE,
self.IMAGE_SIZE, 3)
y_label_one_hot = prepress_labels(y_label)
# 验证应该使用从未见过的图片
train_x, test_x, train_y, test_y = train_test_split(x_data,
y_label_one_hot,
random_state=0,
test_size=0.3)
# 自定义FC层以基本模型的输入为卷积层的最后一层
x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dense(self.FC_NUMS, activation='relu')(x)
prediction = Dense(self.NUM_CLASSES, activation='softmax')(x)
# 构造完新的FC层，加入custom层
model = Model(inputs=base_model.input, outputs=prediction)
# 获取模型的层数
print("layer nums:", len(model.layers))
# 除了FC层，靠近FC层的一部分卷积层可参与参数训练，
# 一般来说，模型结构已经标明一个卷积块包含的层数，
# 在这里我们选择TRAIN_LAYERS为3，表示最后一个卷积块和FC层要参与参数训练
for layer in model.layers:
layer.trainable = False
for layer in model.layers[-self.TRAIN_LAYERS:]:
layer.trainable = True
for layer in model.layers:
print("layer.trainable:", layer.trainable)
# 预编译模型
model.compile(optimizer=SGD(lr=0.0001, momentum=0.9),
loss='categorical_crossentropy',
metrics=['accuracy'])
model.summary()
model.fit(
train_x,
train_y,
validation_data=(test_x, test_y),
# model.fit(x_data, y_label_one_hot,
# validation_split=0.4,
callbacks=[AccuracyLogger(callback)],
epochs=epochs,
batch_size=4,
# steps_per_epoch=1,validation_steps =1 ,
verbose=1,
shuffle=True)
self.model = model
model.save(os.path.join(data_dir, 'model.h5'))
self.label_names = label_names
self.dump_label_name(label_names)
# self.convert_tflite()
self.session = K.get_session()
self.graph = tf.get_default_graph()
self.busy = False