def train(self, args: ClassifierArgs):
# get dataset
dataset, data_loader = self.build_data_loader(args, 'train')
# get trainer
trainer = self.build_trainer(args, dataset, data_loader)
best_metric = None
for epoch_time in range(args.epochs):
trainer.train_epoch(args, epoch_time)
# saving model according to epoch_time
self.saving_model_by_epoch(args, epoch_time)
# evaluate model according to epoch_time
metric = self.evaluate(args, is_training=True)
# update best metric
# if best_metric is None, update it with epoch metric directly, otherwise compare it with epoch_metric
if best_metric is None or metric > best_metric:
best_metric = metric
self.save_model_to_file(
args.saving_dir,
args.build_saving_file_name(description='best'))
if args.training_type == 'sparse' and args.incremental_trick and args.saving_last_epoch:
self.save_model_to_file(
args.saving_dir,
args.build_saving_file_name(description='best'))
self.evaluate(args)
def predict(self, args: ClassifierArgs, **kwargs):
# self.evaluate(args, is_training=False)
self.loading_model_from_file(
args.saving_dir, args.build_saving_file_name(description='best'))
self.model.eval()
predictor = Predictor(self.model, self.data_processor, args.model_type)
dataset, _ = self.build_data_loader(args,
args.evaluation_data_type,
tokenizer=False)
assert isinstance(dataset, ListDataset)
if args.predict_numbers == -1:
predict_dataset = dataset.data
else:
predict_dataset = np.random.choice(dataset.data,
size=(args.predict_numbers, ),
replace=False)
description = tqdm(predict_dataset)
metric = RandomSmoothAccuracyMetrics()
for data in description:
tmp_instances = self.mask_instance_decorator(
args, data, args.predict_ensemble)
tmp_probs = predictor.predict_batch(tmp_instances)
target = self.data_reader.get_label_to_idx(data.label)
pred = predict(tmp_probs, args.alpha)
metric(pred, target)
description.set_description(metric.__str__())
print(metric)
logging.info(metric)
def infer(self, args: ClassifierArgs) -> Dict:
content = args.content
assert content is not None, 'in infer mode, parameter content cannot be None! '
content = content.strip()
assert content != '' and len(
content) != 0, 'in infer mode, parameter content cannot be empty! '
self.loading_model_from_file(
args.saving_dir, args.build_saving_file_name(description='best'))
self.model.eval()
predictor = Predictor(self.model, self.data_processor, args.model_type)
pred_probs = predictor.predict(content)
pred_label = np.argmax(pred_probs)
pred_label = self.data_reader.get_idx_to_label(pred_label)
if pred_label == '100':
pred_label = '0'
elif pred_label == '101':
pred_label = '1'
result_in_dict = {
'content': content,
'pred_label': pred_label,
'pred_confidence': pred_probs
}
result_in_str = ', '.join([
'{}: {}'.format(key, value) if not isinstance(value, list) else
'{}: [{}]'.format(key, ', '.join(["%.4f" % val for val in value]))
for key, value in result_in_dict.items()
])
print(result_in_str)
logging.info(result_in_str)
return result_in_dict
def evaluate(self, args: ClassifierArgs, is_training=False) -> Metric:
if is_training:
logging.info('Using current modeling parameter to evaluate')
data_type = 'dev'
else:
self.loading_model_from_file(
args.saving_dir,
args.build_saving_file_name(description='best'))
data_type = args.evaluation_data_type
self.model.eval()
dataset, data_loader = self.build_data_loader(args, data_type)
epoch_iterator = tqdm(data_loader)
metric = DATASET_TYPE.get_evaluation_metric(
args.dataset_name, compare_key=args.compare_key)
for step, batch in enumerate(epoch_iterator):
assert isinstance(batch[0], torch.Tensor)
batch = tuple(t.cuda() for t in batch)
golds = batch[3]
inputs = convert_batch_to_bert_input_dict(batch, args.model_type)
logits = self.model.forward(**inputs)[0]
losses = self.loss_function(
logits.view(-1, self.data_reader.NUM_LABELS), golds.view(-1))
epoch_iterator.set_description('loss: {:.4f}'.format(
torch.mean(losses)))
metric(losses, logits, golds)
print(metric)
logging.info(metric)
return metric
def saving_model_by_epoch(self, args: ClassifierArgs, epoch: int):
# saving
if args.saving_step is not None and args.saving_step != 0:
if (epoch - 1) % args.saving_step == 0:
self.save_model_to_file(
args.saving_dir,
args.build_saving_file_name(
description='epoch{}'.format(epoch)))
def attack(self, args: ClassifierArgs, **kwargs):
# self.evaluate(args, is_training=False)
# self.evaluate(args, is_training=False)
self.loading_model_from_file(
args.saving_dir, args.build_saving_file_name(description='best'))
self.model.eval()
# build test dataset
dataset, _ = self.build_data_loader(args,
args.evaluation_data_type,
tokenizer=False)
test_instances = dataset.data
# build attacker
attacker = self.build_attacker(args)
attacker_log_path = '{}'.format(args.build_logging_path())
attacker_log_path = os.path.join(args.logging_dir, attacker_log_path)
attacker_log_manager = AttackLogManager()
# attacker_log_manager.enable_stdout()
attacker_log_manager.add_output_file(
os.path.join(attacker_log_path,
'{}.txt'.format(args.attack_method)))
for i in range(args.attack_times):
print("Attack time {}".format(i))
choice_instances = np.random.choice(test_instances,
size=(args.attack_numbers, ),
replace=False)
dataset = CustomTextAttackDataset.from_instances(
args.dataset_name, choice_instances,
self.data_reader.get_labels())
results_iterable = attacker.attack_dataset(dataset)
description = tqdm(results_iterable, total=len(choice_instances))
result_statistics = SimplifidResult()
for result in description:
try:
attacker_log_manager.log_result(result)
result_statistics(result)
description.set_description(result_statistics.__str__())
except RuntimeError as e:
print('error in process')
attacker_log_manager.enable_stdout()
attacker_log_manager.log_summary()
def build_writer(self, args: ClassifierArgs,
**kwargs) -> Union[SummaryWriter, None]:
writer = None
if args.tensorboard == 'yes':
tensorboard_file_name = '{}-tensorboard'.format(
args.build_logging_path())
tensorboard_path = os.path.join(args.logging_dir,
tensorboard_file_name)
writer = SummaryWriter(tensorboard_path)
return writer
def augmentation(self, args: ClassifierArgs, **kwargs):
self.loading_model_from_file(
args.saving_dir, args.build_saving_file_name(description='best'))
self.model.eval()
train_instances, _ = self.build_data_loader(args,
'train',
tokenizer=False)
train_dataset_len = len(train_instances.data)
print('Training Set: {} sentences. '.format(train_dataset_len))
# delete instance whose length is smaller than 3
train_instances_deleted = [
instance for instance in train_instances.data
if instance.length() >= 3
]
dataset_to_aug = np.random.choice(train_instances_deleted,
size=(int(train_dataset_len *
0.5), ),
replace=False)
dataset_to_write = np.random.choice(train_instances.data,
size=(int(train_dataset_len *
0.5), ),
replace=False).tolist()
attacker = self.build_attacker(args)
attacker_log_manager = AttackLogManager()
dataset = CustomTextAttackDataset.from_instances(
args.dataset_name, dataset_to_aug, self.data_reader.get_labels())
results_iterable = attacker.attack_dataset(dataset)
aug_instances = []
for result, instance in tqdm(zip(results_iterable, dataset_to_aug),
total=len(dataset)):
try:
adv_sentence = result.perturbed_text()
aug_instances.append(
InputInstance.from_instance_and_perturb_sentence(
instance, adv_sentence))
except:
print('one error happend, delete one instance')
dataset_to_write.extend(aug_instances)
self.data_reader.saving_instances(dataset_to_write, args.dataset_dir,
'aug_{}'.format(args.attack_method))
print('Writing {} Sentence. '.format(len(dataset_to_write)))
attacker_log_manager.enable_stdout()
attacker_log_manager.log_summary()
def run(cls, args: ClassifierArgs):
# build logging
# including check logging path, and set logging config
args.build_logging_dir()
args.build_logging()
logging.info(args)
args.build_environment()
# check dataset and its path
args.build_dataset_dir()
args.build_saving_dir()
args.build_caching_dir()
if args.dataset_name in ['agnews', 'snli']:
args.keep_sentiment_word = False
classifier = cls(args)
classifier.methods[args.mode](args)
def statistics(self, args: ClassifierArgs, **kwargs):
# self.evaluate(args, is_training=False)
self.loading_model_from_file(
args.saving_dir, args.build_saving_file_name(description='best'))
self.model.eval()
predictor = Predictor(self.model, self.data_processor, args.model_type)
dataset, _ = self.build_data_loader(args,
args.evaluation_data_type,
tokenizer=False)
assert isinstance(dataset, ListDataset)
if args.certify_numbers == -1:
certify_dataset = dataset.data
else:
certify_dataset = np.random.choice(dataset.data,
size=(args.certify_numbers, ),
replace=False)
description = tqdm(certify_dataset)
num_labels = self.data_reader.NUM_LABELS
metric = RandomAblationCertifyMetric()
result_dicts = {"pix": []}
for i in range(11):
result_dicts[str(i)] = list()
for data in description:
target = self.data_reader.get_label_to_idx(data.label)
data_length = data.length()
keep_nums = data_length - round(
data_length * args.sparse_mask_rate)
tmp_instances = self.mask_instance_decorator(
args, data, args.predict_ensemble)
tmp_probs = predictor.predict_batch(tmp_instances)
guess = np.argmax(
np.bincount(np.argmax(tmp_probs, axis=-1),
minlength=num_labels))
if guess != target:
metric(np.nan, data_length)
continue
numbers = args.ceritfy_ensemble * 2
tmp_instances, mask_indexes = self.mask_instance_decorator(
args, data, numbers, return_indexes=True)
ablation_indexes = [
list(set(list(range(data_length))) - set(indexes))
for indexes in mask_indexes
]
tmp_probs = predictor.predict_batch(tmp_instances)
tmp_preds = np.argmax(tmp_probs, axis=-1)
p_i_x = np.bincount(tmp_preds,
minlength=num_labels)[guess] / numbers
result_dicts["pix"].append(p_i_x)
for i in range(1, 11):
lambda_value = population_lambda(tmp_preds, ablation_indexes,
data_length, i, num_labels,
guess)
result_dicts[str(i)].append(lambda_value)
file_name = os.path.join(
args.logging_dir, "{}-probs.txt".format(args.build_logging_path()))
with open(file_name, 'w') as file:
for key, value in result_dicts.items():
file.write(key)
file.write(": ")
file.write(" ".join([str(v) for v in value]))
file.write("\n")
def certify(self, args: ClassifierArgs, **kwargs):
# self.evaluate(args, is_training=False)
self.loading_model_from_file(
args.saving_dir, args.build_saving_file_name(description='best'))
self.model.eval()
predictor = Predictor(self.model, self.data_processor, args.model_type)
dataset, _ = self.build_data_loader(args,
args.evaluation_data_type,
tokenizer=False)
assert isinstance(dataset, ListDataset)
if args.certify_numbers == -1:
certify_dataset = dataset.data
else:
certify_dataset = np.random.choice(dataset.data,
size=(args.certify_numbers, ),
replace=False)
description = tqdm(certify_dataset)
num_labels = self.data_reader.NUM_LABELS
metric = RandomAblationCertifyMetric()
for data in description:
target = self.data_reader.get_label_to_idx(data.label)
data_length = data.length()
keep_nums = data_length - round(
data_length * args.sparse_mask_rate)
tmp_instances = self.mask_instance_decorator(
args, data, args.predict_ensemble)
tmp_probs = predictor.predict_batch(tmp_instances)
guess = np.argmax(
np.bincount(np.argmax(tmp_probs, axis=-1),
minlength=num_labels))
if guess != target:
metric(np.nan, data_length)
continue
tmp_instances = self.mask_instance_decorator(
args, data, args.ceritfy_ensemble)
tmp_probs = predictor.predict_batch(tmp_instances)
guess_counts = np.bincount(np.argmax(tmp_probs, axis=-1),
minlength=num_labels)[guess]
lower_bound, upper_bound = lc_bound(guess_counts,
args.ceritfy_ensemble,
args.alpha)
if args.certify_lambda:
# tmp_instances, mask_indexes = mask_instance(data, args.sparse_mask_rate, self.tokenizer.mask_token,nums=args.ceritfy_ensemble * 2, return_indexes=True)
# tmp_probs = predictor.predict_batch(tmp_instances)
# tmp_preds = np.argmax(tmp_probs, axis=-1)
# ablation_indexes = [list(set(list(range(data_length))) - set(indexes.tolist())) for indexes in mask_indexes]
# radius = population_radius_for_majority_by_estimating_lambda(lower_bound, data_length, keep_nums, tmp_preds, ablation_indexes, num_labels, guess, samplers = 200)
radius = population_radius_for_majority(
lower_bound,
data_length,
keep_nums,
lambda_value=guess_counts / args.ceritfy_ensemble)
else:
radius = population_radius_for_majority(
lower_bound, data_length, keep_nums)
metric(radius, data_length)
result = metric.get_metric()
description.set_description("Accu: {:.2f}%, Median: {}".format(
result['accuracy'] * 100, result['median']))
print(metric)
logging.info(metric)
# logging metric certify_radius and length
logging.info(metric.certify_radius())
logging.info(metric.sentence_length())
import logging
from args import ClassifierArgs
from classifier import Classifier
if __name__ == '__main__':
args = ClassifierArgs()._parse_args()
print(args)
logging.info(args)
Classifier.run(args)