def main():
clsf = OpenAttack.load("Victim.BiLSTM.SST")
dataset = OpenAttack.load("Dataset.SST.sample")[:10]
attacker = OpenAttack.attackers.GeneticAttacker()
attack_eval = CustomAttackEval(attacker, clsf)
attack_eval.eval(dataset, visualize=True)
def main():
victim = OpenAttack.loadVictim("BERT.SST")
dataset = datasets.load_dataset(
"sst", split="train[:10]").map(function=dataset_mapping)
attacker = MyAttacker()
attack_eval = OpenAttack.AttackEval(attacker, victim)
attack_eval.eval(dataset, visualize=True)
def main():
victim = OpenAttack.loadVictim("BERT.SST")
dataset = datasets.load_dataset(
"sst", split="train[:20]").map(function=dataset_mapping)
attacker = OpenAttack.attackers.PWWSAttacker()
attack_eval = OpenAttack.AttackEval(attacker,
victim,
metrics=[SentenceLength()])
attack_eval.eval(dataset, visualize=True)
def main():
dataset = OpenAttack.loadDataset("SST")[0][:5]
clsf = OpenAttack.loadVictim("BiLSTM.SST")
attackers = get_attackers(dataset, clsf)
for attacker in attackers:
print(attacker.__class__.__name__)
try:
print(
OpenAttack.attack_evals.DefaultAttackEval(
attacker, clsf, progress_bar=False).eval(dataset))
except Exception as e:
print(e)
print("\n")
def main():
print("New Attacker")
attacker = OpenAttack.attackers.PWWSAttacker(lang="chinese")
print("Building model")
victim = OpenAttack.loadVictim("BERT.AMAZON_ZH").to("cuda:0")
print("Loading dataset")
dataset = datasets.load_dataset(
"amazon_reviews_multi", 'zh',
split="train[:20]").map(function=dataset_mapping)
print("Start attack")
attack_eval = OpenAttack.AttackEval(attacker, victim)
attack_eval.eval(dataset, visualize=True, progress_bar=True)
def main():
victim = OpenAttack.loadVictim("BERT.SST")
# Victim.BiLSTM.SST is a pytorch model which is trained on Dataset.SST. It uses Glove vectors for word representation.
# The load operation returns a PytorchClassifier that can be further used for Attacker and AttackEval.
dataset = datasets.load_dataset("sst", split="train[:20]").map(function=dataset_mapping)
# Dataset.SST.sample is a list of 1k sentences sampled from test dataset of Dataset.SST.
attacker = OpenAttack.attackers.GeneticAttacker()
# After this step, we’ve initialized a GeneticAttacker and uses the default configuration during attack process.
attack_eval = OpenAttack.AttackEval(attacker, victim)
# DefaultAttackEval is the default implementation for AttackEval which supports seven basic metrics.
attack_eval.eval(dataset, visualize=True, num_workers=4)
def main():
print("Loading data")
train, valid, test, vocab = prepare_data() # Load dataset
model = make_model(len(vocab)) # Design a victim model
print("Training")
trained_model = train_model(model, train, valid,
vocab) # Train the victim model
print(
"Generating adversarial samples (this step will take dozens of minutes)"
)
clsf = OpenAttack.PytorchClassifier(trained_model,
word2id=vocab) # Wrap the victim model
adversarial_samples = attack(
clsf,
train) # Conduct adversarial attacks and generate adversarial examples
print("Adversarially training classifier")
finetune_model = train_model(
trained_model, train + adversarial_samples, valid,
vocab) # Retrain the classifier with additional adversarial examples
print("Testing enhanced model (this step will take dozens of minutes)")
attack(
clsf, train
) # Re-attack the victim model to measure the effect of adversarial training
def main():
victim = OpenAttack.loadVictim("BERT.SST")
# BERT.SST is a pytorch model which is fine-tuned on SST-2. It uses Glove vectors for word representation.
# The load operation returns a PytorchClassifier that can be further used for Attacker and AttackEval.
dataset = datasets.load_dataset(
"sst", split="train[:20]").map(function=dataset_mapping)
# We load the sst-2 dataset using `datasets` package, and map the fields.
attacker = OpenAttack.attackers.PWWSAttacker()
# After this step, we’ve initialized a PWWSAttacker and uses the default configuration during attack process.
attack_eval = OpenAttack.AttackEval(attacker, victim)
# Use the default implementation for AttackEval which supports seven basic metrics.
attack_eval.eval(dataset, visualize=True)
def main():
print("Load model")
tokenizer = transformers.AutoTokenizer.from_pretrained(
"echarlaix/bert-base-uncased-sst2-acc91.1-d37-hybrid")
model = transformers.AutoModelForSequenceClassification.from_pretrained(
"echarlaix/bert-base-uncased-sst2-acc91.1-d37-hybrid",
num_labels=2,
output_hidden_states=False)
victim = OpenAttack.classifiers.TransformersClassifier(
model, tokenizer, model.bert.embeddings.word_embeddings)
print("New Attacker")
attacker = OpenAttack.attackers.PWWSAttacker()
dataset = datasets.load_dataset(
"sst", split="train[:20]").map(function=dataset_mapping)
print("Start attack")
attack_eval = OpenAttack.AttackEval(
attacker,
victim,
metrics=[
OpenAttack.metric.EditDistance(),
OpenAttack.metric.ModificationRate()
])
attack_eval.eval(dataset, visualize=True)
def __init__(self,
attacker,
clsf,
processor=OpenAttack.DefaultTextProcessor(),
**kwargs):
super().__init__(attacker, clsf, processor=processor, **kwargs)
self.__processor = processor
def main():
# load a fine-tuned sentiment analysis model from Transformers (you can also use our fine-tuned Victim.BERT.SST)
print("Load model")
tokenizer = transformers.AutoTokenizer.from_pretrained(
"echarlaix/bert-base-uncased-sst2-acc91.1-d37-hybrid")
model = transformers.AutoModelForSequenceClassification.from_pretrained(
"echarlaix/bert-base-uncased-sst2-acc91.1-d37-hybrid",
num_labels=2,
output_hidden_states=False)
victim = OpenAttack.classifiers.TransformersClassifier(
model, tokenizer, model.bert.embeddings.word_embeddings)
print("New Attacker")
attacker = OpenAttack.attackers.PWWSAttacker()
# create your dataset here
dataset = datasets.Dataset.from_dict({
"x": ["I hate this movie.", "I like this apple."],
"y": [
0, # 0 for negative
1, # 1 for positive
]
})
print("Start attack")
attack_eval = OpenAttack.AttackEval(
attacker,
victim,
metrics=[
OpenAttack.metric.EditDistance(),
OpenAttack.metric.ModificationRate()
])
attack_eval.eval(dataset, visualize=True)
def attack(classifier, dataset, attacker = OpenAttack.attackers.PWWSAttacker()):
attack_eval = OpenAttack.AttackEval(
attacker,
classifier,
)
correct_samples = [
inst for inst in dataset if classifier.get_pred( [inst["x"]] )[0] == inst["y"]
]
accuracy = len(correct_samples) / len(dataset)
adversarial_samples = {
"x": [],
"y": [],
"tokens": []
}
for result in tqdm.tqdm(attack_eval.ieval(correct_samples), total=len(correct_samples)):
if result["success"]:
adversarial_samples["x"].append(result["result"])
adversarial_samples["y"].append(result["data"]["y"])
adversarial_samples["tokens"].append(tokenizer.tokenize(result["result"], pos_tagging=False))
attack_success_rate = len(adversarial_samples["x"]) / len(correct_samples)
print("Accuracy: %lf%%\nAttack success rate: %lf%%" % (accuracy * 100, attack_success_rate * 100))
return datasets.Dataset.from_dict(adversarial_samples)
def main():
print("Loading chinese processor and substitute")
chinese_processor = OpenAttack.text_processors.ChineseTextProcessor()
chinese_substitute = OpenAttack.substitutes.ChineseCiLinSubstitute()
print("New Attacker")
attacker = OpenAttack.attackers.PWWSAttacker(processor=chinese_processor,
substitute=chinese_substitute,
threshold=None)
print("Building model")
clsf = OpenAttack.loadVictim("BERT.AMAZON_ZH").to("cuda:0")
print("Loading dataset")
dataset = datasets.load_dataset(
"amazon_reviews_multi", 'zh',
split="train[:20]").map(function=dataset_mapping)
print("Start attack")
options = {
"success_rate": True,
"fluency": True,
"mistake": True,
"semantic": True,
"levenstein": True,
"word_distance": True,
"modification_rate": True,
"running_time": True,
}
attack_eval = OpenAttack.attack_evals.ChineseAttackEval(attacker,
clsf,
**options,
num_process=2)
attack_eval.eval(dataset, visualize=True, progress_bar=True)
def main():
def dataset_mapping(x):
return {
"x": x["sentence"],
"y": 1 if x["label"] > 0.5 else 0,
}
print("Word-Level Attacks")
#attacker = OpenAttack.attackers.HotFlipAttacker()
attacker = OpenAttack.attackers.PWWSAttacker()
print("Build model")
victim = make_model()
dataset = datasets.load_dataset(
"sst", split="train[:20]").map(function=dataset_mapping)
print("Start attack")
attack_eval = OpenAttack.AttackEval(
attacker,
victim,
metrics=[
OpenAttack.metric.Fluency(),
OpenAttack.metric.GrammaticalErrors(),
OpenAttack.metric.SemanticSimilarity(),
OpenAttack.metric.EditDistance(),
OpenAttack.metric.ModificationRate()
])
attack_eval.eval(dataset, visualize=True, progress_bar=True)
def main():
import multiprocessing
if multiprocessing.get_start_method() != "spawn":
multiprocessing.set_start_method("spawn", force=True)
dataset = datasets.load_dataset(
"sst", split="train[:100]").map(function=dataset_mapping)
clsf = OpenAttack.loadVictim("BERT.SST").to("cuda:0")
attackers = get_attackers(dataset, clsf)
for attacker in attackers:
print(attacker.__class__.__name__)
time_clsf = TimeCalcClsf(clsf)
try:
st = time.perf_counter()
print(
OpenAttack.attack_evals.DefaultAttackEval(
attacker, time_clsf,
num_process=2).eval(dataset, progress_bar=True),
time_clsf.total_time,
time.perf_counter() - st)
except Exception as e:
raise e
print(e)
print("\n")
def main():
print("New Attacker")
attacker = OpenAttack.attackers.PWWSAttacker(lang="chinese")
print("Building model")
clsf = OpenAttack.loadVictim("BERT.AMAZON_ZH")
print("Loading dataset")
dataset = datasets.load_dataset("amazon_reviews_multi",'zh',split="train[:20]").map(function=dataset_mapping)
print("Start attack")
attack_eval = OpenAttack.AttackEval(attacker, clsf, metrics=[
OpenAttack.metric.Fluency(),
OpenAttack.metric.GrammaticalErrors(),
OpenAttack.metric.EditDistance(),
OpenAttack.metric.ModificationRate()
])
attack_eval.eval(dataset, visualize=True, progress_bar=True)
def main():
print("New Attacker")
attacker = OpenAttack.attackers.PWWSAttacker()
print("Build model")
clsf = OpenAttack.loadVictim("BERT.SST")
dataset = datasets.load_dataset("sst", split="train[:100]").map(function=dataset_mapping)
print("Start attack")
attack_eval = OpenAttack.AttackEval( attacker, clsf, metrics=[
OpenAttack.metric.Fluency(),
OpenAttack.metric.GrammaticalErrors(),
OpenAttack.metric.SemanticSimilarity(),
OpenAttack.metric.EditDistance(),
OpenAttack.metric.ModificationRate()
] )
attack_eval.eval(dataset, visualize=True, progress_bar=True)
def main():
import multiprocessing
if multiprocessing.get_start_method() != "spawn":
multiprocessing.set_start_method("spawn", force=True)
dataset = datasets.load_dataset("sst", split="train[:100]").map(function=dataset_mapping)
clsf = OpenAttack.loadVictim("BERT.SST") # .to("cuda:0")
attackers = get_attackers(dataset, clsf)
for attacker in attackers:
print(attacker.__class__.__name__)
try:
print(
OpenAttack.AttackEval(attacker, clsf).eval(dataset, progress_bar=True),
)
except Exception as e:
raise e
print(e)
print("\n")
def main():
# load some examples of SST-2 for evaluation
dataset = datasets.load_dataset("sst", split="train[:20]").map(function=dataset_mapping)
# choose the costomized classifier as the victim model
victim = MyClassifier()
# choose PWWS as the attacker and initialize it with default parameters
attacker = OpenAttack.attackers.PWWSAttacker()
# prepare for attacking
attack_eval = OpenAttack.AttackEval(attacker, victim)
# launch attacks and print attack results
attack_eval.eval(dataset, visualize=True)
def main():
import multiprocessing
if multiprocessing.get_start_method() != "spawn":
multiprocessing.set_start_method("spawn", force=True)
dataset = OpenAttack.loadDataset("SST")[0][:5]
clsf = OpenAttack.loadVictim("BiLSTM.SST").to("cuda:0")
attackers = get_attackers(dataset, clsf)
for attacker in attackers:
print(attacker.__class__.__name__)
try:
print(
OpenAttack.attack_evals.MultiProcessAttackEval(
attacker, clsf, num_process=2,
progress_bar=False).eval(dataset))
except Exception as e:
raise e
print(e)
print("\n")
def prepare_data():
vocab = {"<UNK>": 0, "<PAD>": 1}
train, valid, test = OpenAttack.loadDataset("SST")
tp = OpenAttack.text_processors.DefaultTextProcessor()
for dataset in [train, valid, test]:
for inst in dataset:
inst.tokens = list(map(lambda x: x[0], tp.get_tokens(inst.x)))
for token in inst.tokens:
if token not in vocab:
vocab[token] = len(vocab)
return train, valid, test, vocab
def main():
# load Dataset.SST.sample for evaluation
dataset = OpenAttack.load("Dataset.SST.sample")[:10]
# choose the costomized classifier as the victim classification model
clsf = MyClassifier()
# choose PWWS as the attacker and initialize it with default parameters
attacker = OpenAttack.attackers.PWWSAttacker()
# prepare for attacking
attack_eval = OpenAttack.attack_evals.DefaultAttackEval(attacker, clsf)
# launch attacks and print attack results
attack_eval.eval(dataset, visualize=True)
def main():
import multiprocessing
if multiprocessing.get_start_method() != "spawn":
multiprocessing.set_start_method("spawn", force=True)
dataset = datasets.load_dataset("amazon_reviews_multi",'zh',split="train[:5]").map(dataset_mapping)
clsf = OpenAttack.loadVictim("BERT.AMAZON_ZH").to("cuda:0")
attackers = get_attackers_on_chinese(dataset, clsf)
for attacker in attackers:
print(attacker.__class__.__name__)
try:
st = time.perf_counter()
print(
OpenAttack.AttackEval(attacker, clsf, language="chinese").eval(dataset, progress_bar=True),
time.perf_counter() - st
)
except Exception as e:
raise e
print(e)
print("\n")
def train_model(model, data_train, data_valid, vocab, num_epoch=10):
mx_acc = None
mx_model = None
for i in range(num_epoch):
loss = train_epoch(model, data_train, vocab)
clsf = OpenAttack.PytorchClassifier(model, word2id=vocab)
accuracy = len(data_valid.eval(clsf).correct()) / len(data_valid)
print("Epoch %d: loss: %lf, accuracy %lf" % (i, loss, accuracy))
if mx_acc is None or mx_acc < accuracy:
mx_model = model.state_dict()
model.load_state_dict(mx_model)
return model
def main():
print("Load model")
tokenizer = transformers.AutoTokenizer.from_pretrained("roberta-large-mnli")
model = transformers.AutoModelForSequenceClassification.from_pretrained("roberta-large-mnli", output_hidden_states=False)
victim = OpenAttack.classifiers.TransformersClassifier(model, tokenizer, model.roberta.embeddings.word_embeddings)
victim = NLIWrapper(victim)
print("New Attacker")
attacker = OpenAttack.attackers.PWWSAttacker()
dataset = datasets.load_dataset("glue", "mnli", split="train[:20]").map(function=dataset_mapping)
print("Start attack")
attack_eval = OpenAttack.AttackEval(attacker, victim, metrics = [
OpenAttack.metric.EditDistance(),
OpenAttack.metric.ModificationRate()
])
attack_eval.eval(dataset, visualize=True)
def main():
print("Loading data")
train, valid, test, vocab = prepare_data() # Load dataset
model = make_model(len(vocab)) # Design a victim model
print("Training")
trained_model = train_model(model, train, valid,
vocab) # Train the victim model
print(
"Generating adversarial samples (this step will take dozens of minutes)"
)
clsf = OpenAttack.PytorchClassifier(trained_model,
word2id=vocab) # Wrap the victim model
adversarial_samples = attack(
clsf,
train) # Conduct adversarial attacks and generate adversarial examples
print("Adversarially training classifier")
print(train.features)
print(adversarial_samples.features)
new_dataset = {"x": [], "y": [], "tokens": []}
for it in train:
new_dataset["x"].append(it["x"])
new_dataset["y"].append(it["y"])
new_dataset["tokens"].append(it["tokens"])
for it in adversarial_samples:
new_dataset["x"].append(it["x"])
new_dataset["y"].append(it["y"])
new_dataset["tokens"].append(it["tokens"])
finetune_model = train_model(
trained_model, datasets.Dataset.from_dict(new_dataset), valid,
vocab) # Retrain the classifier with additional adversarial examples
print("Testing enhanced model (this step will take dozens of minutes)")
attack(
clsf, train
) # Re-attack the victim model to measure the effect of adversarial training
def main():
print("New Attacker")
#attacker = OpenAttack.attackers.PWWSAttacker()
print("Build model")
#clsf = OpenAttack.loadVictim("BERT.SST")
tokenizer = transformers.AutoTokenizer.from_pretrained("./data/Victim.BERT.SST")
model = transformers.AutoModelForSequenceClassification.from_pretrained("./data/Victim.BERT.SST", num_labels=2, output_hidden_states=True)
clsf = OpenAttack.classifiers.TransformersClassifier(model, tokenizer=tokenizer, max_length=100, embedding_layer=model.bert.embeddings.word_embeddings)
dataset = datasets.load_dataset("sst", split="train[:5]").map(function=dataset_mapping)
print("New Attacker")
attacker = OpenAttack.attackers.GEOAttacker(data=dataset)
print("Start attack")
attack_eval = OpenAttack.AttackEval( attacker, clsf, metrics=[
OpenAttack.metric.Fluency(),
OpenAttack.metric.GrammaticalErrors(),
OpenAttack.metric.SemanticSimilarity(),
OpenAttack.metric.EditDistance(),
OpenAttack.metric.ModificationRate()
] )
attack_eval.eval(dataset, visualize=True, progress_bar=True)
def main():
print("Loading chinese processor and substitute")
chinese_processor = OpenAttack.text_processors.ChineseTextProcessor()
chinese_substitute = OpenAttack.substitutes.ChineseCiLinSubstitute()
print("New Attacker")
attacker = OpenAttack.attackers.PWWSAttacker(processor=chinese_processor,
substitute=chinese_substitute,
threshold=None)
print("Building model")
clsf = OpenAttack.loadVictim("BERT.AMAZON_ZH").to("cuda:0")
print("Loading dataset")
dataset = datasets.load_dataset(
"amazon_reviews_multi", 'zh',
split="train[:20]").map(function=dataset_mapping)
print("Start attack")
attack_eval = OpenAttack.attack_evals.ChineseAttackEval(attacker, clsf)
attack_eval.eval(dataset, visualize=True, progress_bar=True)
def main():
import multiprocessing
if multiprocessing.get_start_method() != "spawn":
multiprocessing.set_start_method("spawn", force=True)
dataset = datasets.load_dataset(
'amazon_reviews_multi', 'zh',
split="train[:100]").map(function=dataset_mapping)
clsf = OpenAttack.loadVictim("BERT.AMAZON_ZH").to("cuda:0")
attackers = get_attackers_on_chinese(dataset, clsf)
for attacker in attackers:
print(attacker.__class__.__name__)
time_clsf = TimeCalcClsf(clsf)
try:
st = time.perf_counter()
print(
OpenAttack.attack_evals.MultiProcessAttackEval(
attacker, time_clsf, num_process=2,
progress_bar=False).eval(dataset), time_clsf.total_time,
time.perf_counter() - st)
except Exception as e:
raise e
print(e)
print("\n")
def __init__(self, processor=OpenAttack.DefaultTextProcessor()):
self.processor = processor