def hack(self, instance):
words, tags, chars, arcs, rels = instance
_, raw_metric = self.task.evaluate([(words, tags, chars, arcs, rels)],
mst=self.config.hkw_mst == 'on')
_, raw_arcs, _ = self.task.predict([(words, tags, None)],
mst=self.config.hkw_mst == 'on')
# Setup some states before attacking a sentence
# WARNING: Operations on variables n__global_ramambc__" passed to a function
# are in-placed! Never try to save an internal state of the variable.
forbidden_idxs__ = [self.vocab.unk_index, self.vocab.pad_index]
change_positions__ = set()
orphans__ = set()
if self.config.hkw_max_change > 0.9999:
max_change_num = int(self.config.hkw_max_change)
else:
max_change_num = max(
1, int(self.config.hkw_max_change * words.size(1)))
iter_change_num = min(max_change_num, self.config.hkw_iter_change)
var_words = words.clone()
raw_words = words.clone()
# HIGHLIGHT: ITERATION
t0 = time.time()
picker = CherryPicker(lower_is_better=True,
compare_fn=lambda m1, m2: m1.uas - m2.uas)
# iter 0 -> raw
picker.add(raw_metric, {
"num_changed": 0,
"logtable": 'No modification'
})
for iter_id in range(1, self.config.hkw_steps):
result = self.single_hack(var_words,
tags,
arcs,
rels,
raw_words=raw_words,
raw_metric=raw_metric,
raw_arcs=raw_arcs,
verbose=False,
max_change_num=max_change_num,
iter_change_num=iter_change_num,
iter_id=iter_id,
forbidden_idxs__=forbidden_idxs__,
change_positions__=change_positions__,
orphans__=orphans__)
# Fail
if result['code'] == 404:
log('Stop in step {}, info: {}'.format(iter_id,
result['info']))
break
# Success
if result['code'] == 200:
picker.add(
result['attack_metric'],
{
# "words": result['new_words'],
"logtable": result['logtable'],
"num_changed": len(change_positions__)
})
if result[
'attack_metric'].uas < raw_metric.uas - self.config.hkw_eps:
log('Succeed in step {}'.format(iter_id))
break
var_words = result['words']
# forbidden_idxs__ = result['forbidden_idxs__']
# change_positions__ = result['change_positions__']
t1 = time.time()
best_iter, best_attack_metric, best_info = picker.select_best_point()
return {
"raw_metric": raw_metric,
"attack_metric": best_attack_metric,
"iters": iter_id,
"best_iter": best_iter,
"num_changed": best_info['num_changed'],
"time": t1 - t0,
"logtable": best_info['logtable']
}
def meta_hack(self, instance, sentence):
t0 = time.time()
ram_reset("hk")
words, tags, chars, arcs, rels = instance
sent_len = words.size(1)
words_text = self.vocab.id2word(words[0])
succ = False
if self.config.hks_color == "black":
spans = filter_spans(gen_spans(sentence), self.config.hks_min_span_len,
self.config.hks_max_span_len, True)
hack_result = None
for pair_id, src_span in enumerate(spans):
log('[Chosen source] ', src_span, ' '.join(words_text[src_span[0]:src_span[1] + 1]))
tgt_span_lst = [
ele for ele in spans if check_gap(ele, src_span, self.config.hks_span_gap)
]
if len(tgt_span_lst) == 0:
continue
hack_result = self.random_hack(instance, sentence, tgt_span_lst, src_span)
if hack_result['succ'] == 1:
succ = True
log("Succ on source span {}.\n".format(src_span))
break
log("Fail on source span {}.\n".format(src_span))
if hack_result is None:
log("Not enough span pairs")
return None
elif self.config.hks_color in ['white', 'grey']:
pairs = self.select_span_pair(instance, sentence)
if pairs is None:
return None
for pair_id, (tgt_span, src_span) in enumerate(pairs[:self.config.hks_topk_pair]):
log('[Chosen span] ', 'tgt-', tgt_span,
' '.join(words_text[tgt_span[0]:tgt_span[1] + 1]), 'src-', src_span,
' '.join(words_text[src_span[0]:src_span[1] + 1]))
if self.config.hks_color == "white":
hack_result = self.white_hack(instance, sentence, tgt_span, src_span)
elif self.config.hks_color == "grey":
hack_result = self.random_hack(instance, sentence, [tgt_span], src_span)
else:
raise Exception
if hack_result['succ'] == 1:
succ = True
log("Succ on the span pair {}/{}.\n".format(pair_id, len(pairs)))
break
log("Fail on the span pair {}/{}.\n".format(pair_id, len(pairs)))
hack_result['meta_trial_pair'] = pair_id + 1
hack_result['meta_total_pair'] = len(pairs)
hack_result['meta_succ_trial_pair'] = pair_id + 1 if succ else 0
hack_result['meta_succ_total_pair'] = len(pairs) if succ else 0
t1 = time.time()
log('Sentence cost {:.1f}s'.format(t1 - t0))
hack_result['meta_time'] = t1 - t0
return hack_result
def random_hack(self, instance, sentence, tgt_span_lst, src_span):
t0 = time.time()
words, tags, chars, arcs, rels = instance
sent_len = words.size(1)
raw_words_lst = cast_list(words)
idxs = gen_idxs_to_substitute(list(range(src_span[0], src_span[1] + 1)),
int(self.config.hks_max_change), self.config.hks_cand_num)
if self.config.hks_blk_repl_tag == 'any':
cand_words_lst = subsitute_by_idxs(raw_words_lst, idxs, self.blackbox_sub_idxs)
elif re.match("[njvri]+", self.config.hks_blk_repl_tag):
cand_words_lst = subsitute_by_idxs(raw_words_lst, idxs,
self.njvri_subidxs(self.config.hks_blk_repl_tag))
elif re.match("keep", self.config.hks_blk_repl_tag):
tag_texts = cast_list(tags[0])
vocab_idxs_lst = []
for i in range(src_span[0], src_span[1] + 1):
vocab_idxs_lst.append(self.univ_subidxs(HACK_TAGS.ptb2uni(self.vocab.tags[tag_texts[i]])))
cand_words_lst = subsitute_by_idxs_2(raw_words_lst, idxs, src_span[0], vocab_idxs_lst)
else:
raise Exception
# First index is raw words
all_words = torch.tensor([raw_words_lst] + cand_words_lst, device=words.device)
raw_words = all_words[0:1]
cand_words = all_words[1:]
tgt_idxs = []
for tgt_span in tgt_span_lst:
tgt_idxs.extend([i for i in range(tgt_span[0], tgt_span[1] + 1) if i != tgt_span[2]])
ex_tgt_idxs = [i for i in range(sent_len) if i not in tgt_idxs]
pred_arcs, pred_rels, gold_arcs, gold_rels = self.task.partial_evaluate(
(all_words, tags.expand_as(all_words), None, arcs.expand_as(all_words),
rels.expand_as(all_words)),
mask_idxs=ex_tgt_idxs,
mst=False,
return_metric=False)
# raw_metric = ParserMetric()
# raw_metric(pred_arcs[0], pred_rels[0], gold_arcs[0], gold_rels[0])
raw_metric = self.task.partial_evaluate((raw_words, tags, None, arcs, rels),
ex_tgt_idxs,
mst=True)
succ = False
arc_delta = (pred_arcs[1:] - pred_arcs[0]).abs().sum(1)
if arc_delta.sum() == 0:
# direct forward to the last attacked sentence
att_id = self.config.hks_cand_num - 1
else:
for att_id in range(0, self.config.hks_cand_num):
if arc_delta[att_id] != 0:
att_metric = self.task.partial_evaluate(
(cand_words[att_id:att_id + 1], tags, None, arcs, rels),
ex_tgt_idxs,
mst=True)
if att_metric.uas < raw_metric.uas - 0.0001:
succ = True
break
# ATT_ID will be equal to att_id when failing
t1 = time.time()
_, raw_arcs, raw_rels = self.task.predict([(raw_words, tags, None)], mst=True)
_, att_arcs, att_rels = self.task.predict([(cand_words[att_id:att_id + 1], tags, None)],
mst=True)
if not succ:
info = 'Nothing'
else:
for tgt_span in tgt_span_lst:
if tgt_span == src_span:
continue
raw_span_corr = 0
att_span_corr = 0
for i in range(tgt_span[0], tgt_span[1] + 1):
if i == tgt_span[2]:
continue
if raw_arcs[0][i - 1] == arcs[0][i - 1]:
raw_span_corr += 1
if att_arcs[0][i - 1] == arcs[0][i - 1]:
att_span_corr += 1
if att_span_corr < raw_span_corr:
break
info = tabulate(self._gen_log_table(words, cand_words[att_id:att_id + 1], tags, arcs,
rels, raw_arcs, att_arcs, src_span, tgt_span),
floatfmt='.6f')
return defaultdict(
lambda: -1, {
"succ": 1 if succ else 0,
"att_id": att_id if att_id < self.config.hks_cand_num - 1 else np.nan,
"num_changed": self.config.hks_max_change,
"time": t1 - t0,
"logtable": info
})
def white_hack(self, instance, sentence, tgt_span, src_span):
words, tags, chars, arcs, rels = instance
sent_len = words.size(1)
raw_words = words.clone()
var_words = words.clone()
raw_metric = self.task.partial_evaluate(instance=(raw_words, tags, None, arcs, rels),
mask_idxs=ex_span_idx(tgt_span, sent_len),
mst=self.config.hks_mst == 'on')
_, raw_arcs, _ = self.task.predict([(raw_words, tags, None)])
forbidden_idxs__ = [self.vocab.unk_index, self.vocab.pad_index]
change_positions__ = set()
if self.config.hks_max_change > 0.9999:
max_change_num = int(self.config.hks_max_change)
else:
max_change_num = max(1, int(self.config.hks_max_change * words.size(1)))
iter_change_num = min(max_change_num, self.config.hks_iter_change)
picker = CherryPicker(lower_is_better=True)
t0 = time.time()
picker.add(raw_metric, {"num_changed": 0, "logtable": 'No modification'})
log('iter -1, uas {:.4f}'.format(raw_metric.uas))
succ = False
for iter_id in range(self.config.hks_steps):
result = self.single_hack(instance=(var_words, tags, None, arcs, rels),
raw_words=raw_words,
raw_metric=raw_metric,
raw_arcs=raw_arcs,
src_span=src_span,
tgt_span=tgt_span,
iter_id=iter_id,
forbidden_idxs__=forbidden_idxs__,
change_positions__=change_positions__,
max_change_num=max_change_num,
iter_change_num=iter_change_num)
if result['code'] == 200:
var_words = result['words']
picker.add(result['attack_metric'], {
'logtable': result['logtable'],
"num_changed": len(change_positions__)
})
if result['attack_metric'].uas < raw_metric.uas - 0.00001:
succ = True
log('Succeed in step {}'.format(iter_id))
break
elif result['code'] == 404:
log('FAILED')
break
t1 = time.time()
best_iter, best_attack_metric, best_info = picker.select_best_point()
return defaultdict(
lambda: -1, {
"succ": 1 if succ else 0,
"raw_metric": raw_metric,
"attack_metric": best_attack_metric,
"iters": iter_id,
"best_iter": best_iter,
"num_changed": best_info['num_changed'],
"time": t1 - t0,
"logtable": best_info['logtable']
})
def hack(self, instance):
words, tags, chars, arcs, rels = instance
_, raw_metric = self.task.evaluate([(words, tags, chars, arcs, rels)],
mst=self.config.hkc_mst == 'on')
_, raw_arcs, _ = self.task.predict([(words, tags, chars)],
mst=self.config.hkc_mst == 'on')
# char_grads, word_grad = self.backward_loss(words, chars, arcs, rels)
forbidden_idxs__ = defaultdict(
lambda: deque(maxlen=5)) # word_sid -> deque()
change_positions__ = dict() # word_sid -> char_wid
if self.config.hkc_max_change > 0.9999:
max_change_num = int(self.config.hkc_max_change)
else:
max_change_num = max(
1, int(self.config.hkc_max_change * words.size(1)))
iter_change_num = min(max_change_num, self.config.hkc_iter_change)
raw_chars = chars.clone()
var_chars = chars.clone()
# HIGHLIGHT: ITERATION
t0 = time.time()
picker = CherryPicker(lower_is_better=True,
compare_fn=lambda m1, m2: m1.uas - m2.uas)
# iter 0 -> raw
picker.add(raw_metric, {
"num_changed": 0,
"logtable": 'No modification'
})
for iter_id in range(1, self.config.hkc_steps):
result = self.single_hack(
words, tags, var_chars, arcs, rels,
raw_chars=raw_chars, raw_metric=raw_metric, raw_arcs=raw_arcs,
verbose=False,
max_change_num=max_change_num,
iter_change_num=iter_change_num,
iter_id=iter_id,
forbidden_idxs__=forbidden_idxs__,
change_positions__=change_positions__,
)
# Fail
if result['code'] == 404:
log('Stop in step {}, info: {}'.format(
iter_id, result['info']))
break
# Success
if result['code'] == 200:
picker.add(result['attack_metric'], {
# "words": result['new_words'],
"logtable": result['logtable'],
"num_changed": len(change_positions__)
})
if result['attack_metric'].uas < raw_metric.uas - self.config.hkw_eps:
log('Succeed in step {}'.format(iter_id))
break
var_chars = result['chars']
t1 = time.time()
best_iter, best_attack_metric, best_info = picker.select_best_point()
return {
"raw_metric": raw_metric,
"attack_metric": best_attack_metric,
"iters": iter_id,
"best_iter": best_iter,
"num_changed": best_info['num_changed'],
"time": t1 - t0,
"logtable": best_info['logtable']
}
def hack(self, instance, sentence):
words, tags, chars, arcs, rels = instance
sent_len = words.size(1)
spans = gen_spans(sentence)
valid_spans = list(filter(lambda ele: 5 <= ele[1] - ele[0] <= 8,
spans))
if len(valid_spans) == 0:
log("Attack error, no valid spans".format())
return None
chosen_span = random.choice(valid_spans)
# chosen_span = (18, 24)
if chosen_span[1] - chosen_span[0] + 1 > 0.5 * sent_len:
return None
# sent_print(sentence, 'tablev')
# print('spans', spans)
# print('valid', valid_spans)
log(
'chosen span: ', chosen_span, ' '.join(
self.vocab.id2word(words[0])[chosen_span[0]:chosen_span[1] +
1]))
raw_words = words.clone()
var_words = words.clone()
words_text = self.vocab.id2word(words[0])
tags_text = self.vocab.id2tag(tags[0])
# eval_idxs = [eval_idx for eval_idx in range(sent_len)
# if not chosen_span[0] <= eval_idx <= chosen_span[1]]
mask_idxs = list(range(chosen_span[0], chosen_span[1] + 1))
raw_metric = self.task.partial_evaluate(instance=(raw_words, tags,
None, arcs, rels),
mask_idxs=mask_idxs)
_, raw_arcs, _ = self.task.predict([(raw_words, tags, None)])
forbidden_idxs__ = [self.vocab.unk_index, self.vocab.pad_index]
change_positions__ = set()
if isinstance(self.config.hko_max_change, int):
max_change_num = self.config.hko_max_change
elif isinstance(self.config.hk_max_change, float):
max_change_num = int(self.config.hk_max_change * words.size(1))
else:
raise Exception("hk_max_change must be a float or an int")
picker = CherryPicker(lower_is_better=True)
t0 = time.time()
for iter_id in range(self.config.hko_steps):
result = self.single_hack(instance=(var_words, tags, None, arcs,
rels),
raw_words=raw_words,
raw_metric=raw_metric,
raw_arcs=raw_arcs,
mask_idxs=mask_idxs,
iter_id=iter_id,
forbidden_idxs__=forbidden_idxs__,
change_positions__=change_positions__,
max_change_num=max_change_num)
if result['code'] == 200:
var_words = result['words']
picker.add(
result['attack_metric'], {
'logtable': result['logtable'],
"num_changed": len(change_positions__)
})
elif result['code'] == 404:
print('FAILED')
break
t1 = time.time()
best_iter, best_attack_metric, best_info = picker.select_best_point()
return {
"raw_metric": raw_metric,
"attack_metric": best_attack_metric,
"iters": iter_id,
"best_iter": best_iter,
"num_changed": best_info['num_changed'],
"time": t1 - t0,
"logtable": best_info['logtable']
}