def attack_run(master_settings):
"""
"""
align = master_settings["align"]
loss = master_settings["loss"]
decoder = master_settings["decoder"]
outdir = master_settings["outdir"]
attack_type = os.path.basename(__file__).replace(".py", "")
outdir = os.path.join(outdir, attack_type)
outdir = os.path.join(outdir, "baselines/biggio/")
outdir = os.path.join(outdir, "{}/".format(align))
outdir = os.path.join(outdir, "{}/".format(loss))
outdir = os.path.join(outdir, "{}/".format(decoder))
master_settings["outdir"] = outdir
batch_gen = data.ingress.etl.batch_generators.PATH_GENERATORS[align](
master_settings)
default_manager(
master_settings,
create_attack_graph,
batch_gen,
)
log("Finished run.")
def attack_run(master_settings):
"""
"""
align = master_settings["align"]
decoder = master_settings["decoder"]
kappa = master_settings["kappa"]
outdir = master_settings["outdir"]
attack_type = os.path.basename(__file__).replace(".py", "")
outdir = os.path.join(outdir, attack_type)
outdir = os.path.join(outdir, "confidence/invertedctc-cwmaxdiff/")
outdir = os.path.join(outdir, "{}/".format(align))
outdir = os.path.join(outdir, "{}/".format(decoder))
outdir = os.path.join(outdir, "{}/".format(kappa))
master_settings["outdir"] = outdir
batch_gen = data.ingress.etl.batch_generators.PATH_GENERATORS[align](
master_settings)
default_manager(
master_settings,
create_attack_graph,
batch_gen,
)
log("Finished run.")
def attack_run(master_settings):
"""
CTC Loss attack modified from the original Carlini & Wagner work.
"""
loss = master_settings["loss"]
decoder = master_settings["decoder"]
outdir = master_settings["outdir"]
attack_type = os.path.basename(__file__).replace(".py", "")
outdir = os.path.join(outdir, attack_type)
outdir = os.path.join(outdir, "baselines/ctc/")
outdir = os.path.join(outdir, "{}/".format(loss))
outdir = os.path.join(outdir, "{}/".format(decoder))
master_settings["outdir"] = outdir
batch_gen = data.ingress.etl.batch_generators.standard(master_settings)
default_manager(
master_settings,
create_attack_graph,
batch_gen,
)
log("Finished run.")
def attack_run(master_settings):
align = master_settings["align"]
decoder = master_settings["decoder"]
outdir = master_settings["outdir"]
attack_type = os.path.basename(__file__).replace(".py", "")
outdir = os.path.join(outdir, attack_type)
outdir = os.path.join(outdir, "confidence/weightedmaxmin/")
outdir = os.path.join(outdir, "{}/".format(align))
outdir = os.path.join(outdir, "{}/".format(decoder))
master_settings["outdir"] = outdir
batch_gen = data.ingress.etl.batch_generators.PATH_GENERATORS[align](
master_settings)
manager(
master_settings,
create_attack_graph,
batch_gen,
results_extract_fn=custom_extract_results,
results_transform_fn=data.egress.transform.evasion_gen,
)
log("Finished run.")
def attack_run(master_settings):
align = master_settings["align"]
decoder = master_settings["decoder"]
loss = master_settings["loss"]
kappa = master_settings["kappa"]
outdir = master_settings["outdir"]
attack_type = os.path.basename(__file__).replace(".py", "")
outdir = os.path.join(outdir, attack_type)
outdir = os.path.join(outdir, "confidence/logprobs-greedydiff/")
outdir = os.path.join(outdir, "{}/".format(align))
outdir = os.path.join(outdir, "{}/".format(decoder))
outdir = os.path.join(outdir, "{}/".format(loss))
outdir = os.path.join(outdir, "{}/".format(kappa))
master_settings["outdir"] = outdir
batch_gen = data.ingress.etl.batch_generators.PATH_GENERATORS[align](
master_settings)
manager(
master_settings,
create_attack_graph,
batch_gen,
results_extract_fn=custom_extract_results,
results_transform_fn=data.egress.transform.unbounded_gen,
)
log("Finished run.")
def attack_run(master_settings):
"""
"""
align = master_settings["align"]
decoder = master_settings["decoder"]
procedure = master_settings["procedure"]
loss_threshold = master_settings["loss_threshold"]
outdir = master_settings["outdir"]
attack_type = os.path.basename(__file__).replace(".py", "")
outdir = os.path.join(outdir, attack_type)
outdir = os.path.join(outdir, "confidence/ctc-edge-case/")
outdir = os.path.join(outdir, "{}/".format(align))
outdir = os.path.join(outdir, "{}/".format(decoder))
outdir = os.path.join(outdir, "{}/".format(procedure))
if procedure == "extreme":
outdir = os.path.join(outdir, "{}/".format(loss_threshold))
master_settings["outdir"] = outdir
batch_gen = data.ingress.etl.batch_generators.PATH_GENERATORS[align](
master_settings)
default_manager(
master_settings,
create_attack_graph,
batch_gen,
)
log("Finished run.")
def writer_boilerplate_fn(results_transforms, queue, settings):
import traceback
settings_writer = settings["writer"].split("_")[0]
SETTINGS_WRITER_FUNCS[settings_writer](settings["outdir"], settings)
while True:
results = queue.get()
if results == "dead":
queue.task_done()
break
else:
try:
for example in results_transforms(results, settings):
if example["success"] is True:
RESULTS_WRITER_FUNCS[settings["writer"]](
settings["outdir"], example)
except Exception as e:
tb = "".join(
traceback.format_exception(None, e, e.__traceback__))
s = "Something broke during file writes!"
s += "\n\nError Traceback:\n{e}".format(e=tb)
log(s, wrap=True)
raise
finally:
queue.task_done()
def standard(settings):
# get N samples of all the data. alsp make sure to limit example length,
# otherwise we'd have to do adaptive batch sizes.
audio_file_paths_pool = etls.get_audio_file_path_pool(
settings["audio_indir"],
settings["max_examples"],
filter_term=".wav",
max_file_size=settings["max_audio_file_bytes"]
)
targets_pool = etls.get_target_phrase_pool(
settings["targets_path"], settings["max_targets"],
)
# Generate the batches in turn, rather than all in one go ...
# ... To save resources by only running the final ETLs on a batch of data
total_numb_examples = len(audio_file_paths_pool)
batch_size = settings["batch_size"]
log(
"New Run",
"Number of test examples: {}".format(total_numb_examples),
''.join(
["{k}: {v}\n".format(k=k, v=v) for k, v in settings.items()]),
)
for idx in range(0, total_numb_examples, batch_size):
# Handle remainders: number of examples // desired batch size != 0
if len(audio_file_paths_pool) < settings["batch_size"]:
batch_size = len(audio_file_paths_pool)
# get n files paths and create the audio batch data
audio_batch_data = BatchGen.popper(audio_file_paths_pool, batch_size)
audios_batch = etls.create_audio_batch_from_wav_files(audio_batch_data)
# load the correct transcription for the audio files
trues_batch = etls.create_true_batch(audios_batch)
targets_batch = etls.create_standard_target_batch(
targets_pool,
batch_size,
trues_batch,
audios_batch,
)
batch = Batch(
batch_size,
audios_batch,
targets_batch,
trues_batch,
)
yield idx, batch
def evasion_gen(results, settings):
for example_data in evasion_transforms(results):
log(
evasion_logging(example_data),
wrap=False,
outdir=settings["outdir"],
stdout=False,
timings=True,
)
yield example_data
def unbounded_gen(results, settings):
for example_data in unbounded_transforms(results):
log(
unbounded_logging(example_data),
wrap=False,
outdir=settings["outdir"],
stdout=False,
timings=True,
)
yield example_data
def attack_run(master_settings):
batch_gen = data.ingress.etl.batch_generators.standard(master_settings)
custom_manager(
master_settings,
create_validation_graph,
batch_gen,
)
log("Finished all runs.")
def main(search_type):
# Create the factory we'll use to iterate over N examples at a time.
attack_spawner = AttackSpawner(gpu_device=GPU_DEVICE,
max_processes=MAX_PROCESSES,
delay=SPAWN_DELAY,
file_writer=None)
with attack_spawner as spawner:
all_audio_file_paths = etls.get_audio_file_path_pool(
INDIR,
AUDIO_EXAMPLES_POOL,
file_size_sort="desc",
filter_term=".wav",
max_samples=MAX_AUDIO_LENGTH)
all_transcriptions = etls.get_target_phrase_pool(
TARGETS_PATH, TARGETS_POOL)
log("New Run")
for idx in range(0, AUDIO_EXAMPLES_POOL, BATCH_SIZE):
# get n files paths and create the audio batch data
audio_batch_data = utils.BatchGen.popper(all_audio_file_paths,
BATCH_SIZE)
audios_batch = etls.create_audio_batch(audio_batch_data)
# we need to make sure target phrase length < n audio feats.
# also, the first batch should also have the longest target phrase
# and longest audio examples so we can easily manage GPU Memory
# resources with the AttackSpawner context manager.
target_phrase = utils.BatchGen.pop_target_phrase(
all_transcriptions, min(audios_batch["real_feats"]))
# each target must be the same length else numpy throws a hissyfit
# because it can't understand skewed matrices
target_batch_data = l_map(lambda _: target_phrase,
range(BATCH_SIZE))
# actually load the n phrases as a batch of target data
targets_batch = etls.create_standard_target_batch(
target_batch_data)
batch = Batch(
BATCH_SIZE,
audios_batch,
targets_batch,
)
spawner.spawn(run, batch, search_type)
def write_results(result):
# TODO: skip later repeats if *NOTHING* was successful
# store the db_outputs in a json file with the
# absolute filepath *and* the original example as it
# makes loading data for the actual optimisation attack
# a hell of a lot easier
if not os.path.exists(OUTDIR):
os.mkdir(OUTDIR)
# write out for each repeat value *and* the last success (is most confident)
db_path = "{b}_targid-{t}_rep-{r}".format(
b=result["audio_basename"].rstrip(".wav"),
t=result["targ_id"],
r=result["repeats"],
)
example_db = SingleJsonDB(OUTDIR)
example_db.open(db_path).put(result)
db_path = "{b}_targid-{t}_latest".format(
b=result["audio_basename"].rstrip(".wav"),
t=result["targ_id"],
)
example_db = SingleJsonDB(OUTDIR)
example_db.open(db_path).put(result)
# log how we've done
s = "Success: {b} for {r} repeats and target {t}".format(
b=result["audio_basename"],
r=result["repeats"],
t=result["targ_id"],
)
s += " kappa: {k:.3f} orig score p.c.: {o:.1f} new score p.c.: {n:.1f}".format(
k=result["kappa"],
o=result["original_spc"],
n=result["spc"],
)
s += " orig score : {o:.1f} new score: {n:.1f}".format(
o=result["original_score"],
n=result["score"],
)
s += " logits diff: {:.0f}".format(
np.abs(np.sum(result["original_logits"] - result["new_logits"])))
s += " entropy: {}".format(entropy(result["new_softmax"]))
s += " Wrote targeting data."
log(s, wrap=False)
def __init__(self, attack, alignment=None, weight_settings=(1.0, 1.0)):
super().__init__(
attack.sess,
attack.batch.size,
weight_settings=weight_settings,
)
seq_lengths = attack.batch.audios["ds_feats"]
if alignment is not None:
log("Using CTC alignment search.", wrap=True)
self.ctc_target = tf.keras.backend.ctc_label_dense_to_sparse(
alignment,
attack.batch.audios["ds_feats"],
)
else:
log("Using repeated alignment.", wrap=True)
self.ctc_target = tf.keras.backend.ctc_label_dense_to_sparse(
attack.placeholders.targets,
attack.placeholders.target_lengths,
)
logits_shape = attack.victim.raw_logits.get_shape().as_list()
blank_token_pad = tf.zeros(
[logits_shape[0], logits_shape[1], 1],
tf.float32
)
logits_mod = tf.concat(
[attack.victim.raw_logits, blank_token_pad],
axis=2
)
self.loss_fn = tf.nn.ctc_loss(
labels=tf.cast(self.ctc_target, tf.int32),
inputs=logits_mod,
sequence_length=seq_lengths,
preprocess_collapse_repeated=False,
ctc_merge_repeated=False,
) * self.weights
def attack_run(master_settings):
"""
Use Carlini & Wagner's improved loss function form the original audio paper,
but reintroduce kappa from the image attack as we're looking to perform
targeted maximum-confidence evasion attacks --- i.e. not just find minimum
perturbations.
:param master_settings: a dictionary of arguments to run the attack, as
defined by command line arguments. Will override the settings dictionary
defined below.
:return: None
"""
align = master_settings["align"]
loss = master_settings["loss"]
decoder = master_settings["decoder"]
kappa = master_settings["kappa"]
outdir = master_settings["outdir"]
attack_type = os.path.basename(__file__).replace(".py", "")
outdir = os.path.join(outdir, attack_type)
outdir = os.path.join(outdir, "baselines/cwmaxdiff/")
outdir = os.path.join(outdir, "{}/".format(align))
outdir = os.path.join(outdir, "{}/".format(loss))
outdir = os.path.join(outdir, "{}/".format(decoder))
outdir = os.path.join(outdir, "{}/".format(kappa))
master_settings["outdir"] = outdir
master_settings["attack type"] = attack_type
batch_gen = data.ingress.etl.batch_generators.PATH_GENERATORS[align](
master_settings)
default_manager(
master_settings,
create_attack_graph,
batch_gen,
)
log("Finished run.")
def custom_manager(settings, attack_fn, batch_gen):
for b_id, batch in batch_gen:
attack_process = mp.Process(target=custom_executor,
args=(settings, batch, attack_fn))
try:
attack_process.start()
attack_process.join()
except Exception as e:
tb = "".join(traceback.format_exception(None, e, e.__traceback__))
s = "Something broke! Attack failed to run for these examples:\n"
s += '\n'.join(batch.audios["basenames"])
s += "\n\nError Traceback:\n{e}".format(e=tb)
log(s, wrap=True)
attack_process.terminate()
raise
def create_ctcalign_target_batch_from_standard(data):
"""
:param data: a full starter batch generated by batch_gen.standard
:return: a new batch of target data
"""
target_phrases = data.targets["phrases"]
target_ids = data.targets["row_ids"]
orig_indices = data.targets["indices"]
tokens = data.targets["tokens"]
log("Searching for high likelihood CTC alignments...", wrap=False)
results = utils.subprocess_ctcalign_search(data)
if results == "dead":
raise NoValidCTCAlignmentException(
"Could not find any optimal CTC alignments for you..."
)
else:
log(
"Found CTC alignments, continuing to initialise the attack...",
wrap=True
)
target_alignments = np.asarray(results, dtype=np.int32)
lengths = l_map(
lambda x: x.size,
target_alignments
)
return {
"tokens": tokens,
"phrases": target_phrases,
"row_ids": target_ids,
"indices": target_alignments,
"original_indices": orig_indices,
"lengths": lengths,
}
def executor_boilerplate_fn(extract_fn, results_queue, settings, batch,
attack_fn):
# tensorflow sessions can't be passed between processes
tf_runtime = TFRuntime(settings["gpu_device"])
with tf_runtime.session as sess, tf_runtime.device as tf_device:
# Initialise attack graph constructor function
attack = attack_fn(sess, batch, settings)
# log some useful things for debugging before the attack runs
attack.validate()
s = "Created Attack Graph and Feeds. Loaded TF Operations:"
log(s, wrap=False)
log(funcs=tf_runtime.log_attack_tensors)
s = "Beginning attack run...\nMonitor progress in: {}".format(
settings["outdir"] + "log.txt")
log(s)
for is_results_step in attack.run():
if is_results_step:
res = extract_fn(attack)
results_queue.put(res)
def write_to_csv(stats_data, out_file_path):
flat_row = flatten_dict(stats_data)
headers = [".".join(map(str, key)) for key in flat_row.keys()]
row_data = [str(flat_row.get(key, "")) for key in flat_row.keys()]
if not os.path.exists(out_file_path):
write_headers = True
else:
write_headers = False
with open(out_file_path, 'a+') as outfile:
writer = csv.writer(outfile)
if write_headers:
writer.writerow(headers)
log("Wrote headers.", wrap=False)
writer.writerow(row_data)
return True
def attack_run(master_settings):
graph_type = master_settings["graph"]
decoder = master_settings["decoder"]
loss = master_settings["loss"]
nbatch_max = master_settings["nbatch_max"]
nbatch_step = master_settings["nbatch_step"]
initial_outdir = master_settings["outdir"]
assert nbatch_max >= 1
assert nbatch_step >= 1
assert nbatch_max >= nbatch_step
for batch_size in range(0, nbatch_max + 1, nbatch_step):
if batch_size == 0:
batch_size = 1
outdir = os.path.join(initial_outdir, "unbounded/batch-vs-indy/")
outdir = os.path.join(outdir, "{}/".format(graph_type))
outdir = os.path.join(outdir, "{}/".format(decoder))
outdir = os.path.join(outdir, "{}/".format(loss))
outdir = os.path.join(outdir, "{}/".format(batch_size))
master_settings["outdir"] = outdir
master_settings["batch_size"] = batch_size
master_settings["max_examples"] = batch_size
batch_gen = data.ingress.etl.batch_generators.standard(master_settings)
default_manager(
master_settings,
create_attack_graph,
batch_gen,
)
log("Finished batch run {}.".format(batch_size))
log("Finished all runs.")
def custom_executor(settings, batch, model_fn):
tf_runtime = TFRuntime(settings["gpu_device"])
with tf_runtime.session as sess, tf_runtime.device as tf_device:
model = model_fn(sess, batch, settings)
decodings, probs = model.inference(
batch,
feed=model.feeds.examples,
)
logits, softmax = sess.run(
[tf.transpose(model.raw_logits, [1, 0, 2]), model.logits],
feed_dict=model.feeds.examples,
)
print(logits.shape)
outdir = os.path.join(settings["outdir"], "validation")
if not os.path.exists(outdir):
os.makedirs(outdir, exist_ok=True)
for idx, basename in enumerate(batch.audios["basenames"]):
outpath = os.path.join(outdir, basename.rstrip(".wav") + ".json")
res = {
"decoding": decodings[idx],
"probs": probs[idx],
"logits": logits[idx],
"softmax": softmax[idx],
}
json_res = data.egress.load.prepare_json_data(res)
with open(outpath, "w+") as f:
f.write(json_res)
log("Ran validation for example {}".format(idx), wrap=False)
def attack_run(master_settings):
"""
"""
loss = master_settings["loss"]
outdir = master_settings["outdir"]
attack_type = os.path.basename(__file__).replace(".py", "")
outdir = os.path.join(outdir, attack_type)
outdir = os.path.join(outdir, "confidence/maxctc-mintruectc/")
outdir = os.path.join(outdir, "{}/".format(loss))
master_settings["outdir"] = outdir
batch_gen = data.ingress.etl.batch_generators.standard(master_settings)
default_manager(
master_settings,
create_attack_graph,
batch_gen,
)
log("Finished run.") # {}.".format(run))
def attack_run(master_settings):
"""
Special variant of Carlini & Wagner's improved loss function from the
original audio paper, with kappa as a vector of frame-wise differences
between max(other_classes) and min(other_classes).
:param master_settings: a dictionary of arguments to run the attack, as
defined by command line arguments. Will override the settings dictionary
defined below.
:return: None
"""
align = master_settings["align"]
decoder = master_settings["decoder"]
kappa = master_settings["kappa"]
outdir = master_settings["outdir"]
attack_type = os.path.basename(__file__).replace(".py", "")
outdir = os.path.join(outdir, attack_type)
outdir = os.path.join(outdir, "confidence/adaptive-kappa/")
outdir = os.path.join(outdir, "{}/".format(align))
outdir = os.path.join(outdir, "{}/".format(decoder))
outdir = os.path.join(outdir, "{}/".format(kappa))
master_settings["outdir"] = outdir
batch_gen = data.ingress.etl.batch_generators.PATH_GENERATORS[align](
master_settings)
default_manager(
master_settings,
create_attack_graph,
batch_gen,
)
log("Finished run.")
def validate(self):
"""
Do an initial decoding to verify everything is working
"""
decodings, probs = self.victim.inference(
self.batch,
feed=self.feeds.examples,
decoder="batch"
)
z = zip(self.batch.audios["basenames"], probs, decodings)
s = ["{}\t{:.3f}\t{}".format(b, p, d) for b, p, d in z]
log("Initial decodings:", '\n'.join(s), wrap=True)
s = ["{:.0f}".format(x) for x in self.batch.audios["real_feats"]]
log("Real Features: ", "\n".join(s), wrap=True)
s = ["{:.0f}".format(x) for x in self.batch.audios["ds_feats"]]
log("DS Features: ", "\n".join(s), wrap=True)
s = ["{:.0f}".format(x) for x in self.batch.audios["n_samples"]]
log("Real Samples: ", "\n".join(s), wrap=True)
def attack_run(master_settings):
everything_is_okay = True
batch_size = 1
while everything_is_okay:
master_settings["batch_size"] = batch_size
master_settings["max_examples"] = batch_size
master_settings["nsteps"] = 10
master_settings["decode_step"] = 5
if master_settings["graph"] == "ctc":
batch_gen = data.ingress.etl.batch_generators.standard(
master_settings)
attack_graph = create_ctc_attack_graph
elif master_settings["graph"] == "cw":
batch_gen = data.ingress.etl.batch_generators.sparse(
master_settings)
attack_graph = create_cw_attack_graph
else:
raise NotImplementedError
if batch_size >= 1024:
batch_size = 1024
everything_is_okay = False
try:
log("testing for batch size: {}".format(batch_size), wrap=True)
custom_manager(
master_settings,
attack_graph,
batch_gen,
)
except tf_errors.ResourceExhaustedError as e:
everything_is_okay = False
batch_size = batch_size // 2
except AssertionError as e:
everything_is_okay = False
batch_size = batch_size // 2
else:
batch_size *= 2
log("biggest batch size: {}".format(batch_size), wrap=True)
log("Finished all runs.")
def custom_manager(settings, attack_fn, batch_gen):
for b_id, batch in batch_gen:
attack_process = mp.Process(
target=custom_executor,
args=(settings, batch, attack_fn)
)
try:
attack_process.start()
while attack_process.is_alive():
os_mem = os.popen('free -t').readlines()[-1].split()[1:]
tot_m, used_m, free_m = map(int, os_mem)
assert free_m > 0.1 * tot_m
except tf_errors.ResourceExhaustedError as e:
log("Attack failed due to GPU memory constraints.")
attack_process.terminate()
raise
except AssertionError as e:
log("Attack failed due to CPU memory constraints.")
attack_process.terminate()
raise
except Exception as e:
log("Attack failed due to some code problem.")
tb = "".join(traceback.format_exception(None, e, e.__traceback__))
s = "Something broke! Attack failed to run for these examples:\n"
s += '\n'.join(batch.audios["basenames"])
s += "\n\nError Traceback:\n{e}".format(e=tb)
log(s, wrap=True)
attack_process.terminate()
raise
def generate_stats_file(indir):
stats_out_filepath = os.path.join(indir, "stats.csv")
example_json_results_file_paths = [
fp for fp in get_fps(indir) if "sample" in fp and ".json" in fp
]
s = "Found {n} results files in director {d}... Processing now.".format(
n=len(example_json_results_file_paths), d=indir)
log(s)
ds = start_deepspeech_package_model()
# settings = load_settings_file(indir)
for idx, json_file_path in enumerate(example_json_results_file_paths):
# ==== EXTRACT
metadata = get_file_metadata(json_file_path)
with open(json_file_path, "r") as in_f:
data = json.load(in_f)[0]
log("Loaded: {}".format(json_file_path), wrap=False, timings=True)
# ==== TRANSFORM
audio_data, weights, audio_file_names = preprocess_audio_data(
data["deltas"],
data["audio"],
data["advs"],
)
client_decodings = OrderedDict([(k, ds.stt(audio_data[k]["none"],
16000).replace(" ", "="))
for k in audio_file_names])
decodings = OrderedDict([("client", client_decodings),
("cleverspeech",
OrderedDict([
("decoding", data["decodings"][0]),
("target", data["phrases"][0]),
("log_probs", data["probs"][0]),
]))])
misc_data = OrderedDict([
("step", data["step"][0]),
("loss", data["total_loss"][0]),
("n_samples", data["n_samples"][0]),
("real_feats", data["real_feats"][0]),
("bounds",
OrderedDict([
("raw", data["bounds_raw"][0]),
("eps", data["bounds_eps"][0]),
("initital", data["initial_taus"][0]),
])),
("distances",
OrderedDict([
("raw", data["distances_raw"][0]),
("eps", data["distances_eps"][0]),
])),
("decode", decodings),
])
def calc_lnorm(d, norm_int, weight):
for audio_file in audio_file_names:
value = DetectionMetrics.lnorm(d[audio_file][weight],
norm=norm_int)
yield audio_file, value
lnorm_keys = OrderedDict([
("l1", 1),
("l2", 2),
("linf", np.inf),
])
l_norms = OrderedDict([
(norm_str,
OrderedDict([
(weight,
OrderedDict([
(file_k, v)
for file_k, v in calc_lnorm(audio_data, norm_int, weight)
])) for weight in weights
])) for norm_str, norm_int in lnorm_keys.items()
])
snr_analysis_fns = OrderedDict([
("snr_energy_db", DetectionMetrics.snr_energy_db),
("snr_energy", DetectionMetrics.snr_energy),
("snr_pow_db", DetectionMetrics.snr_power_db),
("snr_pow", DetectionMetrics.snr_power),
("snr_seg_db", DetectionMetrics.snr_segmented),
])
snr_stats = OrderedDict([
(snr_key,
OrderedDict([(weight,
snr_fn(audio_data["deltas"][weight],
audio_data["originals"][weight]))
for weight in weights]))
for snr_key, snr_fn in snr_analysis_fns.items()
])
dsp_analysis_fns = OrderedDict([
("rms_amp_db", DetectionMetrics.rms_amplitude_db),
("rms_amp", DetectionMetrics.rms_amplitude),
("energy_db", DetectionMetrics.energy_db),
("energy", DetectionMetrics.energy),
("power_db", DetectionMetrics.power_db),
("power", DetectionMetrics.power),
])
dsp_stats = OrderedDict([
(dsp_key,
OrderedDict([(weight,
OrderedDict([
(audio_file,
dsp_fn(audio_data[audio_file][weight]))
for audio_file in audio_file_names
])) for weight in weights]))
for dsp_key, dsp_fn in dsp_analysis_fns.items()
])
stats = metadata
stats.update(misc_data)
stats.update(l_norms)
stats.update(snr_stats)
stats.update(dsp_stats)
# ==== LOAD
write_to_csv(stats, stats_out_filepath)
s = "Wrote statistics for {f_in} to {f_out} | {a} of {b}.".format(
f_in=json_file_path,
f_out=stats_out_filepath,
a=idx + 1,
b=len(example_json_results_file_paths))
log(s, wrap=False, timings=True)
def main(indir, tokens=" abcdefghijklmnopqrstuvwxyz'-"):
# TODO: Load individual examples from JSON.
# Create the factory we'll use to iterate over N examples at a time.
settings = {
"audio_indir": indir,
"max_examples": None,
"max_audio_file_bytes": None,
"targets_path": "./samples/cv-valid-test.csv",
"max_targets": 2000,
"batch_size": 10,
"gpu_device": 0,
}
batch_gen = get_validation_batch_generator(settings)
for b_id, batch in batch_gen:
tf_runtime = TFRuntime(settings["gpu_device"])
with tf_runtime.session as sess, tf_runtime.device as tf_device:
feeds = Feeds.Validation(batch)
ph_examples = tf.placeholder(
tf.float32, shape=[batch.size, batch.audios["max_samples"]])
ph_lens = tf.placeholder(tf.float32, shape=[batch.size])
model = DeepSpeech.Model(sess,
ph_examples,
batch,
tokens=tokens,
beam_width=500)
feeds.create_feeds(ph_examples, ph_lens)
decodings, probs = model.inference(batch,
feed=feeds.examples,
decoder="batch",
top_five=False)
raw, smax = sess.run(
[tf.transpose(model.raw_logits, [1, 0, 2]), model.logits],
feed_dict=feeds.examples)
outdir = "original-logits/"
if not os.path.exists(outdir):
os.mkdir(outdir)
for idx, basename in enumerate(batch.audios["basenames"]):
#TODO
stats = {
"basename": basename,
"decoding": decodings[idx],
"score": probs[idx],
"raw_logits": raw[idx],
"softmax": smax[idx],
"size": batch.audios["n_samples"][idx],
"n_feats": batch.audios["real_feats"][idx],
}
example_db = SingleJsonDB(outdir)
example_db.open(basename.rstrip(".wav")).put(stats)
log("Processed file: {b}".format(b=basename), wrap=False)
log("Decoding: {}".format(decodings[idx]), wrap=False)
def branched_grid_search(indir, sess, model, batch, original_probs, real_logits, reference=np.max):
for idx in range(batch.size):
basename = batch.audios["basenames"][idx]
target_phrase = batch.targets["phrases"][idx]
indices = batch.targets["indices"][idx]
n_padded_feats = batch.audios["ds_feats"][idx]
n_feats = batch.audios["real_feats"][idx]
targs_id = batch.targets["row_ids"][idx]
original_audio = batch.audios["audio"][idx]
absolute_file_path = os.path.abspath(
os.path.join(indir, basename)
)
new_target_phrases = np.array(
insert_target_blanks(indices),
dtype=np.int32
)
max_repeats = n_feats // len(new_target_phrases)
for current_repeat in range(0, max_repeats):
# s = "Processing: {b} for {r} repeats and target {t}".format(
# b=basename,
# r=current_repeat,
# t=targs_id
# )
# log(s, wrap=False)
kappa = MIN_KAPPA
current_depth = 1
max_depth = 10 ** MAX_DEPTH
result = {
"kappa": kappa,
"decoding": None,
"score": float('inf'),
"spc": float('inf'),
"new_logits": None,
"new_softmax": None,
"argmax": None,
"audio_filepath": absolute_file_path,
"audio_basename": basename,
"audio_data": original_audio,
"repeats": current_repeat,
"original_score": original_probs[idx],
"original_spc": original_probs[idx] / len(target_phrase),
"n_feats": n_feats,
"targ_id": targs_id,
"target_phrase": target_phrase,
"new_target": new_target_phrases,
"original_logits": real_logits[idx][:n_feats]
}
while current_depth <= max_depth:
if len(new_target_phrases) * current_repeat > n_feats:
# repeats won't fit logits space so completely
# skip any further processing.
break
else:
# otherwise, make some logits!
initial_logits = real_logits[idx].copy()
new_logits = add_kappa_to_all_logits(
n_feats,
n_padded_feats,
initial_logits,
new_target_phrases,
kappa,
current_repeat
)
new_logits = np.asarray([new_logits])
# TODO: Test that only one character per frame has changed
# and the remainder are the same as before (i.e. diff = 0)
# TODO: how can we escape the need to run as
new_softmaxes = sess.run(
tf.nn.softmax(new_logits)
)
decodings, probs = model.inference(
batch,
logits=np.asarray(new_softmaxes),
decoder="ds",
top_five=False
)
score_per_char = probs / len(target_phrase)
# scores increase as the token probabilities get
# closer. this seems counter intuitive, but it works
current_decoding_correct = decodings == target_phrase
current_score_better = result["spc"] > score_per_char
best_score_non_zero = result["score"] != float('inf')
if current_decoding_correct and current_score_better:
# great success!
best_kappa = kappa
kappa = update_kappa(kappa, current_depth, max_depth)
argmax = "".join(
l_map(
lambda x: TOKENS[x],
np.argmax(
new_logits[0][:n_feats],
axis=1
)
)
)
result["kappa"] = best_kappa
result["decoding"] = decodings
result["score"] = probs
result["spc"] = score_per_char
result["new_softmax"] = new_softmaxes[:n_feats]
result["new_logits"] = new_logits[0][:n_feats]
result["argmax"] = argmax [:n_feats]
elif best_score_non_zero:
# we have been successful at some point, so
# reduce the search depth
d = current_depth * 10
# there's a weird bug where search depths
# become 0, and then kappa start to tend
# toward negative infinity, e.g.
# kappa: -0.11 -> -0.111 -> -0.11 -> -inf
if d == 0:
# something went wrong...
current_depth = max_depth
elif d > max_depth:
# we've hit the maximum, update depths,
# but don't update kappa
break
else:
# we're not at maximum search depth, so we
# must have just seen something good so
# change the depth and update kappa
current_depth = d
kappa = result["kappa"]
kappa = update_kappa(kappa, current_depth, max_depth)
# elif kappa >= MIN_KAPPA:
# # we've hit a boundary condition
# break
else:
# we haven't found anything yet
kappa = update_kappa(kappa, current_depth, max_depth)
best_decoding_check = result["decoding"] != target_phrase
best_spc_check = result["spc"] >= result["original_spc"]
if best_decoding_check:
# we've not been successful, increase the number of repeats
# and try again
s = "Failure: {b} for {r} repeats and target {t}".format(
b=result["audio_basename"],
r=result["repeats"],
t=result["targ_id"],
)
s += " (decoding does not match target phrase)."
log(s, wrap=False)
elif best_spc_check:
# we've not been successful, increase the number of repeats
# and try again
s = "Failure: {b} for {r} repeats and target {t}".format(
b=result["audio_basename"],
r=result["repeats"],
t=result["targ_id"],
)
s += " adversarial score per char. <= original score per char.:"
s += " {a:.1f} vs. {b:.1f}".format(
a=result["spc"],
b=result["original_spc"]
)
log(s, wrap=False)
else:
yield idx, result
outdir = master_settings["outdir"]
attack_type = os.path.basename(__file__).replace(".py", "")
outdir = os.path.join(outdir, attack_type)
outdir = os.path.join(outdir, "baselines/ctc/")
outdir = os.path.join(outdir, "{}/".format(loss))
outdir = os.path.join(outdir, "{}/".format(decoder))
master_settings["outdir"] = outdir
batch_gen = data.ingress.etl.batch_generators.standard(master_settings)
default_manager(
master_settings,
create_attack_graph,
batch_gen,
)
log("Finished run.")
if __name__ == '__main__':
log("", wrap=True)
extra_args = {
"loss": [str, "ctc", False, LOSS_CHOICES.keys()],
}
args(attack_run, additional_args=extra_args)