def quantile_threshold(self, features, savepath=''):
qtpath = os.path.join(settings.OUTPUT_FOLDER, savepath)
if savepath and os.path.exists(qtpath):
return np.load(qtpath)
print("calculating quantile threshold")
quant = vecquantile.QuantileVector(depth=features.shape[1], seed=1)
start_time = time.time()
last_batch_time = start_time
batch_size = 64
for i in range(0, features.shape[0], batch_size):
batch_time = time.time()
rate = i / (batch_time - start_time + 1e-15)
batch_rate = batch_size / (batch_time - last_batch_time + 1e-15)
last_batch_time = batch_time
print('Processing quantile index %d: %f %f' %
(i, rate, batch_rate))
batch = features[i:i + batch_size]
batch = np.transpose(batch,
axes=(0, 2, 3,
1)).reshape(-1, features.shape[1])
quant.add(batch)
ret = quant.readout(1000)[:, int(1000 * (1 - settings.QUANTILE) - 1)]
if savepath:
np.save(qtpath, ret)
return ret
def quantile_threshold(self, features, savepath=""):
"""
Determine thresholds for neuron activations for each neuron.
"""
qtpath = os.path.join(settings.OUTPUT_FOLDER, savepath)
if savepath and os.path.exists(qtpath):
print(f"Loading cached quantiles {qtpath}")
return np.load(qtpath)
print("calculating quantile threshold")
quant = vecquantile.QuantileVector(depth=features.shape[1], seed=1)
batch_size = 64
for i in trange(0, features.shape[0], batch_size, desc="Processing quantiles"):
batch = features[i : i + batch_size]
batch = np.transpose(batch, axes=(0, 2, 3, 1)).reshape(
-1, features.shape[1]
)
quant.add(batch)
ret = quant.readout(1000)[:, int(1000 * (1 - settings.QUANTILE) - 1)]
if savepath:
np.save(qtpath, ret)
return ret