def load_model(args):
'''Loads one of the benchmark classifiers or generators.'''
if args.model in ['alexnet', 'vgg16', 'resnet152', 'resnet18']:
model = setting.load_classifier(args.model)
elif args.model == 'progan':
model = setting.load_proggan(args.dataset)
model = nethook.InstrumentedModel(model).cuda().eval()
return model
def load_model(args):
'''Loads one of the benchmark classifiers or generators.'''
if args.model in ['alexnet', 'vgg16', 'resnet152']:
if (args.dataset == 'ucf101'):
model = setting.load_ucf101_classifier(args.model)
else:
model = setting.load_classifier(args.model)
elif args.model == 'progan':
model = setting.load_proggan(args.dataset)
#Original model, use the if-else block below
if (use_cuda):
model = nethook.InstrumentedModel(model).cuda().eval()
else:
model = nethook.InstrumentedModel(model).eval()
return model
def get_moco_model(dataset, epoch=240):
folder_path = "CMC/CMC_data/{}_models".format(dataset)
model_name = "/{}_MoCo0.999_softmax_16384_resnet50".format(dataset) + \
"_lr_0.03_decay_0.0001_bsz_128_crop_0.2_aug_CJ"
epoch_name = "/ckpt_epoch_{}.pth".format(epoch)
my_path = folder_path + model_name + epoch_name
checkpoint = torch.load(my_path)
model_checkpoint = {
key.replace(".module", ""): val
for key, val in checkpoint['model'].items()
}
model = InsResNet50(parallel=False)
model.load_state_dict(model_checkpoint)
model = nethook.InstrumentedModel(model)
return model
def __init__(self,
model,
dataset,
dataset_path,
model_layer,
seglabels=None,
segcatlabels=None,
model_nm=None):
model = nethook.InstrumentedModel(model)
model.cuda()
model.eval()
self.model = model
self.layername = model_layer
self.model.retain_layer(self.layername)
self.model_name = model_nm
self.topk = None
self.unit_images = None
self.iou99 = None
self.upfn = upsample.upsampler(
target_shape=(56, 56),
data_shape=(7, 7),
)
if dataset == 'nih_seg':
if seglabels is not None:
self.seglabels = seglabels
else:
self.seglabels = [
'No Class', 'Atelectasis', 'Cardiomegaly', 'Effusion',
'Infiltrate', 'Mass', 'Nodule', 'Pneumonia',
'Pneumothorax', 'Consolidation', 'Edema', 'Emphysema',
'Fibrosis', 'Pleural_Thickening', 'Hernia'
]
if segcatlabels is not None:
self.segcatlabels = segcatlabels
else:
self.segcatlabels = [('No Class', 'No Class'),
('Atelectasis', 'Atelectasis'),
('Cardiomegaly', 'Cardiomegaly'),
('Effusion', 'Effusion'),
('Infiltrate', 'Infiltrate'),
('Mass', 'Mass'), ('Nodule', 'Nodule'),
('Pneumonia', 'Pneumonia'),
('Pneumothorax', 'Pneumothorax'),
('Consolidation', 'Consolidation'),
('Edema', 'Edema'),
('Emphysema', 'Emphysema'),
('Fibrosis', 'Fibrosis'),
('Pleural_Thickening',
'Pleural_Thickening'),
('Hernia', 'Hernia')]
if model_nm == 'chexpert_noweights':
batch_sz = 10
else:
batch_sz = 20
config = {'batch_size': batch_sz, 'input_size': (224, 224)}
# Creating the dataloaders
_, _, self.ds_loader = get_nih_segmented_dataloaders(
dataset_path, **config)
self.ds = self.ds_loader.dataset
# Setting sample size
self.sample_size = 100
self.rq = self._get_rq_vals()
self.iv = imgviz.ImageVisualizer(224,
source=self.ds,
percent_level=0.99,
quantiles=self.rq)
def __init__(self,
model,
dataset,
dataset_path,
model_layer,
seglabels=None,
segcatlabels=None):
model = nethook.InstrumentedModel(model)
model.cuda()
model.eval()
self.model = model
self.layername = model_layer
self.model.retain_layer(self.layername)
self.topk = None
self.unit_images = None
self.iou99 = None
self.upfn = upsample.upsampler(
target_shape=(56, 56),
data_shape=(7, 7),
)
if dataset == 'covid_seg':
self.seglabels = [
'No class', 'Left Lung', 'Right Lung', 'Cardiomediastinum',
'Airways', 'Ground Glass Opacities', 'Consolidation',
'Pleural Effusion', 'Pneumothorax', 'Endotracheal Tube',
'Central Venous Line', 'Monitoring Probes', 'Nosogastric Tube',
'Chest tube', 'Tubings'
]
self.segcatlabels = [
('No class', 'No class'), ('Left Lung', 'Left Lung'),
('Right Lung', 'Right Lung'),
('Cardiomediastinum', 'Cardiomediastinum'),
('Airways', 'Airways'),
('Ground Glass Opacities', 'Ground Glass Opacities'),
('Consolidation', 'Consolidation'),
('Pleural Effusion', 'Pleural Effusion'),
('Pneumothorax', 'Pneumothorax'),
('Endotracheal Tube', 'Endotracheal Tube'),
('Central Venous Line', 'Central Venous Line'),
('Monitoring Probes', 'Monitoring Probes'),
('Nosogastric Tube', 'Nosogastric Tube'),
('Chest tube', 'Chest tube'), ('Tubings', 'Tubings')
]
config = {
'batch_size': 1,
'input_size': (224, 224),
}
# Creating the dataloaders
self.ds_loader = get_segmentation_dataloader(
dataset_path, **config)
self.ds = self.ds_loader.dataset
# Specify the sample size in case of bigger dataset. Default is 100 for covid seg
self.sample_size = 100
self.rq = self._get_rq_vals()
self.iv = imgviz.ImageVisualizer(224,
source=self.ds,
percent_level=0.99,
quantiles=self.rq)
# result = PIL.Image.open(os.path.join(qd.dir(layername), 's_imgs/unit_%d.png' % unit))
# result.load()
# return result
for layername in layers:
#if os.path.isfile(os.path.join(qd.dir(layername), 'intersect_99.npz')):
# continue
busy_fn = os.path.join(qd.dir(layername), 'busy.txt')
if os.path.isfile(busy_fn):
print(busy_fn)
continue
with open(busy_fn, 'w') as f:
f.write('busy')
print('working on', layername)
inst_net = nethook.InstrumentedModel(copy.deepcopy(net)).cuda()
inst_net.retain_layer('features.' + layername)
inst_net(ds[0][0][None].cuda())
sample_act = inst_net.retained_layer('features.' + layername).cpu()
upfn = upsample.upsampler((64, 64), sample_act.shape[2:])
def flat_acts(batch):
inst_net(batch.cuda())
acts = upfn(inst_net.retained_layer('features.' + layername))
return acts.permute(0, 2, 3, 1).contiguous().view(-1, acts.shape[1])
s_rq = tally.tally_quantile(flat_acts,
sds,
cachefile=os.path.join(qd.dir(layername),
's_rq.npz'))
u_rq = qd.rq(layername)
return os.path.join(resdir, filename)
# Download and instantiate the model.
model = oldresnet152.OldResNet152()
url = ('http://gandissect.csail.mit.edu/' +
'models/resnet152_places365-f928166e5c.pth')
try:
sd = torch.hub.load_state_dict_from_url(url) # pytorch 1.1
except:
sd = torch.hub.model_zoo.load_url(url) # pytorch 1.0
model.load_state_dict(sd)
layername = '7'
sample_size = 36500
model = nethook.InstrumentedModel(model)
model = model.cuda()
model.retain_layer(layername)
# Load labels
from urllib.request import urlopen
synset_url = 'http://gandissect.csail.mit.edu/models/categories_places365.txt'
classlabels = [r.split(' ')[0][3:]
for r in urlopen(synset_url).read().decode('utf-8').split('\n')]
# Load segmenter
from netdissect import segmenter
segmodel = segmenter.UnifiedParsingSegmenter(segsizes=[256])
seglabels = [l for l, c in segmodel.get_label_and_category_names()[0]]
def main():
args = parseargs()
os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu)
# layername = "encoder.layer4.0.conv2"
layername = args.layername
# dataset_split = "v32.1"
resdir = '/scratch/users/abhishekm/vwm-dissect/results/%s-%s-%s-%s-%s-finetuned' % (
args.model, args.dataset, args.seg, layername, args.data_split)
if args.layer is not None:
resdir += '-' + args.layer
if args.quantile != 0.005:
resdir += ('-%g' % (args.quantile * 1000))
if args.thumbsize != 100:
resdir += ('-t%d' % (args.thumbsize))
resfile = pidfile.exclusive_dirfn(resdir)
# model = load_model(args)
cfg = cfg_parser(
"/home/users/abhishekm/vwm/cfg/experiments/abm_dissect.json")
_trainer = trainer.factory.create(cfg["model_cfg"].trainer_key, **cfg)
model = _trainer.model
model = nethook.InstrumentedModel(model).cuda().eval()
# dataloader = _trainer.data_loaders["val"]["real_baseline"]
# layername = "encoder.layer4.1.conv2" # instrumented_layername(args)
model.retain_layer(layername)
dataset = _trainer.datasets['val'][0][
0] # load_dataset(args, model=model.model)
upfn = make_upfn(args, dataset, model, layername)
sample_size = None # len(dataset)
is_generator = (args.model == 'progan')
percent_level = 1.0 - args.quantile
iou_threshold = args.miniou
image_row_width = 5
torch.set_grad_enabled(False)
# Tally rq.np (representation quantile, unconditional).
pbar.descnext('rq')
def compute_samples(batch, *args):
data_batch = batch.cuda()
_ = model(data_batch)
acts = model.retained_layer(layername)
hacts = upfn(acts)
return hacts.permute(0, 2, 3, 1).contiguous().view(-1, acts.shape[1])
rq = tally.tally_quantile(compute_samples,
dataset,
sample_size=sample_size,
r=8192,
num_workers=10,
pin_memory=True,
cachefile=resfile('rq.npz'))
# Create visualizations - first we need to know the topk
pbar.descnext('topk')
def compute_image_max(batch, *args):
data_batch = batch.cuda()
_ = model(data_batch)
acts = model.retained_layer(layername)
acts = acts.view(acts.shape[0], acts.shape[1], -1)
acts = acts.max(2)[0]
return acts
topk = tally.tally_topk(compute_image_max,
dataset,
sample_size=sample_size,
batch_size=50,
num_workers=30,
pin_memory=True,
cachefile=resfile('topk.npz'))
# Visualize top-activating patches of top-activatin images.
pbar.descnext('unit_images')
image_size, image_source = (224, 224), None
if is_generator:
image_size = model(dataset[0][0].cuda()[None, ...]).shape[2:]
else:
image_source = dataset
iv = imgviz.ImageVisualizer((args.thumbsize, args.thumbsize),
image_size=image_size,
source=dataset,
quantiles=rq,
level=rq.quantiles(percent_level))
def compute_acts(data_batch, *ignored_class):
data_batch = data_batch.cuda()
out_batch = model(data_batch)
acts_batch = model.retained_layer(layername)
if is_generator:
return (acts_batch, out_batch)
else:
return (acts_batch, data_batch)
unit_images = iv.masked_images_for_topk(
compute_acts,
dataset,
topk,
k=image_row_width,
num_workers=30,
pin_memory=True,
cachefile=resfile('top%dimages.npz' % image_row_width))
pbar.descnext('saving images')
imgsave.save_image_set(unit_images,
resfile('image/unit%d.jpg'),
sourcefile=resfile('top%dimages.npz' %
image_row_width))
# Compute IoU agreement between segmentation labels and every unit
# Grab the 99th percentile, and tally conditional means at that level.
level_at_99 = rq.quantiles(percent_level).cuda()[None, :, None, None]
segmodel, seglabels, segcatlabels = setting.load_segmenter(args.seg)
renorm = renormalize.renormalizer(dataset, target='zc')
def compute_conditional_indicator(batch, *args):
data_batch = batch.cuda()
out_batch = model(data_batch)
image_batch = out_batch if is_generator else renorm(data_batch)
seg = segmodel.segment_batch(image_batch, downsample=4)
acts = model.retained_layer(layername)
hacts = upfn(acts)
iacts = (hacts > level_at_99).float() # indicator
return tally.conditional_samples(iacts, seg)
pbar.descnext('condi99')
condi99 = tally.tally_conditional_mean(compute_conditional_indicator,
dataset,
sample_size=sample_size,
num_workers=3,
pin_memory=True,
cachefile=resfile('condi99.npz'))
# Now summarize the iou stats and graph the units
iou_99 = tally.iou_from_conditional_indicator_mean(condi99)
unit_label_99 = [(concept.item(), seglabels[concept],
segcatlabels[concept], bestiou.item())
for (bestiou, concept) in zip(*iou_99.max(0))]
labelcat_list = [
labelcat for concept, label, labelcat, iou in unit_label_99
if iou > iou_threshold
]
save_conceptcat_graph(resfile('concepts_99.svg'), labelcat_list)
dump_json_file(
resfile('report.json'),
dict(header=dict(name='%s %s %s' %
(args.model, args.dataset, args.seg),
image='concepts_99.svg'),
units=[
dict(image='image/unit%d.jpg' % u,
unit=u,
iou=iou,
label=label,
cat=labelcat[1]) for u, (concept, label, labelcat,
iou) in enumerate(unit_label_99)
]))
copy_static_file('report.html', resfile('+report.html'))
resfile.done()
def main():
# Load the arguments
args = parse_option()
dataset = args.dataset
sample_size = args.sample_size
layername = args.layer
# Other values for places and imagenet MoCo model
epoch = 240
image_size = 224
crop = 0.2
crop_padding = 32
batch_size = 1
num_workers = 24
train_sampler = None
moco = True
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
normalize = transforms.Normalize(mean=mean, std=std)
# Set appropriate paths
folder_path = "/data/vision/torralba/ganprojects/yyou/CMC_data/{}_models".format(
dataset)
model_name = "/{}_MoCo0.999_softmax_16384_resnet50_lr_0.03".format(dataset) \
+ "_decay_0.0001_bsz_128_crop_0.2_aug_CJ"
epoch_name = "/ckpt_epoch_{}.pth".format(epoch)
my_path = folder_path + model_name + epoch_name
data_path = "/data/vision/torralba/datasets/"
web_path = "/data/vision/torralba/scratch/yyou/wednesday/dissection/"
if dataset == "imagenet":
data_path += "imagenet_pytorch"
web_path += dataset + "/" + layername
elif dataset == "places365":
data_path += "places/places365_standard/places365standard_easyformat"
web_path += dataset + "/" + layername
# Create web path folder directory for this layer
if not os.path.exists(web_path):
os.makedirs(web_path)
# Load validation data loader
val_folder = data_path + "/val"
val_transform = transforms.Compose([
transforms.Resize(image_size + crop_padding),
transforms.CenterCrop(image_size),
transforms.ToTensor(),
normalize,
])
ds = QuickImageFolder(val_folder,
transform=val_transform,
shuffle=True,
two_crop=False)
ds_loader = torch.utils.data.DataLoader(ds,
batch_size=batch_size,
shuffle=(train_sampler is None),
num_workers=num_workers,
pin_memory=True,
sampler=train_sampler)
# Load model from checkpoint
checkpoint = torch.load(my_path)
model_checkpoint = {
key.replace(".module", ""): val
for key, val in checkpoint['model'].items()
}
model = InsResNet50(parallel=False)
model.load_state_dict(model_checkpoint)
model = nethook.InstrumentedModel(model)
model.cuda()
# Renormalize RGB data from the staistical scaling in ds to [-1...1] range
renorm = renormalize.renormalizer(source=ds, target='zc')
# Retain desired layer with nethook
batch = next(iter(ds_loader))[0]
model.retain_layer(layername)
model(batch.cuda())
acts = model.retained_layer(layername).cpu()
upfn = upsample.upsampler(
target_shape=(56, 56),
data_shape=(7, 7),
)
def flatten_activations(batch, *args):
image_batch = batch
_ = model(image_batch.cuda())
acts = model.retained_layer(layername).cpu()
hacts = upfn(acts)
return hacts.permute(0, 2, 3, 1).contiguous().view(-1, acts.shape[1])
def tally_quantile_for_layer(layername):
rq = tally.tally_quantile(
flatten_activations,
dataset=ds,
sample_size=sample_size,
batch_size=100,
cachefile='results/{}/{}_rq_cache.npz'.format(dataset, layername))
return rq
rq = tally_quantile_for_layer(layername)
# Visualize range of activations (statistics of each filter over the sample images)
fig, axs = plt.subplots(2, 2, figsize=(10, 8))
axs = axs.flatten()
quantiles = [0.5, 0.8, 0.9, 0.99]
for i in range(4):
axs[i].plot(rq.quantiles(quantiles[i]))
axs[i].set_title("Rq quantiles ({})".format(quantiles[i]))
fig.suptitle("{} - sample size of {}".format(dataset, sample_size))
plt.savefig(web_path + "/rq_quantiles")
# Set the image visualizer with the rq and percent level
iv = imgviz.ImageVisualizer(224,
source=ds,
percent_level=0.95,
quantiles=rq)
# Tally top k images that maximize the mean activation of the filter
def max_activations(batch, *args):
image_batch = batch.cuda()
_ = model(image_batch)
acts = model.retained_layer(layername)
return acts.view(acts.shape[:2] + (-1, )).max(2)[0]
def mean_activations(batch, *args):
image_batch = batch.cuda()
_ = model(image_batch)
acts = model.retained_layer(layername)
return acts.view(acts.shape[:2] + (-1, )).mean(2)
topk = tally.tally_topk(
mean_activations,
dataset=ds,
sample_size=sample_size,
batch_size=100,
cachefile='results/{}/{}_cache_mean_topk.npz'.format(
dataset, layername))
top_indexes = topk.result()[1]
# Visualize top-activating images for a particular unit
if not os.path.exists(web_path + "/top_activating_imgs"):
os.makedirs(web_path + "/top_activating_imgs")
def top_activating_imgs(unit):
img_ids = [i for i in top_indexes[unit, :12]]
images = [iv.masked_image(ds[i][0], \
model.retained_layer(layername)[0], unit) \
for i in img_ids]
preds = [ds.classes[model(ds[i][0][None].cuda()).max(1)[1].item()]\
for i in img_ids]
fig, axs = plt.subplots(3, 4, figsize=(16, 12))
axs = axs.flatten()
for i in range(12):
axs[i].imshow(images[i])
axs[i].tick_params(axis='both', which='both', bottom=False, \
left=False, labelbottom=False, labelleft=False)
axs[i].set_title("img {} \n pred: {}".format(img_ids[i], preds[i]))
fig.suptitle("unit {}".format(unit))
plt.savefig(web_path + "/top_activating_imgs/unit_{}".format(unit))
for unit in np.random.randint(len(top_indexes), size=10):
top_activating_imgs(unit)
def compute_activations(image_batch):
image_batch = image_batch.cuda()
_ = model(image_batch)
acts_batch = model.retained_layer(layername)
return acts_batch
unit_images = iv.masked_images_for_topk(
compute_activations,
ds,
topk,
k=5,
num_workers=10,
pin_memory=True,
cachefile='results/{}/{}_cache_top10images.npz'.format(
dataset, layername))
file = open("results/{}/unit_images.pkl".format(dataset, layername), 'wb')
pickle.dump(unit_images, file)
# Load a segmentation model
segmodel, seglabels, segcatlabels = setting.load_segmenter('netpqc')
# Intersections between every unit's 99th activation
# and every segmentation class identified
level_at_99 = rq.quantiles(0.99).cuda()[None, :, None, None]
def compute_selected_segments(batch, *args):
image_batch = batch.cuda()
seg = segmodel.segment_batch(renorm(image_batch), downsample=4)
_ = model(image_batch)
acts = model.retained_layer(layername)
hacts = upfn(acts)
iacts = (hacts >
level_at_99).float() # indicator where > 0.99 percentile.
return tally.conditional_samples(iacts, seg)
condi99 = tally.tally_conditional_mean(
compute_selected_segments,
dataset=ds,
sample_size=sample_size,
cachefile='results/{}/{}_cache_condi99.npz'.format(dataset, layername))
iou99 = tally.iou_from_conditional_indicator_mean(condi99)
file = open("results/{}/{}_iou99.pkl".format(dataset, layername), 'wb')
pickle.dump(iou99, file)
# Show units with best match to a segmentation class
iou_unit_label_99 = sorted(
[(unit, concept.item(), seglabels[concept], bestiou.item())
for unit, (bestiou, concept) in enumerate(zip(*iou99.max(0)))],
key=lambda x: -x[-1])
fig, axs = plt.subplots(20, 1, figsize=(20, 80))
axs = axs.flatten()
for i, (unit, concept, label, score) in enumerate(iou_unit_label_99[:20]):
axs[i].imshow(unit_images[unit])
axs[i].set_title('unit %d; iou %g; label "%s"' % (unit, score, label))
axs[i].set_xticks([])
axs[i].set_yticks([])
plt.savefig(web_path + "/best_unit_segmentation")
def main():
CHOSEN_UNITS_DIR = os.path.join("/", "home", "dwijaya", "dissect",
"experiment", "ucf101", "datas",
"chosen_units.csv")
report_dir = os.path.join(
"/", "home", "dwijaya", "dissect", "experiment",
"results/vgg16-ucf101-netpqc-conv5_3-10/report.json")
result_test = load_json(report_dir)['units']
# getUnitLabel(result_test)
# groupUnitByLabel(result_test)
chosen_units_df = pd.read_csv(CHOSEN_UNITS_DIR)
args = parseargs()
model = setting.load_ucf101_classifier(args.model)
model = nethook.InstrumentedModel(model).cuda().eval()
layername = args.layer
model.retain_layer(layername)
dataset = setting.load_ucf101_dataset(crop_size=224,
in_dataloader=False,
is_all_frames=True)
train_dataset = setting.load_ucf101_dataset(crop_size=224,
in_dataloader=False,
is_all_frames=True,
is_train=True)
num_units = len(chosen_units_df)
classlabels = dataset.classes
def zeroingTopK(k=14):
directory = os.path.join(os.getcwd(), 'results/shared',
'pra-vgg16-ucf101/per_class')
save_dir = os.path.join(directory, 'topK_target.csv')
if (os.path.exists(save_dir)):
df = pd.read_csv(save_dir)
print(df)
else:
# topK_all_class = []
# for idx, cl in enumerate(classlabels):
# cachefile = sharedfile('pra-%s-%s/%s/%s.npz' % (args.model, args.dataset, args.experiments, cl))
# df = pd.read_csv(save_dir)
# df2 = pd.read_csv(os.path.join(directory, '%s.csv' % cl))
# to_save = []
# for idx, (unit, concept) in enumerate(zip(df2['Unit'].loc[14:], df2['Concept'].loc[14:])):
# to_save.append((unit, concept))
# topK_all_class.append(to_save)
# df = df.rename(columns={'Unnamed: 0': 'Class', '0': 'Acc_dropped'})
# df['Unit/Concept'] = topK_all_class
# df['Class'] = classlabels
# df.to_csv(save_dir)
topK_all_class = []
acc_per_class_list, target_acc_class = [], []
for idx, cl in enumerate(classlabels):
cachefile = sharedfile(
'pra-%s-%s/%s/%s.npz' %
(args.model, args.dataset, args.experiments, cl))
df = pd.read_csv(os.path.join(directory, '%s.csv' % cl))
units_to_remove = df['Unit'].loc[:k - 1].to_list()
accuracy, acc_per_class = my_test_perclass(
model,
dataset,
layername=layername,
ablated_units=units_to_remove,
cachefile=cachefile)
target_acc_class.append(acc_per_class[idx])
acc_per_class_list.append(acc_per_class)
to_save = []
for idx, (unit, concept) in enumerate(
zip(df['Unit'].loc[:k - 1],
df['Concept'].loc[:k - 1])):
to_save.append((unit, concept))
topK_all_class.append(to_save)
result_df = pd.DataFrame(target_acc_class, columns=['Acc_dropped'])
# result_df = result_df.rename(columns={'Unnamed: 0': 'Class', '0': 'Acc_dropped'})
result_df['Unit/Concept'] = topK_all_class
result_df['Class'] = classlabels
result_df.to_csv(os.path.join(directory, "topK_target.csv"))
pd.DataFrame(acc_per_class_list).to_csv(
os.path.join('topK_per_class.csv'))
def zeroingBottomK(k=498): #previously is 498, so it is wrong.
directory = os.path.join(os.getcwd(), 'results/shared',
'pra-vgg16-ucf101/per_class')
topK_all_class = []
acc_per_class_list, target_acc_class = [], []
for idx, cl in enumerate(classlabels):
cachefile = sharedfile(
'pra-%s-%s/%s/%s.npz' %
(args.model, args.dataset, args.experiments, cl))
df = pd.read_csv(os.path.join(directory, '%s.csv' % cl))
units_to_remove = df['Unit'].loc[k:].to_list()
accuracy, acc_per_class = my_test_perclass(
model,
dataset,
layername=layername,
ablated_units=units_to_remove,
cachefile=cachefile)
target_acc_class.append(acc_per_class[idx])
acc_per_class_list.append(acc_per_class)
to_save = []
for idx, (unit, concept) in enumerate(
zip(df['Unit'].loc[k:], df['Concept'].loc[k:])):
to_save.append((unit, concept))
topK_all_class.append(to_save)
result_df = pd.DataFrame(target_acc_class, columns=['Acc_dropped'])
result_df['Unit/Concept'] = topK_all_class
result_df['Class'] = classlabels
result_df.to_csv(os.path.join(directory, "bottomK_target_new.csv"))
# pd.DataFrame(target_acc_class).to_csv(os.path.join(directory, "bottomK_target_new.csv"))
pd.DataFrame(acc_per_class_list).to_csv(
os.path.join(directory, 'bottomK_per_class_new.csv'))
def zeroKWithConcepts():
directory = os.path.join(os.getcwd(), 'results/shared',
'pra-vgg16-ucf101/per_class')
save_dir = os.path.join(directory, 'bottomK_target.csv')
topK_all_class = []
for idx, cl in enumerate(classlabels):
cachefile = sharedfile(
'pra-%s-%s/%s/%s.npz' %
(args.model, args.dataset, args.experiments, cl))
df = pd.read_csv(save_dir)
df2 = pd.read_csv(os.path.join(directory, '%s.csv' % cl))
to_save = []
for idx, (unit, concept) in enumerate(
zip(df2['Unit'].loc[14:], df2['Concept'].loc[14:])):
to_save.append((unit, concept))
topK_all_class.append(to_save)
df = df.rename(columns={'Unnamed: 0': 'Class', '0': 'Acc_dropped'})
df['Unit/Concept'] = topK_all_class
df['Class'] = classlabels
df.to_csv(save_dir)
print("HELLO")
# df = df.rename(columns={"Unnamed: 0": "Concept"})
# df.to_csv(os.path.join(directory, '%s.csv' % cl))
# coba()
# sortAcc()
#Getting the baseline accuracy.
baseline_acc_dir = os.path.join(
os.getcwd(), 'results/shared',
'pra-%s-%s/baseline_acc.npz' % (args.model, args.dataset))
if (os.path.exists(baseline_acc_dir)):
baseline_ = np.load(baseline_acc_dir)
baseline_acc, baseline_acc_per_class = baseline_['acc'], baseline_[
'acc_per_class']
else:
pbar.descnext('baseline_pra')
baseline_acc, baseline_acc_per_class = my_test_perclass(
model,
dataset,
ablated_units=None,
cachefile=sharedfile('pra-%s-%s/%s_acc.npz' %
(args.model, args.dataset, args.experiments)))
cachefile = sharedfile('pra-%s-%s/%s_acc.npz' %
(args.model, args.dataset, args.experiments))
np.savez(cachefile,
acc=baseline_acc,
acc_per_class=baseline_acc_per_class)
baseline_acc_per_class = np.expand_dims(baseline_acc_per_class, axis=0)
pd.DataFrame(baseline_acc_per_class,
index=['Baseline'],
columns=classlabels).to_csv("base_line.csv")
#Now erase each unit, one at a time, and retest accuracy.
cached_results_dir = os.path.join(
os.getcwd(), 'results/shared',
'pra-%s-%s/%s_acc.npz' % (args.model, args.dataset, args.experiments))
cachefile = sharedfile('pra-%s-%s/%s_acc.npz' %
(args.model, args.dataset, args.experiments))
all_units = []
if (args.experiments == "topK"):
zeroingTopK()
elif (args.experiments == "bottomK"):
zeroingBottomK()
if (args.extract_data):
baseline_ = {
'acc': baseline_acc,
'acc_per_class': baseline_acc_per_class
}
# npzToCSV(args.experiments, columns=classlabels, baseline_=baseline_, export_csv=False)
# sortUnitByClass(baseline_, args.experiments, classlabels)
else:
if (args.experiments == 'exp1'):
df = pd.read_csv(os.path.join(datas_dir, 'Sensible units.csv'))
if (os.path.exists(cached_results_dir)):
# IF THE RESULT ALREADY EXISTS
acc_per_class_list = np.load(
cached_results_dir)['acc_per_class']
acc_list = np.load(cached_results_dir)['acc']
else:
#Remove unit one at a time.
units_to_remove, concepts = df['Unit'], df['Concepts']
for idx, (units,
concept) in enumerate(zip(units_to_remove,
concepts)):
units = units.split(',')
units = [(int(u), concept) for u in units]
all_units.extend(units)
acc_per_class_list = np.zeros(
[len(all_units), len(classlabels)])
acc_list = np.zeros(len(classlabels))
for idx, (unit, c) in enumerate(all_units):
accuracy, acc_per_class = my_test_perclass(
model,
dataset,
layername=layername,
ablated_units=[unit],
cachefile=cachefile)
acc_list[idx] = accuracy
acc_per_class_list[idx] = acc_per_class
np.savez(cachefile,
acc=acc_list,
acc_per_class=acc_per_class_list)
elif (args.experiments == 'exp2'):
df = pd.read_csv(os.path.join(datas_dir, 'Sensible units.csv'))
if (os.path.exists(cached_results_dir)):
#IF THE RESULT ALREADY EXISTS
acc_per_class_list = np.load(
cached_results_dir)['acc_per_class']
acc_list = np.load(cached_results_dir)['acc']
else:
#Remove multiple units at a time
units_to_remove, concepts = df['Unit'], df['Concepts']
acc_per_class_list = np.zeros([num_units, len(classlabels)])
acc_list = np.zeros(len(classlabels))
for idx, (units,
concept) in enumerate(zip(units_to_remove,
concepts)):
units = units.split(',')
units = [int(u) for u in units]
accuracy, acc_per_class = my_test_perclass(
model,
dataset,
layername=layername,
ablated_units=units,
cachefile=cachefile)
acc_list[idx] = accuracy
acc_per_class_list[idx] = acc_per_class # in a list
np.savez(cachefile,
acc=acc_list,
acc_per_class=acc_per_class_list)
elif (args.experiments == 'exp3'):
df = pd.read_csv(os.path.join(datas_dir, 'units_label.csv'))
if (os.path.exists(cached_results_dir)):
acc_per_class_list = np.load(
cached_results_dir)['acc_per_class']
acc_list = np.load(cached_results_dir)['acc']
else:
# Remove multiple units at a time
concepts = df['Concepts']
# acc_per_class_list = np.zeros([len(concepts), len(classlabels)])
# acc_list = np.zeros(len(concepts))
# acc_per_class = np.zeros(len(classlabels))
# acc_per_class = np.zeros(len(classlabels))
process_complete = tqdm.tqdm(total=len(concepts),
desc='Units Complete',
position=0)
for idx, (concept) in enumerate(concepts):
cachefile = sharedfile(
'pra-%s-%s/%s/%s.npz' %
(args.model, args.dataset, args.experiments,
"unit" + str(idx)))
if (not os.path.exists(cachefile)):
unit = idx
# units = [int(u) for u in units]
accuracy, acc_per_class = my_test_perclass(
model,
dataset,
layername=layername,
ablated_units=[unit],
cachefile=cachefile)
# acc_list[idx] = accuracy
# acc_per_class_list[idx] = acc_per_class # in a list
np.savez(cachefile,
acc=accuracy,
acc_per_class=acc_per_class,
concept=concept)
else:
print("Unit %s is done" % (str(idx)))
process_complete.update(1)
elif (args.experiments == 'exp4'):
with open(os.path.join(datas_dir, 'units_by_labels.json'),
'r') as file:
df = json.load(file)
acc_per_class_list = np.zeros([len(df), len(classlabels)])
acc_list = np.zeros(len(df))
process_complete = tqdm.tqdm(total=len(df),
desc='Concepts Complete',
position=0)
for idx, (concept, units) in enumerate(df.items()):
cachefile = sharedfile(
'pra-%s-%s/%s/%s.npz' %
(args.model, args.dataset, args.experiments, concept))
if (not os.path.exists(cachefile)):
#i.e (concept, units) = ('arm', [42,260,462,464])
accuracy, acc_per_class = my_test_perclass(
model,
dataset,
layername=layername,
ablated_units=units,
cachefile=cachefile)
acc_list[idx] = accuracy
acc_per_class_list[idx] = acc_per_class
np.savez(cachefile,
acc=acc_list,
acc_per_class=acc_per_class_list)
else:
print("Concept : %s is done" % (concept))
process_complete.update(1)
def main():
args = parseargs()
model = setting.load_classifier(args.model)
model = nethook.InstrumentedModel(model).cuda().eval()
layername = args.layer
model.retain_layer(layername)
dataset = setting.load_dataset(args.dataset, crop_size=224)
train_dataset = setting.load_dataset(args.dataset,
crop_size=224,
split='train')
sample_size = len(dataset)
# Probe layer to get sizes
model(dataset[0][0][None].cuda())
num_units = model.retained_layer(layername).shape[1]
classlabels = dataset.classes
# Measure baseline classification accuracy on val set, and cache.
pbar.descnext('baseline_pra')
baseline_precision, baseline_recall, baseline_accuracy, baseline_ba = (
test_perclass_pra(model,
dataset,
cachefile=sharedfile('pra-%s-%s/pra_baseline.npz' %
(args.model, args.dataset))))
pbar.print('baseline acc', baseline_ba.mean().item())
# Now erase each unit, one at a time, and retest accuracy.
unit_list = random.sample(list(range(num_units)), num_units)
val_single_unit_ablation_ba = torch.zeros(num_units, len(classlabels))
for unit in pbar(unit_list):
pbar.descnext('test unit %d' % unit)
# Get binary accuracy if the model after ablating the unit.
_, _, _, ablation_ba = test_perclass_pra(
model,
dataset,
layername=layername,
ablated_units=[unit],
cachefile=sharedfile('pra-%s-%s/pra_ablate_unit_%d.npz' %
(args.model, args.dataset, unit)))
val_single_unit_ablation_ba[unit] = ablation_ba
# For the purpose of ranking units by importance to a class, we
# measure using the training set (to avoid training unit ordering
# on the test set).
sample_size = None
# Measure baseline classification accuracy, and cache.
pbar.descnext('train_baseline_pra')
baseline_precision, baseline_recall, baseline_accuracy, baseline_ba = (
test_perclass_pra(
model,
train_dataset,
sample_size=sample_size,
cachefile=sharedfile('ttv-pra-%s-%s/pra_train_baseline.npz' %
(args.model, args.dataset))))
pbar.print('baseline acc', baseline_ba.mean().item())
# Measure accuracy on the val set.
pbar.descnext('val_baseline_pra')
_, _, _, val_baseline_ba = (test_perclass_pra(
model,
dataset,
cachefile=sharedfile('ttv-pra-%s-%s/pra_val_baseline.npz' %
(args.model, args.dataset))))
pbar.print('val baseline acc', val_baseline_ba.mean().item())
# Do in shuffled order to allow multiprocessing.
single_unit_ablation_ba = torch.zeros(num_units, len(classlabels))
for unit in pbar(unit_list):
pbar.descnext('test unit %d' % unit)
_, _, _, ablation_ba = test_perclass_pra(
model,
train_dataset,
layername=layername,
ablated_units=[unit],
sample_size=sample_size,
cachefile=sharedfile('ttv-pra-%s-%s/pra_train_ablate_unit_%d.npz' %
(args.model, args.dataset, unit)))
single_unit_ablation_ba[unit] = ablation_ba
# Now for every class, remove a set of the N most-important
# and N least-important units for that class, and measure accuracy.
for classnum in pbar(
random.sample(range(len(classlabels)), len(classlabels))):
# For a few classes, let's chart the whole range of ablations.
if classnum in [100, 169, 351, 304]:
num_best_list = range(1, num_units)
else:
num_best_list = [1, 2, 3, 4, 5, 20, 64, 128, 256]
pbar.descnext('numbest')
for num_best in pbar(random.sample(num_best_list, len(num_best_list))):
num_worst = num_units - num_best
unitlist = single_unit_ablation_ba[:,
classnum].sort(0)[1][:num_best]
_, _, _, testba = test_perclass_pra(
model,
dataset,
layername=layername,
ablated_units=unitlist,
cachefile=sharedfile(
'ttv-pra-%s-%s/pra_val_ablate_classunits_%s_ba_%d.npz' %
(args.model, args.dataset, classlabels[classnum],
len(unitlist))))
unitlist = (
single_unit_ablation_ba[:, classnum].sort(0)[1][-num_worst:])
_, _, _, testba2 = test_perclass_pra(
model,
dataset,
layername=layername,
ablated_units=unitlist,
cachefile=sharedfile(
'ttv-pra-%s-%s/pra_val_ablate_classunits_%s_worstba_%d.npz'
% (args.model, args.dataset, classlabels[classnum],
len(unitlist))))
pbar.print('%s: best %d %.3f vs worst N %.3f' %
(classlabels[classnum], num_best,
testba[classnum] - val_baseline_ba[classnum],
testba2[classnum] - val_baseline_ba[classnum]))
