def contaminate(self, selected_agents, selected_cells):
""" both arguments have same length. If an agent with sensitivity > 0 is in the same cell
than an agent with contagiousity > 0: possibility of contagion """
t_start = time()
i = 0
t0 = time()
selected_unsafeties = self.unsafeties[selected_cells]
selected_agents = selected_agents.astype(cp.uint32)
selected_states = self.current_state_ids[selected_agents]
selected_contagiousities = self.unique_contagiousities[selected_states]
selected_sensitivities = self.unique_sensitivities[selected_states]
print(f'ttt first part contaminate: {time() - t0}')
# Find cells where max contagiousity == 0 (no contagiousity can happen there)
t0 = time()
cont_sens = cp.multiply(selected_contagiousities, selected_sensitivities)
print(f'ttt group max sensitivities: {time() - t0}')
# Combine them
if cp.max(cont_sens) == 0:
return
t0 = time()
mask_zero = (cont_sens > 0)
selected_agents = selected_agents[mask_zero]
selected_contagiousities = selected_contagiousities[mask_zero]
selected_sensitivities = selected_sensitivities[mask_zero]
selected_cells = selected_cells[mask_zero]
selected_unsafeties = selected_unsafeties[mask_zero]
print(f'ttt mask zero all: {time() - t0}')
# Compute proportion (contagious agent) / (non contagious agent) by cell
t0 = time()
_, n_contagious_by_cell = cp.unique(selected_cells[selected_contagiousities > 0], return_counts=True)
_, n_non_contagious_by_cell = cp.unique(selected_cells[selected_contagiousities == 0], return_counts=True)
print(f'ttt non contagious: {time() - t0}')
i += 1
t0 = time()
p_contagious = cp.divide(n_contagious_by_cell, n_non_contagious_by_cell)
n_selected_agents = selected_agents.shape[0]
print(f'ttt p_contagious: {time() - t0}')
if self.verbose > 1:
print(f'{n_selected_agents} selected agents after removing cells with max sensitivity or max contagiousity==0')
if n_selected_agents == 0:
return
# Find for each cell which agent has the max contagiousity inside (it will be the contaminating agent)
t0 = time()
max_contagiousities, mask_max_contagiousities = group_max(data=selected_contagiousities, groups=selected_cells)
print(f'ttt max contagious: {time() - t0}')
t0 = time()
infecting_agents = selected_agents[mask_max_contagiousities]
selected_contagiousities = selected_contagiousities[mask_max_contagiousities]
print(f'ttt mask max contagious: {time() - t0}')
# Select agents that can be potentially infected ("pinfected") and corresponding variables
t0 = time()
pinfected_mask = (selected_sensitivities > 0)
pinfected_agents = selected_agents[pinfected_mask]
selected_sensitivities = selected_sensitivities[pinfected_mask]
selected_unsafeties = selected_unsafeties[pinfected_mask]
selected_cells = selected_cells[pinfected_mask]
print(f'ttt p_infected_mask: {time() - t0}')
# Group `selected_cells` and expand `infecting_agents` and `selected_contagiousities` accordingly
# There is one and only one infecting agent by pinselected_agentsfected_cell so #`counts` == #`infecting_agents`
t0 = time()
_, inverse = cp.unique(selected_cells, return_inverse=True)
print(f'ttt inverse select cell: {time() - t0}')
# TODO: ACHTUNG: count repeat replace by inverse here
t0 = time()
infecting_agents = infecting_agents[inverse]
selected_contagiousities = selected_contagiousities[inverse]
p_contagious = p_contagious[inverse]
print(f'ttt p_contagious inverse: {time() - t0}')
# Compute contagions
t0 = time()
res = cp.multiply(selected_contagiousities, selected_sensitivities)
res = cp.multiply(res, selected_unsafeties)
print(f'ttt cp.multiply: {time() - t0}')
# Modifiy probas contamination according to `p_contagious`
t0 = time()
mask_p = (p_contagious < 1)
res[mask_p] = cp.multiply(res[mask_p], p_contagious[mask_p])
res[~mask_p] = 1 - cp.divide(1 - res[~mask_p], p_contagious[~mask_p])
print(f'ttt res mask p: {time() - t0}')
t0 = time()
draw = cp.random.uniform(size=infecting_agents.shape[0])
draw = (draw < res)
infecting_agents = infecting_agents[draw]
infected_agents = pinfected_agents[draw]
n_infected_agents = infected_agents.shape[0]
print(f'ttt n_infected draw: {time() - t0}')
if self.verbose > 1:
print(f'Infecting and infected agents should be all different, are they? {((infecting_agents == infected_agents).sum() == 0)}')
print(f'Number of infected agents: {n_infected_agents}')
t0 = time()
self.current_state_ids[infected_agents] = self.least_state_ids[infected_agents]
self.current_state_durations[infected_agents] = 0
self.n_infected_period += n_infected_agents
self.infecting_agents = append(self.infecting_agents, infecting_agents)
self.infected_agents = append(self.infected_agents, infected_agents)
self.infected_periods = append(self.infected_periods, cp.multiply(cp.ones(n_infected_agents), self.current_period))
print(f'ttt final: {time() - t0}')
print(f'contaminate computed in {time() - t_start}')
def contaminate(self, selected_agents, selected_cells, prop_cont_factor=10, p_mask=0, family=False):
""" both arguments have same length. If an agent with sensitivity > 0 is in the same cell
than an agent with contagiousity > 0: possibility of contagion
prop_cont_factor: influence of the proportion of contagious people in a cell on contagion risk"""
t0 = time()
order_cells = np.argsort(selected_cells, kind='heapsort')
selected_cells = np.sort(selected_cells, kind='heapsort').astype(np.uint32)
# Sort other datas
selected_unsafeties = self.unsafeties[selected_cells]
selected_agents = selected_agents[order_cells].astype(np.uint32)
selected_states = self.current_state_ids[selected_agents]
selected_contagiousities = self.unique_contagiousities[selected_states]
if p_mask > 0:
pos_contagiousities = np.where(selected_contagiousities > 0)[0]
n_switchoff = int(pos_contagiousities.shape[0] * p_mask)
to_switchoff = np.random.choice(pos_contagiousities, size=n_switchoff, replace=False)
selected_contagiousities[to_switchoff] = 0
selected_sensitivities = self.unique_sensitivities[selected_states]
# Find cells where max contagiousity == 0 (no contagiousity can happen there)
max_contagiousities, _ = group_max(data=selected_contagiousities, groups=selected_cells)
if self.verbose > 1:
print(f'{max_contagiousities[max_contagiousities > 0].shape[0]} cells with contagious agent(s)')
# Find cells where max sensitivitity == 0 (no contagiousity can happen there)
max_sensitivities, _ = group_max(data=selected_sensitivities, groups=selected_cells)
# Combine them
mask_zero = ((max_contagiousities > 0) & (max_sensitivities > 0))
_, count = np.unique(selected_cells, return_counts=True)
mask_zero = np.repeat(mask_zero, count)
# select agents being on cells with max contagiousity and max sensitivity > 0 (and their corresponding data)
selected_agents = selected_agents[mask_zero]
selected_contagiousities = selected_contagiousities[mask_zero]
selected_sensitivities = selected_sensitivities[mask_zero]
selected_cells = selected_cells[mask_zero]
selected_unsafeties = selected_unsafeties[mask_zero]
# Compute proportion (contagious agent) / (non contagious agent) by cell
_, n_contagious_by_cell = np.unique(selected_cells[selected_contagiousities > 0], return_counts=True)
_, n_non_contagious_by_cell = np.unique(selected_cells[selected_contagiousities == 0], return_counts=True)
p_contagious = np.divide(n_contagious_by_cell, n_non_contagious_by_cell)
n_selected_agents = selected_agents.shape[0]
if self.verbose > 1:
print(f'{n_selected_agents} selected agents after removing cells with max sensitivity or max contagiousity==0')
if n_selected_agents == 0:
return
# Find for each cell which agent has the max contagiousity inside (it will be the contaminating agent)
max_contagiousities, mask_max_contagiousities = group_max(data=selected_contagiousities, groups=selected_cells)
infecting_agents = selected_agents[mask_max_contagiousities]
selected_contagiousities = selected_contagiousities[mask_max_contagiousities]
# Select agents that can be potentially infected ("pinfected") and corresponding variables
pinfected_mask = (selected_sensitivities > 0)
pinfected_agents = selected_agents[pinfected_mask]
selected_sensitivities = selected_sensitivities[pinfected_mask]
selected_unsafeties = selected_unsafeties[pinfected_mask]
selected_cells = selected_cells[pinfected_mask]
if self.verbose > 1:
print(f'selected_contagiousities: {selected_contagiousities}')
print(f'selected_sensitivities: {selected_sensitivities}')
print(f'selected_unsafeties: {selected_unsafeties}')
# Group `selected_cells` and expand `infecting_agents` and `selected_contagiousities` accordingly
# There is one and only one infecting agent by pinselected_agentsfected_cell so #`counts` == #`infecting_agents`
_, counts = np.unique(selected_cells, return_counts=True)
infecting_agents = np.repeat(infecting_agents, counts)
selected_contagiousities = np.repeat(selected_contagiousities, counts)
p_contagious = np.repeat(p_contagious, counts)
# Compute contagions
res = np.multiply(selected_contagiousities, selected_sensitivities)
res = np.multiply(res, selected_unsafeties)
# Modifiy probas contamination according to `p_contagious`
"""
mask_p = (p_contagious < 1)
res[mask_p] = np.multiply(res[mask_p], p_contagious[mask_p])
res[~mask_p] = 1 - np.divide(1 - res[~mask_p], p_contagious[~mask_p])
"""
draw = np.random.uniform(size=infecting_agents.shape[0])
if family:
draw = np.zeros(infecting_agents.shape[0])
draw = (draw < res)
"""
mask_p = (p_contagious < 1)
res[mask_p] = np.multiply(res[mask_p], p_contagious[mask_p])
res[~mask_p] = 1 - np.divide(1 - res[~mask_p], p_contagious[~mask_p])
"""
infecting_agents = infecting_agents[draw]
infected_agents = pinfected_agents[draw]
n_infected_agents = infected_agents.shape[0]
"""
if self.verbose > 1:
print(f'Infecting and infected agents should be all different, are they? {((infecting_agents == infected_agents).sum() == 0)}')
print(f'Number of infected agents: {n_infected_agents}')
"""
# self.current_state_ids[infected_agents] = self.least_state_ids[infected_agents]
self.current_state_ids[infected_agents] = 1
self.current_state_ids[infected_agents] = self.least_state_ids[infected_agents]
self.current_state_ids[infected_agents] = self.least_state_ids[infected_agents]
self.current_state_durations[infected_agents] = 0
self.n_infected_period += n_infected_agents
self.infecting_agents = np.append(self.infecting_agents, infecting_agents)
self.infected_agents = np.append(self.infected_agents, infected_agents)
self.infected_periods = np.append(self.infected_periods, np.repeat([self.current_period], n_infected_agents))
def contaminate(self, selected_agents, selected_cells):
""" both arguments have same length. If an agent with sensitivity > 0 is in the same cell
than an agent with contagiousity > 0: possibility of contagion """
order_cells = np.argsort(selected_cells, kind='heapsort')
selected_cells = np.sort(selected_cells,
kind='heapsort').astype(np.uint32)
# Sort other datas
selected_unsafeties = self.unsafeties[selected_cells]
selected_agents = selected_agents[order_cells].astype(np.uint32)
selected_states = self.current_state_ids[selected_agents]
selected_contagiousities = self.unique_contagiousities[selected_states]
selected_sensitivities = self.unique_sensitivities[selected_states]
# Find cells where max contagiousity == 0 (no contagiousity can happen there)
max_contagiousities, _ = group_max(data=selected_contagiousities,
groups=selected_cells)
if self.verbose > 1:
print(
f'{max_contagiousities[max_contagiousities > 0].shape[0]} cells with contagious agent(s)'
)
# Find cells where max sensitivitity == 0 (no contagiousity can happen there)
max_sensitivities, _ = group_max(data=selected_sensitivities,
groups=selected_cells)
# Combine them
mask_zero = (np.multiply(max_contagiousities, max_sensitivities) > 0)
_, counts = np.unique(selected_cells, return_counts=True)
mask_zero = np.repeat(mask_zero, counts)
# select agents being on cells with max contagiousity and max sensitivity > 0 (and their corresponding data)
selected_agents = selected_agents[mask_zero]
selected_contagiousities = selected_contagiousities[mask_zero]
selected_sensitivities = selected_sensitivities[mask_zero]
selected_cells = selected_cells[mask_zero]
selected_unsafeties = selected_unsafeties[mask_zero]
n_selected_agents = selected_agents.shape[0]
if self.verbose > 1:
print(
f'{n_selected_agents} selected agents after removing cells with max sensitivity or max contagiousity==0'
)
if n_selected_agents == 0:
return
# Find for each cell which agent has the max contagiousity inside (it will be the contaminating agent)
max_contagiousities, mask_max_contagiousities = group_max(
data=selected_contagiousities, groups=selected_cells)
infecting_agents = selected_agents[mask_max_contagiousities]
selected_contagiousities = selected_contagiousities[
mask_max_contagiousities]
# Select agents that can be potentially infected ("pinfected") and corresponding variables
pinfected_mask = (selected_sensitivities > 0)
pinfected_agents = selected_agents[pinfected_mask]
selected_sensitivities = selected_sensitivities[pinfected_mask]
selected_unsafeties = selected_unsafeties[pinfected_mask]
selected_cells = selected_cells[pinfected_mask]
# Group `selected_cells` and expand `infecting_agents` and `selected_contagiousities` accordingly
# There is one and only one infecting agent by pinselected_agentsfected_cell so #`counts` == #`infecting_agents`
_, counts = np.unique(selected_cells, return_counts=True)
infecting_agents = np.repeat(infecting_agents, counts)
selected_contagiousities = np.repeat(selected_contagiousities, counts)
# Compute contagions
res = np.multiply(selected_contagiousities, selected_sensitivities)
print(
f'DEBUG: res.shape: {res.shape}, selected_unsafeties.shape: {selected_unsafeties.shape}'
)
res = np.multiply(res, selected_unsafeties)
draw = np.random.uniform(size=infecting_agents.shape[0])
draw = (draw < res)
print(
f'DEBUG: draw.shape: {draw.shape}, infecting_agents.shape: {infecting_agents.shape}'
)
infecting_agents = infecting_agents[draw]
infected_agents = pinfected_agents[draw]
n_infected_agents = infected_agents.shape[0]
if self.verbose > 1:
print(
f'Infecting and infected agents should be all different, are they? {((infecting_agents == infected_agents).sum() == 0)}'
)
print(f'Number of infected agents: {n_infected_agents}')
self.current_state_ids[infected_agents] = self.least_state_ids[
infected_agents]
self.current_state_durations[infected_agents] = 0
self.n_infected_period += n_infected_agents
self.infecting_agents = np.append(self.infecting_agents,
infecting_agents)
self.infected_agents = np.append(self.infected_agents, infected_agents)
self.infected_periods = np.append(
self.infected_periods,
np.repeat([self.current_period], n_infected_agents))
def main():
fmoment = int(time.time())
args = parse_args()
norm = args.norm
backbone = args.backbone
pretrained = args.pretrained
lossfunc = args.loss
size = args.size
pk = args.pk
nk = args.nk
n_epoch = args.n_epoch
gpu = args.gpu
test_every = args.test_every
ckpt = args.ckpt
print(
'norm=%s backbone=%s pretrained=%s lossfunc=%s size=%s pk=%d nk=%d epoch=%d gpu=%d test_every=%d ckpt=%s'
% (norm, backbone, pretrained, lossfunc, size, pk, nk, n_epoch, gpu,
test_every, ckpt))
if backbone == 'resnet18':
model = resnet18.resnet18(norm=norm).cuda(device=gpu)
if pretrained == 'pretrained':
ckpt_dict = torch.load('resnet18-pretrained.pth')
model_dict = model.state_dict()
ckpt_dict = {k: v for k, v in ckpt_dict.items() if k in model_dict}
model_dict.update(ckpt_dict)
model.load_state_dict(model_dict)
if lossfunc == 'CE':
criterion = nn.CrossEntropyLoss().cuda(device=gpu)
elif lossfunc == 'Focal':
criterion = FocalLoss(class_num=2, gpu=gpu).cuda(device=gpu)
for m in model.modules():
if isinstance(m, nn.Linear):
nn.init.constant_(m.bias, -math.log(99))
elif lossfunc == 'BCE':
criterion = BCE(class_num=2, gpu=gpu).cuda(device=gpu)
optimizer = optim.Adam(model.parameters(), lr=1e-4, weight_decay=1e-4)
cudnn.benchmark = True
train_trans = transforms.Compose([
transforms.RandomHorizontalFlip(p=0.5),
transforms.RandomVerticalFlip(p=0.5),
transforms.ToTensor(),
transforms.Normalize(mean=[0.2005, 0.1490, 0.1486],
std=[0.1445, 0.1511, 0.0967])
])
infer_trans = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=[0.2005, 0.1490, 0.1486],
std=[0.1445, 0.1511, 0.0967])
])
train_dset = XDataset('train-%s.lib' % size,
train_trans=train_trans,
infer_trans=infer_trans)
train_loader = torch.utils.data.DataLoader(train_dset,
batch_size=64,
shuffle=False,
pin_memory=True)
test_dset = XDataset('test-%s.lib' % size,
train_trans=train_trans,
infer_trans=infer_trans)
test_loader = torch.utils.data.DataLoader(test_dset,
batch_size=128,
shuffle=False,
pin_memory=True)
if ckpt != 'none':
checkpoint = torch.load(ckpt)
start = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
best_f1 = checkpoint['best_f1']
optimizer.load_state_dict(checkpoint['optimizer'])
if not os.path.exists(
'logs/Training_%s_%s_%s_%s_%s_%d_%d_%d.csv' %
(norm, backbone, pretrained, lossfunc, size, pk, nk, fmoment)):
fconv = open(
'logs/Training_%s_%s_%s_%s_%s_%d_%d_%d.csv' %
(norm, backbone, pretrained, lossfunc, size, pk, nk, fmoment),
'w')
fconv.write('time,epoch,loss,error\n')
fconv.write('%d,0,0,0\n' % fmoment)
fconv.close()
if not os.path.exists(
'logs/Testing_%s_%s_%s_%s_%s_%d_%d_%d.csv' %
(norm, backbone, pretrained, lossfunc, size, pk, nk, fmoment)):
fconv = open(
'logs/Testing_%s_%s_%s_%s_%s_%d_%d_%d.csv' %
(norm, backbone, pretrained, lossfunc, size, pk, nk, fmoment),
'w')
fconv.write('time,epoch,loss,error,tp,tn,fp,fn,f1,S\n')
fconv.write('%d,0,0,0\n' % fmoment)
fconv.close()
else:
start = 0
best_f1 = 0
fconv = open(
'logs/Training_%s_%s_%s_%s_%s_%d_%d_%d.csv' %
(norm, backbone, pretrained, lossfunc, size, pk, nk, fmoment), 'w')
fconv.write('time,epoch,loss,error\n')
fconv.write('%d,0,0,0\n' % fmoment)
fconv.close()
fconv = open(
'logs/Testing_%s_%s_%s_%s_%s_%d_%d_%d.csv' %
(norm, backbone, pretrained, lossfunc, size, pk, nk, fmoment), 'w')
fconv.write('time,epoch,loss,error,tp,tn,fp,fn,f1,S\n')
fconv.write('%d,0,0,0\n' % fmoment)
fconv.close()
for epoch in range(start, n_epoch):
train_dset.setmode(1)
_, probs = inference(epoch, train_loader, model, criterion, gpu)
# torch.save(probs,'probs/train-%d.pth'%(epoch+1))
probs1 = probs[:train_dset.plen]
probs0 = probs[train_dset.plen:]
topk1 = np.array(
group_argtopk(np.array(train_dset.slideIDX[:train_dset.plen]),
probs1, pk))
topk0 = np.array(
group_argtopk(np.array(train_dset.slideIDX[train_dset.plen:]),
probs0, nk)) + train_dset.plen
topk = np.append(topk1, topk0).tolist()
# torch.save(topk,'topk/train-%d.pth'%(epoch+1))
# maxs = group_max(np.array(train_dset.slideIDX), probs, len(train_dset.targets))
# torch.save(maxs, 'maxs/%d.pth'%(epoch+1))
sf(topk)
train_dset.maketraindata(topk)
train_dset.setmode(2)
loss, err = train(train_loader, model, criterion, optimizer, gpu)
moment = time.time()
writecsv([moment, epoch + 1, loss, err],
'logs/Training_%s_%s_%s_%s_%s_%d_%d_%d.csv' %
(norm, backbone, pretrained, lossfunc, size, pk, nk, fmoment))
print('Training epoch=%d, loss=%.5f, error=%.5f' %
(epoch + 1, loss, err))
if (epoch + 1) % test_every == 0:
test_dset.setmode(1)
loss, probs = inference(epoch, test_loader, model, criterion, gpu)
# torch.save(probs,'probs/test-%d.pth'%(epoch+1))
# topk = group_argtopk(np.array(test_dset.slideIDX), probs, pk)
# torch.save(topk, 'topk/test-%d.pth'%(epoch+1))
maxs = group_max(
np.array(test_dset.slideIDX), probs,
len(test_dset.targets)) #è¿åæ¯ä¸ªåççæ大æ?ç
# torch.save(maxs, 'maxs/test-%d.pth'%(epoch+1))
pred = [1 if x >= 0.5 else 0 for x in maxs]
tp, tn, fp, fn = tfpn(pred, test_dset.targets)
err = calc_err(pred, test_dset.targets)
S, f1 = score(tp, tn, fp, fn)
moment = time.time()
writecsv(
[moment, epoch + 1, loss, err, tp, tn, fp, fn, f1, S],
'logs/Testing_%s_%s_%s_%s_%s_%d_%d_%d.csv' %
(norm, backbone, pretrained, lossfunc, size, pk, nk, fmoment))
print('Testing epoch=%d, loss=%.5f, error=%.5f' %
(epoch + 1, loss, err))
#Save best model
if f1 >= best_f1:
best_f1 = f1
obj = {
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_f1': best_f1,
'optimizer': optimizer.state_dict()
}
torch.save(
obj, 'ckpt_%s_%s_%s_%s_%s_%d_%d_%d.pth' %
(norm, backbone, pretrained, lossfunc, size, pk, nk,
fmoment))