def contaminate(self, selected_agents, selected_cells): """ both arguments have same length. If an agent with sensitivity > 0 is in the same cell than an agent with contagiousity > 0: possibility of contagion """ t_start = time() i = 0 t0 = time() selected_unsafeties = self.unsafeties[selected_cells] selected_agents = selected_agents.astype(cp.uint32) selected_states = self.current_state_ids[selected_agents] selected_contagiousities = self.unique_contagiousities[selected_states] selected_sensitivities = self.unique_sensitivities[selected_states] print(f'ttt first part contaminate: {time() - t0}') # Find cells where max contagiousity == 0 (no contagiousity can happen there) t0 = time() cont_sens = cp.multiply(selected_contagiousities, selected_sensitivities) print(f'ttt group max sensitivities: {time() - t0}') # Combine them if cp.max(cont_sens) == 0: return t0 = time() mask_zero = (cont_sens > 0) selected_agents = selected_agents[mask_zero] selected_contagiousities = selected_contagiousities[mask_zero] selected_sensitivities = selected_sensitivities[mask_zero] selected_cells = selected_cells[mask_zero] selected_unsafeties = selected_unsafeties[mask_zero] print(f'ttt mask zero all: {time() - t0}') # Compute proportion (contagious agent) / (non contagious agent) by cell t0 = time() _, n_contagious_by_cell = cp.unique(selected_cells[selected_contagiousities > 0], return_counts=True) _, n_non_contagious_by_cell = cp.unique(selected_cells[selected_contagiousities == 0], return_counts=True) print(f'ttt non contagious: {time() - t0}') i += 1 t0 = time() p_contagious = cp.divide(n_contagious_by_cell, n_non_contagious_by_cell) n_selected_agents = selected_agents.shape[0] print(f'ttt p_contagious: {time() - t0}') if self.verbose > 1: print(f'{n_selected_agents} selected agents after removing cells with max sensitivity or max contagiousity==0') if n_selected_agents == 0: return # Find for each cell which agent has the max contagiousity inside (it will be the contaminating agent) t0 = time() max_contagiousities, mask_max_contagiousities = group_max(data=selected_contagiousities, groups=selected_cells) print(f'ttt max contagious: {time() - t0}') t0 = time() infecting_agents = selected_agents[mask_max_contagiousities] selected_contagiousities = selected_contagiousities[mask_max_contagiousities] print(f'ttt mask max contagious: {time() - t0}') # Select agents that can be potentially infected ("pinfected") and corresponding variables t0 = time() pinfected_mask = (selected_sensitivities > 0) pinfected_agents = selected_agents[pinfected_mask] selected_sensitivities = selected_sensitivities[pinfected_mask] selected_unsafeties = selected_unsafeties[pinfected_mask] selected_cells = selected_cells[pinfected_mask] print(f'ttt p_infected_mask: {time() - t0}') # Group `selected_cells` and expand `infecting_agents` and `selected_contagiousities` accordingly # There is one and only one infecting agent by pinselected_agentsfected_cell so #`counts` == #`infecting_agents` t0 = time() _, inverse = cp.unique(selected_cells, return_inverse=True) print(f'ttt inverse select cell: {time() - t0}') # TODO: ACHTUNG: count repeat replace by inverse here t0 = time() infecting_agents = infecting_agents[inverse] selected_contagiousities = selected_contagiousities[inverse] p_contagious = p_contagious[inverse] print(f'ttt p_contagious inverse: {time() - t0}') # Compute contagions t0 = time() res = cp.multiply(selected_contagiousities, selected_sensitivities) res = cp.multiply(res, selected_unsafeties) print(f'ttt cp.multiply: {time() - t0}') # Modifiy probas contamination according to `p_contagious` t0 = time() mask_p = (p_contagious < 1) res[mask_p] = cp.multiply(res[mask_p], p_contagious[mask_p]) res[~mask_p] = 1 - cp.divide(1 - res[~mask_p], p_contagious[~mask_p]) print(f'ttt res mask p: {time() - t0}') t0 = time() draw = cp.random.uniform(size=infecting_agents.shape[0]) draw = (draw < res) infecting_agents = infecting_agents[draw] infected_agents = pinfected_agents[draw] n_infected_agents = infected_agents.shape[0] print(f'ttt n_infected draw: {time() - t0}') if self.verbose > 1: print(f'Infecting and infected agents should be all different, are they? {((infecting_agents == infected_agents).sum() == 0)}') print(f'Number of infected agents: {n_infected_agents}') t0 = time() self.current_state_ids[infected_agents] = self.least_state_ids[infected_agents] self.current_state_durations[infected_agents] = 0 self.n_infected_period += n_infected_agents self.infecting_agents = append(self.infecting_agents, infecting_agents) self.infected_agents = append(self.infected_agents, infected_agents) self.infected_periods = append(self.infected_periods, cp.multiply(cp.ones(n_infected_agents), self.current_period)) print(f'ttt final: {time() - t0}') print(f'contaminate computed in {time() - t_start}')
def contaminate(self, selected_agents, selected_cells, prop_cont_factor=10, p_mask=0, family=False): """ both arguments have same length. If an agent with sensitivity > 0 is in the same cell than an agent with contagiousity > 0: possibility of contagion prop_cont_factor: influence of the proportion of contagious people in a cell on contagion risk""" t0 = time() order_cells = np.argsort(selected_cells, kind='heapsort') selected_cells = np.sort(selected_cells, kind='heapsort').astype(np.uint32) # Sort other datas selected_unsafeties = self.unsafeties[selected_cells] selected_agents = selected_agents[order_cells].astype(np.uint32) selected_states = self.current_state_ids[selected_agents] selected_contagiousities = self.unique_contagiousities[selected_states] if p_mask > 0: pos_contagiousities = np.where(selected_contagiousities > 0)[0] n_switchoff = int(pos_contagiousities.shape[0] * p_mask) to_switchoff = np.random.choice(pos_contagiousities, size=n_switchoff, replace=False) selected_contagiousities[to_switchoff] = 0 selected_sensitivities = self.unique_sensitivities[selected_states] # Find cells where max contagiousity == 0 (no contagiousity can happen there) max_contagiousities, _ = group_max(data=selected_contagiousities, groups=selected_cells) if self.verbose > 1: print(f'{max_contagiousities[max_contagiousities > 0].shape[0]} cells with contagious agent(s)') # Find cells where max sensitivitity == 0 (no contagiousity can happen there) max_sensitivities, _ = group_max(data=selected_sensitivities, groups=selected_cells) # Combine them mask_zero = ((max_contagiousities > 0) & (max_sensitivities > 0)) _, count = np.unique(selected_cells, return_counts=True) mask_zero = np.repeat(mask_zero, count) # select agents being on cells with max contagiousity and max sensitivity > 0 (and their corresponding data) selected_agents = selected_agents[mask_zero] selected_contagiousities = selected_contagiousities[mask_zero] selected_sensitivities = selected_sensitivities[mask_zero] selected_cells = selected_cells[mask_zero] selected_unsafeties = selected_unsafeties[mask_zero] # Compute proportion (contagious agent) / (non contagious agent) by cell _, n_contagious_by_cell = np.unique(selected_cells[selected_contagiousities > 0], return_counts=True) _, n_non_contagious_by_cell = np.unique(selected_cells[selected_contagiousities == 0], return_counts=True) p_contagious = np.divide(n_contagious_by_cell, n_non_contagious_by_cell) n_selected_agents = selected_agents.shape[0] if self.verbose > 1: print(f'{n_selected_agents} selected agents after removing cells with max sensitivity or max contagiousity==0') if n_selected_agents == 0: return # Find for each cell which agent has the max contagiousity inside (it will be the contaminating agent) max_contagiousities, mask_max_contagiousities = group_max(data=selected_contagiousities, groups=selected_cells) infecting_agents = selected_agents[mask_max_contagiousities] selected_contagiousities = selected_contagiousities[mask_max_contagiousities] # Select agents that can be potentially infected ("pinfected") and corresponding variables pinfected_mask = (selected_sensitivities > 0) pinfected_agents = selected_agents[pinfected_mask] selected_sensitivities = selected_sensitivities[pinfected_mask] selected_unsafeties = selected_unsafeties[pinfected_mask] selected_cells = selected_cells[pinfected_mask] if self.verbose > 1: print(f'selected_contagiousities: {selected_contagiousities}') print(f'selected_sensitivities: {selected_sensitivities}') print(f'selected_unsafeties: {selected_unsafeties}') # Group `selected_cells` and expand `infecting_agents` and `selected_contagiousities` accordingly # There is one and only one infecting agent by pinselected_agentsfected_cell so #`counts` == #`infecting_agents` _, counts = np.unique(selected_cells, return_counts=True) infecting_agents = np.repeat(infecting_agents, counts) selected_contagiousities = np.repeat(selected_contagiousities, counts) p_contagious = np.repeat(p_contagious, counts) # Compute contagions res = np.multiply(selected_contagiousities, selected_sensitivities) res = np.multiply(res, selected_unsafeties) # Modifiy probas contamination according to `p_contagious` """ mask_p = (p_contagious < 1) res[mask_p] = np.multiply(res[mask_p], p_contagious[mask_p]) res[~mask_p] = 1 - np.divide(1 - res[~mask_p], p_contagious[~mask_p]) """ draw = np.random.uniform(size=infecting_agents.shape[0]) if family: draw = np.zeros(infecting_agents.shape[0]) draw = (draw < res) """ mask_p = (p_contagious < 1) res[mask_p] = np.multiply(res[mask_p], p_contagious[mask_p]) res[~mask_p] = 1 - np.divide(1 - res[~mask_p], p_contagious[~mask_p]) """ infecting_agents = infecting_agents[draw] infected_agents = pinfected_agents[draw] n_infected_agents = infected_agents.shape[0] """ if self.verbose > 1: print(f'Infecting and infected agents should be all different, are they? {((infecting_agents == infected_agents).sum() == 0)}') print(f'Number of infected agents: {n_infected_agents}') """ # self.current_state_ids[infected_agents] = self.least_state_ids[infected_agents] self.current_state_ids[infected_agents] = 1 self.current_state_ids[infected_agents] = self.least_state_ids[infected_agents] self.current_state_ids[infected_agents] = self.least_state_ids[infected_agents] self.current_state_durations[infected_agents] = 0 self.n_infected_period += n_infected_agents self.infecting_agents = np.append(self.infecting_agents, infecting_agents) self.infected_agents = np.append(self.infected_agents, infected_agents) self.infected_periods = np.append(self.infected_periods, np.repeat([self.current_period], n_infected_agents))
def contaminate(self, selected_agents, selected_cells): """ both arguments have same length. If an agent with sensitivity > 0 is in the same cell than an agent with contagiousity > 0: possibility of contagion """ order_cells = np.argsort(selected_cells, kind='heapsort') selected_cells = np.sort(selected_cells, kind='heapsort').astype(np.uint32) # Sort other datas selected_unsafeties = self.unsafeties[selected_cells] selected_agents = selected_agents[order_cells].astype(np.uint32) selected_states = self.current_state_ids[selected_agents] selected_contagiousities = self.unique_contagiousities[selected_states] selected_sensitivities = self.unique_sensitivities[selected_states] # Find cells where max contagiousity == 0 (no contagiousity can happen there) max_contagiousities, _ = group_max(data=selected_contagiousities, groups=selected_cells) if self.verbose > 1: print( f'{max_contagiousities[max_contagiousities > 0].shape[0]} cells with contagious agent(s)' ) # Find cells where max sensitivitity == 0 (no contagiousity can happen there) max_sensitivities, _ = group_max(data=selected_sensitivities, groups=selected_cells) # Combine them mask_zero = (np.multiply(max_contagiousities, max_sensitivities) > 0) _, counts = np.unique(selected_cells, return_counts=True) mask_zero = np.repeat(mask_zero, counts) # select agents being on cells with max contagiousity and max sensitivity > 0 (and their corresponding data) selected_agents = selected_agents[mask_zero] selected_contagiousities = selected_contagiousities[mask_zero] selected_sensitivities = selected_sensitivities[mask_zero] selected_cells = selected_cells[mask_zero] selected_unsafeties = selected_unsafeties[mask_zero] n_selected_agents = selected_agents.shape[0] if self.verbose > 1: print( f'{n_selected_agents} selected agents after removing cells with max sensitivity or max contagiousity==0' ) if n_selected_agents == 0: return # Find for each cell which agent has the max contagiousity inside (it will be the contaminating agent) max_contagiousities, mask_max_contagiousities = group_max( data=selected_contagiousities, groups=selected_cells) infecting_agents = selected_agents[mask_max_contagiousities] selected_contagiousities = selected_contagiousities[ mask_max_contagiousities] # Select agents that can be potentially infected ("pinfected") and corresponding variables pinfected_mask = (selected_sensitivities > 0) pinfected_agents = selected_agents[pinfected_mask] selected_sensitivities = selected_sensitivities[pinfected_mask] selected_unsafeties = selected_unsafeties[pinfected_mask] selected_cells = selected_cells[pinfected_mask] # Group `selected_cells` and expand `infecting_agents` and `selected_contagiousities` accordingly # There is one and only one infecting agent by pinselected_agentsfected_cell so #`counts` == #`infecting_agents` _, counts = np.unique(selected_cells, return_counts=True) infecting_agents = np.repeat(infecting_agents, counts) selected_contagiousities = np.repeat(selected_contagiousities, counts) # Compute contagions res = np.multiply(selected_contagiousities, selected_sensitivities) print( f'DEBUG: res.shape: {res.shape}, selected_unsafeties.shape: {selected_unsafeties.shape}' ) res = np.multiply(res, selected_unsafeties) draw = np.random.uniform(size=infecting_agents.shape[0]) draw = (draw < res) print( f'DEBUG: draw.shape: {draw.shape}, infecting_agents.shape: {infecting_agents.shape}' ) infecting_agents = infecting_agents[draw] infected_agents = pinfected_agents[draw] n_infected_agents = infected_agents.shape[0] if self.verbose > 1: print( f'Infecting and infected agents should be all different, are they? {((infecting_agents == infected_agents).sum() == 0)}' ) print(f'Number of infected agents: {n_infected_agents}') self.current_state_ids[infected_agents] = self.least_state_ids[ infected_agents] self.current_state_durations[infected_agents] = 0 self.n_infected_period += n_infected_agents self.infecting_agents = np.append(self.infecting_agents, infecting_agents) self.infected_agents = np.append(self.infected_agents, infected_agents) self.infected_periods = np.append( self.infected_periods, np.repeat([self.current_period], n_infected_agents))
def main(): fmoment = int(time.time()) args = parse_args() norm = args.norm backbone = args.backbone pretrained = args.pretrained lossfunc = args.loss size = args.size pk = args.pk nk = args.nk n_epoch = args.n_epoch gpu = args.gpu test_every = args.test_every ckpt = args.ckpt print( 'norm=%s backbone=%s pretrained=%s lossfunc=%s size=%s pk=%d nk=%d epoch=%d gpu=%d test_every=%d ckpt=%s' % (norm, backbone, pretrained, lossfunc, size, pk, nk, n_epoch, gpu, test_every, ckpt)) if backbone == 'resnet18': model = resnet18.resnet18(norm=norm).cuda(device=gpu) if pretrained == 'pretrained': ckpt_dict = torch.load('resnet18-pretrained.pth') model_dict = model.state_dict() ckpt_dict = {k: v for k, v in ckpt_dict.items() if k in model_dict} model_dict.update(ckpt_dict) model.load_state_dict(model_dict) if lossfunc == 'CE': criterion = nn.CrossEntropyLoss().cuda(device=gpu) elif lossfunc == 'Focal': criterion = FocalLoss(class_num=2, gpu=gpu).cuda(device=gpu) for m in model.modules(): if isinstance(m, nn.Linear): nn.init.constant_(m.bias, -math.log(99)) elif lossfunc == 'BCE': criterion = BCE(class_num=2, gpu=gpu).cuda(device=gpu) optimizer = optim.Adam(model.parameters(), lr=1e-4, weight_decay=1e-4) cudnn.benchmark = True train_trans = transforms.Compose([ transforms.RandomHorizontalFlip(p=0.5), transforms.RandomVerticalFlip(p=0.5), transforms.ToTensor(), transforms.Normalize(mean=[0.2005, 0.1490, 0.1486], std=[0.1445, 0.1511, 0.0967]) ]) infer_trans = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean=[0.2005, 0.1490, 0.1486], std=[0.1445, 0.1511, 0.0967]) ]) train_dset = XDataset('train-%s.lib' % size, train_trans=train_trans, infer_trans=infer_trans) train_loader = torch.utils.data.DataLoader(train_dset, batch_size=64, shuffle=False, pin_memory=True) test_dset = XDataset('test-%s.lib' % size, train_trans=train_trans, infer_trans=infer_trans) test_loader = torch.utils.data.DataLoader(test_dset, batch_size=128, shuffle=False, pin_memory=True) if ckpt != 'none': checkpoint = torch.load(ckpt) start = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) best_f1 = checkpoint['best_f1'] optimizer.load_state_dict(checkpoint['optimizer']) if not os.path.exists( 'logs/Training_%s_%s_%s_%s_%s_%d_%d_%d.csv' % (norm, backbone, pretrained, lossfunc, size, pk, nk, fmoment)): fconv = open( 'logs/Training_%s_%s_%s_%s_%s_%d_%d_%d.csv' % (norm, backbone, pretrained, lossfunc, size, pk, nk, fmoment), 'w') fconv.write('time,epoch,loss,error\n') fconv.write('%d,0,0,0\n' % fmoment) fconv.close() if not os.path.exists( 'logs/Testing_%s_%s_%s_%s_%s_%d_%d_%d.csv' % (norm, backbone, pretrained, lossfunc, size, pk, nk, fmoment)): fconv = open( 'logs/Testing_%s_%s_%s_%s_%s_%d_%d_%d.csv' % (norm, backbone, pretrained, lossfunc, size, pk, nk, fmoment), 'w') fconv.write('time,epoch,loss,error,tp,tn,fp,fn,f1,S\n') fconv.write('%d,0,0,0\n' % fmoment) fconv.close() else: start = 0 best_f1 = 0 fconv = open( 'logs/Training_%s_%s_%s_%s_%s_%d_%d_%d.csv' % (norm, backbone, pretrained, lossfunc, size, pk, nk, fmoment), 'w') fconv.write('time,epoch,loss,error\n') fconv.write('%d,0,0,0\n' % fmoment) fconv.close() fconv = open( 'logs/Testing_%s_%s_%s_%s_%s_%d_%d_%d.csv' % (norm, backbone, pretrained, lossfunc, size, pk, nk, fmoment), 'w') fconv.write('time,epoch,loss,error,tp,tn,fp,fn,f1,S\n') fconv.write('%d,0,0,0\n' % fmoment) fconv.close() for epoch in range(start, n_epoch): train_dset.setmode(1) _, probs = inference(epoch, train_loader, model, criterion, gpu) # torch.save(probs,'probs/train-%d.pth'%(epoch+1)) probs1 = probs[:train_dset.plen] probs0 = probs[train_dset.plen:] topk1 = np.array( group_argtopk(np.array(train_dset.slideIDX[:train_dset.plen]), probs1, pk)) topk0 = np.array( group_argtopk(np.array(train_dset.slideIDX[train_dset.plen:]), probs0, nk)) + train_dset.plen topk = np.append(topk1, topk0).tolist() # torch.save(topk,'topk/train-%d.pth'%(epoch+1)) # maxs = group_max(np.array(train_dset.slideIDX), probs, len(train_dset.targets)) # torch.save(maxs, 'maxs/%d.pth'%(epoch+1)) sf(topk) train_dset.maketraindata(topk) train_dset.setmode(2) loss, err = train(train_loader, model, criterion, optimizer, gpu) moment = time.time() writecsv([moment, epoch + 1, loss, err], 'logs/Training_%s_%s_%s_%s_%s_%d_%d_%d.csv' % (norm, backbone, pretrained, lossfunc, size, pk, nk, fmoment)) print('Training epoch=%d, loss=%.5f, error=%.5f' % (epoch + 1, loss, err)) if (epoch + 1) % test_every == 0: test_dset.setmode(1) loss, probs = inference(epoch, test_loader, model, criterion, gpu) # torch.save(probs,'probs/test-%d.pth'%(epoch+1)) # topk = group_argtopk(np.array(test_dset.slideIDX), probs, pk) # torch.save(topk, 'topk/test-%d.pth'%(epoch+1)) maxs = group_max( np.array(test_dset.slideIDX), probs, len(test_dset.targets)) #è¿åæ¯ä¸ªåççæ大æ?ç # torch.save(maxs, 'maxs/test-%d.pth'%(epoch+1)) pred = [1 if x >= 0.5 else 0 for x in maxs] tp, tn, fp, fn = tfpn(pred, test_dset.targets) err = calc_err(pred, test_dset.targets) S, f1 = score(tp, tn, fp, fn) moment = time.time() writecsv( [moment, epoch + 1, loss, err, tp, tn, fp, fn, f1, S], 'logs/Testing_%s_%s_%s_%s_%s_%d_%d_%d.csv' % (norm, backbone, pretrained, lossfunc, size, pk, nk, fmoment)) print('Testing epoch=%d, loss=%.5f, error=%.5f' % (epoch + 1, loss, err)) #Save best model if f1 >= best_f1: best_f1 = f1 obj = { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'best_f1': best_f1, 'optimizer': optimizer.state_dict() } torch.save( obj, 'ckpt_%s_%s_%s_%s_%s_%d_%d_%d.pth' % (norm, backbone, pretrained, lossfunc, size, pk, nk, fmoment))