# data parallel with multiple GPUs on a single node net = nn.DataParallel(net, device_ids=range(torch.cuda.device_count())) #-------------------------------------------------------------------------- # Setup the direction file and the surface file #-------------------------------------------------------------------------- dir_file = net_plotter.name_direction_file(args) # name the direction file if rank == 0: net_plotter.setup_direction(args, dir_file, net) surf_file = name_surface_file(args, dir_file) if rank == 0: setup_surface_file(args, surf_file, dir_file) # wait until master has setup the direction file and surface file mpi4pytorch.barrier(comm) # load directions d = net_plotter.load_directions(dir_file) # calculate the consine similarity of the two directions if len(d) == 2 and rank == 0: similarity = proj.cal_angle(proj.nplist_to_tensor(d[0]), proj.nplist_to_tensor(d[1])) print('cosine similarity between x-axis and y-axis: %f' % similarity) #-------------------------------------------------------------------------- # Setup dataloader #-------------------------------------------------------------------------- # download CIFAR10 if it does not exit if rank == 0 and args.dataset == 'cifar10': torchvision.datasets.CIFAR10(root=args.dataset + '/data', train=True, download=True)
# data parallel with multiple GPUs on a single node net = nn.DataParallel(net, device_ids=range(torch.cuda.device_count())) #-------------------------------------------------------------------------- # Setup the direction file and the surface file #-------------------------------------------------------------------------- dir_file = net_plotter.name_direction_file(args) # name the direction file if rank == 0: net_plotter.setup_direction(args, dir_file, net) surf_file = name_surface_file(args, dir_file) if rank == 0: setup_surface_file(args, surf_file, dir_file) # wait until master has setup the direction file and surface file mpi.barrier(comm) # load directions d = net_plotter.load_directions(dir_file) # calculate the consine similarity of the two directions if len(d) == 2 and rank == 0: similarity = proj.cal_angle(proj.nplist_to_tensor(d[0]), proj.nplist_to_tensor(d[1])) print('cosine similarity between x-axis and y-axis: %f' % similarity) #-------------------------------------------------------------------------- # Setup dataloader #-------------------------------------------------------------------------- # download CIFAR10 if it does not exit if rank == 0 and args.dataset == 'cifar10': torchvision.datasets.CIFAR10(root=args.dataset + '/data',
device_ids=range( min(args.ngpu, torch.cuda.device_count()))) #-------------------------------------------------------------------------- # Setup the direction file and the surface file #-------------------------------------------------------------------------- dir_file = net_plotter.name_direction_file(args) # name the direction file if rank == 0: net_plotter.setup_direction(args, dir_file, net) surf_file = name_surface_file(args, dir_file) if rank == 0: setup_surface_file(args, surf_file, dir_file) # wait until master has setup the direction file and surface file mpi4pytorch.barrier(comm) # load directions d = net_plotter.load_directions(dir_file) # calculate the consine similarity of the two directions if len(d) == 2 and rank == 0: similarity = proj.cal_angle(proj.nplist_to_tensor(d[0]), proj.nplist_to_tensor(d[1])) print('cosine similarity between x-axis and y-axis: %f' % similarity) #-------------------------------------------------------------------------- # Setup dataloader #-------------------------------------------------------------------------- # download CIFAR10 if it does not exit if rank == 0 and args.dataset == 'cifar10': torchvision.datasets.CIFAR10(root=args.dataset + '/data',